contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp

   1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This contains code to emit Builtin calls as LLVM code.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "CGCXXABI.h"
  14 #include "CGObjCRuntime.h"
  15 #include "CGOpenCLRuntime.h"
  16 #include "CGRecordLayout.h"
  17 #include "CodeGenFunction.h"
  18 #include "CodeGenModule.h"
  19 #include "ConstantEmitter.h"
  20 #include "PatternInit.h"
  21 #include "TargetInfo.h"
  22 #include "clang/AST/ASTContext.h"
  23 #include "clang/AST/Attr.h"
  24 #include "clang/AST/Decl.h"
  25 #include "clang/AST/OSLog.h"
  26 #include "clang/Basic/TargetBuiltins.h"
  27 #include "clang/Basic/TargetInfo.h"
  28 #include "clang/CodeGen/CGFunctionInfo.h"
  29 #include "llvm/ADT/SmallPtrSet.h"
  30 #include "llvm/ADT/StringExtras.h"
  31 #include "llvm/Analysis/ValueTracking.h"
  32 #include "llvm/IR/DataLayout.h"
  33 #include "llvm/IR/InlineAsm.h"
  34 #include "llvm/IR/Intrinsics.h"
  35 #include "llvm/IR/IntrinsicsAArch64.h"
  36 #include "llvm/IR/IntrinsicsAMDGPU.h"
  37 #include "llvm/IR/IntrinsicsARM.h"
  38 #include "llvm/IR/IntrinsicsBPF.h"
  39 #include "llvm/IR/IntrinsicsHexagon.h"
  40 #include "llvm/IR/IntrinsicsNVPTX.h"
  41 #include "llvm/IR/IntrinsicsPowerPC.h"
  42 #include "llvm/IR/IntrinsicsR600.h"
  43 #include "llvm/IR/IntrinsicsS390.h"
  44 #include "llvm/IR/IntrinsicsWebAssembly.h"
  45 #include "llvm/IR/IntrinsicsX86.h"
  46 #include "llvm/IR/MDBuilder.h"
  47 #include "llvm/IR/MatrixBuilder.h"
  48 #include "llvm/Support/ConvertUTF.h"
  49 #include "llvm/Support/ScopedPrinter.h"
  50 #include "llvm/Support/X86TargetParser.h"
  51 #include <sstream>
  52
  53 using namespace clang;
  54 using namespace CodeGen;
  55 using namespace llvm;
  56
  57 static
  58 int64_t clamp(int64_t Value, int64_t Low, int64_t High) {
  59   return std::min(High, std::max(Low, Value));
  60 }
  61
  62 static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
  63                              Align AlignmentInBytes) {
  64   ConstantInt *Byte;
  65   switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
  66   case LangOptions::TrivialAutoVarInitKind::Uninitialized:
  67     // Nothing to initialize.
  68     return;
  69   case LangOptions::TrivialAutoVarInitKind::Zero:
  70     Byte = CGF.Builder.getInt8(0x00);
  71     break;
  72   case LangOptions::TrivialAutoVarInitKind::Pattern: {
  73     llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(CGF.CGM.getLLVMContext());
  74     Byte = llvm::dyn_cast<llvm::ConstantInt>(
  75         initializationPatternFor(CGF.CGM, Int8));
  76     break;
  77   }
  78   }
  79   if (CGF.CGM.stopAutoInit())
  80     return;
  81   CGF.Builder.CreateMemSet(AI, Byte, Size, AlignmentInBytes);
  82 }
  83
  84 /// getBuiltinLibFunction - Given a builtin id for a function like
  85 /// "__builtin_fabsf", return a Function* for "fabsf".
  86 llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
  87                                                      unsigned BuiltinID) {
  88   assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
  89
  90   // Get the name, skip over the __builtin_ prefix (if necessary).
  91   StringRef Name;
  92   GlobalDecl D(FD);
  93
  94   // If the builtin has been declared explicitly with an assembler label,
  95   // use the mangled name. This differs from the plain label on platforms
  96   // that prefix labels.
  97   if (FD->hasAttr<AsmLabelAttr>())
  98     Name = getMangledName(D);
  99   else
 100     Name = Context.BuiltinInfo.getName(BuiltinID) + 10;
 101
 102   llvm::FunctionType *Ty =
 103     cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
 104
 105   return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
 106 }
 107
 108 /// Emit the conversions required to turn the given value into an
 109 /// integer of the given size.
 110 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
 111                         QualType T, llvm::IntegerType *IntType) {
 112   V = CGF.EmitToMemory(V, T);
 113
 114   if (V->getType()->isPointerTy())
 115     return CGF.Builder.CreatePtrToInt(V, IntType);
 116
 117   assert(V->getType() == IntType);
 118   return V;
 119 }
 120
 121 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
 122                           QualType T, llvm::Type *ResultType) {
 123   V = CGF.EmitFromMemory(V, T);
 124
 125   if (ResultType->isPointerTy())
 126     return CGF.Builder.CreateIntToPtr(V, ResultType);
 127
 128   assert(V->getType() == ResultType);
 129   return V;
 130 }
 131
 132 /// Utility to insert an atomic instruction based on Intrinsic::ID
 133 /// and the expression node.
 134 static Value *MakeBinaryAtomicValue(
 135     CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
 136     AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
 137   QualType T = E->getType();
 138   assert(E->getArg(0)->getType()->isPointerType());
 139   assert(CGF.getContext().hasSameUnqualifiedType(T,
 140                                   E->getArg(0)->getType()->getPointeeType()));
 141   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
 142
 143   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
 144   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
 145
 146   llvm::IntegerType *IntType =
 147     llvm::IntegerType::get(CGF.getLLVMContext(),
 148                            CGF.getContext().getTypeSize(T));
 149   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
 150
 151   llvm::Value *Args[2];
 152   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
 153   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
 154   llvm::Type *ValueType = Args[1]->getType();
 155   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
 156
 157   llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
 158       Kind, Args[0], Args[1], Ordering);
 159   return EmitFromInt(CGF, Result, T, ValueType);
 160 }
 161
 162 static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
 163   Value *Val = CGF.EmitScalarExpr(E->getArg(0));
 164   Value *Address = CGF.EmitScalarExpr(E->getArg(1));
 165
 166   // Convert the type of the pointer to a pointer to the stored type.
 167   Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
 168   Value *BC = CGF.Builder.CreateBitCast(
 169       Address, llvm::PointerType::getUnqual(Val->getType()), "cast");
 170   LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType());
 171   LV.setNontemporal(true);
 172   CGF.EmitStoreOfScalar(Val, LV, false);
 173   return nullptr;
 174 }
 175
 176 static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
 177   Value *Address = CGF.EmitScalarExpr(E->getArg(0));
 178
 179   LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType());
 180   LV.setNontemporal(true);
 181   return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
 182 }
 183
 184 static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
 185                                llvm::AtomicRMWInst::BinOp Kind,
 186                                const CallExpr *E) {
 187   return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));
 188 }
 189
 190 /// Utility to insert an atomic instruction based Intrinsic::ID and
 191 /// the expression node, where the return value is the result of the
 192 /// operation.
 193 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
 194                                    llvm::AtomicRMWInst::BinOp Kind,
 195                                    const CallExpr *E,
 196                                    Instruction::BinaryOps Op,
 197                                    bool Invert = false) {
 198   QualType T = E->getType();
 199   assert(E->getArg(0)->getType()->isPointerType());
 200   assert(CGF.getContext().hasSameUnqualifiedType(T,
 201                                   E->getArg(0)->getType()->getPointeeType()));
 202   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
 203
 204   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
 205   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
 206
 207   llvm::IntegerType *IntType =
 208     llvm::IntegerType::get(CGF.getLLVMContext(),
 209                            CGF.getContext().getTypeSize(T));
 210   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
 211
 212   llvm::Value *Args[2];
 213   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
 214   llvm::Type *ValueType = Args[1]->getType();
 215   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
 216   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
 217
 218   llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
 219       Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
 220   Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
 221   if (Invert)
 222     Result =
 223         CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
 224                                 llvm::ConstantInt::getAllOnesValue(IntType));
 225   Result = EmitFromInt(CGF, Result, T, ValueType);
 226   return RValue::get(Result);
 227 }
 228
 229 /// Utility to insert an atomic cmpxchg instruction.
 230 ///
 231 /// @param CGF The current codegen function.
 232 /// @param E   Builtin call expression to convert to cmpxchg.
 233 ///            arg0 - address to operate on
 234 ///            arg1 - value to compare with
 235 ///            arg2 - new value
 236 /// @param ReturnBool Specifies whether to return success flag of
 237 ///                   cmpxchg result or the old value.
 238 ///
 239 /// @returns result of cmpxchg, according to ReturnBool
 240 ///
 241 /// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
 242 /// invoke the function EmitAtomicCmpXchgForMSIntrin.
 243 static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
 244                                      bool ReturnBool) {
 245   QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
 246   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
 247   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
 248
 249   llvm::IntegerType *IntType = llvm::IntegerType::get(
 250       CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
 251   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
 252
 253   Value *Args[3];
 254   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
 255   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
 256   llvm::Type *ValueType = Args[1]->getType();
 257   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
 258   Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
 259
 260   Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
 261       Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
 262       llvm::AtomicOrdering::SequentiallyConsistent);
 263   if (ReturnBool)
 264     // Extract boolean success flag and zext it to int.
 265     return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
 266                                   CGF.ConvertType(E->getType()));
 267   else
 268     // Extract old value and emit it using the same type as compare value.
 269     return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
 270                        ValueType);
 271 }
 272
 273 /// This function should be invoked to emit atomic cmpxchg for Microsoft's
 274 /// _InterlockedCompareExchange* intrinsics which have the following signature:
 275 /// T _InterlockedCompareExchange(T volatile *Destination,
 276 ///                               T Exchange,
 277 ///                               T Comparand);
 278 ///
 279 /// Whereas the llvm 'cmpxchg' instruction has the following syntax:
 280 /// cmpxchg *Destination, Comparand, Exchange.
 281 /// So we need to swap Comparand and Exchange when invoking
 282 /// CreateAtomicCmpXchg. That is the reason we could not use the above utility
 283 /// function MakeAtomicCmpXchgValue since it expects the arguments to be
 284 /// already swapped.
 285
 286 static
 287 Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
 288     AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
 289   assert(E->getArg(0)->getType()->isPointerType());
 290   assert(CGF.getContext().hasSameUnqualifiedType(
 291       E->getType(), E->getArg(0)->getType()->getPointeeType()));
 292   assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
 293                                                  E->getArg(1)->getType()));
 294   assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
 295                                                  E->getArg(2)->getType()));
 296
 297   auto *Destination = CGF.EmitScalarExpr(E->getArg(0));
 298   auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
 299   auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
 300
 301   // For Release ordering, the failure ordering should be Monotonic.
 302   auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
 303                          AtomicOrdering::Monotonic :
 304                          SuccessOrdering;
 305
 306   auto *Result = CGF.Builder.CreateAtomicCmpXchg(
 307                    Destination, Comparand, Exchange,
 308                    SuccessOrdering, FailureOrdering);
 309   Result->setVolatile(true);
 310   return CGF.Builder.CreateExtractValue(Result, 0);
 311 }
 312
 313 static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,
 314     AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
 315   assert(E->getArg(0)->getType()->isPointerType());
 316
 317   auto *IntTy = CGF.ConvertType(E->getType());
 318   auto *Result = CGF.Builder.CreateAtomicRMW(
 319                    AtomicRMWInst::Add,
 320                    CGF.EmitScalarExpr(E->getArg(0)),
 321                    ConstantInt::get(IntTy, 1),
 322                    Ordering);
 323   return CGF.Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));
 324 }
 325
 326 static Value *EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E,
 327     AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
 328   assert(E->getArg(0)->getType()->isPointerType());
 329
 330   auto *IntTy = CGF.ConvertType(E->getType());
 331   auto *Result = CGF.Builder.CreateAtomicRMW(
 332                    AtomicRMWInst::Sub,
 333                    CGF.EmitScalarExpr(E->getArg(0)),
 334                    ConstantInt::get(IntTy, 1),
 335                    Ordering);
 336   return CGF.Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));
 337 }
 338
 339 // Build a plain volatile load.
 340 static Value *EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E) {
 341   Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
 342   QualType ElTy = E->getArg(0)->getType()->getPointeeType();
 343   CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(ElTy);
 344   llvm::Type *ITy =
 345       llvm::IntegerType::get(CGF.getLLVMContext(), LoadSize.getQuantity() * 8);
 346   Ptr = CGF.Builder.CreateBitCast(Ptr, ITy->getPointerTo());
 347   llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(Ptr, LoadSize);
 348   Load->setVolatile(true);
 349   return Load;
 350 }
 351
 352 // Build a plain volatile store.
 353 static Value *EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E) {
 354   Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
 355   Value *Value = CGF.EmitScalarExpr(E->getArg(1));
 356   QualType ElTy = E->getArg(0)->getType()->getPointeeType();
 357   CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(ElTy);
 358   llvm::Type *ITy =
 359       llvm::IntegerType::get(CGF.getLLVMContext(), StoreSize.getQuantity() * 8);
 360   Ptr = CGF.Builder.CreateBitCast(Ptr, ITy->getPointerTo());
 361   llvm::StoreInst *Store =
 362       CGF.Builder.CreateAlignedStore(Value, Ptr, StoreSize);
 363   Store->setVolatile(true);
 364   return Store;
 365 }
 366
 367 // Emit a simple mangled intrinsic that has 1 argument and a return type
 368 // matching the argument type. Depending on mode, this may be a constrained
 369 // floating-point intrinsic.
 370 static Value *emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
 371                                 const CallExpr *E, unsigned IntrinsicID,
 372                                 unsigned ConstrainedIntrinsicID) {
 373   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 374
 375   if (CGF.Builder.getIsFPConstrained()) {
 376     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
 377     return CGF.Builder.CreateConstrainedFPCall(F, { Src0 });
 378   } else {
 379     Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 380     return CGF.Builder.CreateCall(F, Src0);
 381   }
 382 }
 383
 384 // Emit an intrinsic that has 2 operands of the same type as its result.
 385 // Depending on mode, this may be a constrained floating-point intrinsic.
 386 static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
 387                                 const CallExpr *E, unsigned IntrinsicID,
 388                                 unsigned ConstrainedIntrinsicID) {
 389   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 390   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
 391
 392   if (CGF.Builder.getIsFPConstrained()) {
 393     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
 394     return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
 395   } else {
 396     Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 397     return CGF.Builder.CreateCall(F, { Src0, Src1 });
 398   }
 399 }
 400
 401 // Emit an intrinsic that has 3 operands of the same type as its result.
 402 // Depending on mode, this may be a constrained floating-point intrinsic.
 403 static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
 404                                  const CallExpr *E, unsigned IntrinsicID,
 405                                  unsigned ConstrainedIntrinsicID) {
 406   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 407   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
 408   llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
 409
 410   if (CGF.Builder.getIsFPConstrained()) {
 411     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
 412     return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
 413   } else {
 414     Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 415     return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
 416   }
 417 }
 418
 419 // Emit an intrinsic where all operands are of the same type as the result.
 420 // Depending on mode, this may be a constrained floating-point intrinsic.
 421 static Value *emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
 422                                                 unsigned IntrinsicID,
 423                                                 unsigned ConstrainedIntrinsicID,
 424                                                 llvm::Type *Ty,
 425                                                 ArrayRef<Value *> Args) {
 426   Function *F;
 427   if (CGF.Builder.getIsFPConstrained())
 428     F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Ty);
 429   else
 430     F = CGF.CGM.getIntrinsic(IntrinsicID, Ty);
 431
 432   if (CGF.Builder.getIsFPConstrained())
 433     return CGF.Builder.CreateConstrainedFPCall(F, Args);
 434   else
 435     return CGF.Builder.CreateCall(F, Args);
 436 }
 437
 438 // Emit a simple mangled intrinsic that has 1 argument and a return type
 439 // matching the argument type.
 440 static Value *emitUnaryBuiltin(CodeGenFunction &CGF,
 441                                const CallExpr *E,
 442                                unsigned IntrinsicID) {
 443   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 444
 445   Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 446   return CGF.Builder.CreateCall(F, Src0);
 447 }
 448
 449 // Emit an intrinsic that has 2 operands of the same type as its result.
 450 static Value *emitBinaryBuiltin(CodeGenFunction &CGF,
 451                                 const CallExpr *E,
 452                                 unsigned IntrinsicID) {
 453   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 454   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
 455
 456   Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 457   return CGF.Builder.CreateCall(F, { Src0, Src1 });
 458 }
 459
 460 // Emit an intrinsic that has 3 operands of the same type as its result.
 461 static Value *emitTernaryBuiltin(CodeGenFunction &CGF,
 462                                  const CallExpr *E,
 463                                  unsigned IntrinsicID) {
 464   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 465   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
 466   llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
 467
 468   Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 469   return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
 470 }
 471
 472 // Emit an intrinsic that has 1 float or double operand, and 1 integer.
 473 static Value *emitFPIntBuiltin(CodeGenFunction &CGF,
 474                                const CallExpr *E,
 475                                unsigned IntrinsicID) {
 476   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 477   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
 478
 479   Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 480   return CGF.Builder.CreateCall(F, {Src0, Src1});
 481 }
 482
 483 // Emit an intrinsic that has overloaded integer result and fp operand.
 484 static Value *
 485 emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E,
 486                                         unsigned IntrinsicID,
 487                                         unsigned ConstrainedIntrinsicID) {
 488   llvm::Type *ResultType = CGF.ConvertType(E->getType());
 489   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 490
 491   if (CGF.Builder.getIsFPConstrained()) {
 492     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID,
 493                                        {ResultType, Src0->getType()});
 494     return CGF.Builder.CreateConstrainedFPCall(F, {Src0});
 495   } else {
 496     Function *F =
 497         CGF.CGM.getIntrinsic(IntrinsicID, {ResultType, Src0->getType()});
 498     return CGF.Builder.CreateCall(F, Src0);
 499   }
 500 }
 501
 502 /// EmitFAbs - Emit a call to @llvm.fabs().
 503 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
 504   Function *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
 505   llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);
 506   Call->setDoesNotAccessMemory();
 507   return Call;
 508 }
 509
 510 /// Emit the computation of the sign bit for a floating point value. Returns
 511 /// the i1 sign bit value.
 512 static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
 513   LLVMContext &C = CGF.CGM.getLLVMContext();
 514
 515   llvm::Type *Ty = V->getType();
 516   int Width = Ty->getPrimitiveSizeInBits();
 517   llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
 518   V = CGF.Builder.CreateBitCast(V, IntTy);
 519   if (Ty->isPPC_FP128Ty()) {
 520     // We want the sign bit of the higher-order double. The bitcast we just
 521     // did works as if the double-double was stored to memory and then
 522     // read as an i128. The "store" will put the higher-order double in the
 523     // lower address in both little- and big-Endian modes, but the "load"
 524     // will treat those bits as a different part of the i128: the low bits in
 525     // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
 526     // we need to shift the high bits down to the low before truncating.
 527     Width >>= 1;
 528     if (CGF.getTarget().isBigEndian()) {
 529       Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
 530       V = CGF.Builder.CreateLShr(V, ShiftCst);
 531     }
 532     // We are truncating value in order to extract the higher-order
 533     // double, which we will be using to extract the sign from.
 534     IntTy = llvm::IntegerType::get(C, Width);
 535     V = CGF.Builder.CreateTrunc(V, IntTy);
 536   }
 537   Value *Zero = llvm::Constant::getNullValue(IntTy);
 538   return CGF.Builder.CreateICmpSLT(V, Zero);
 539 }
 540
 541 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
 542                               const CallExpr *E, llvm::Constant *calleeValue) {
 543   CGCallee callee = CGCallee::forDirect(calleeValue, GlobalDecl(FD));
 544   return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
 545 }
 546
 547 /// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
 548 /// depending on IntrinsicID.
 549 ///
 550 /// \arg CGF The current codegen function.
 551 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
 552 /// \arg X The first argument to the llvm.*.with.overflow.*.
 553 /// \arg Y The second argument to the llvm.*.with.overflow.*.
 554 /// \arg Carry The carry returned by the llvm.*.with.overflow.*.
 555 /// \returns The result (i.e. sum/product) returned by the intrinsic.
 556 static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
 557                                           const llvm::Intrinsic::ID IntrinsicID,
 558                                           llvm::Value *X, llvm::Value *Y,
 559                                           llvm::Value *&Carry) {
 560   // Make sure we have integers of the same width.
 561   assert(X->getType() == Y->getType() &&
 562          "Arguments must be the same type. (Did you forget to make sure both "
 563          "arguments have the same integer width?)");
 564
 565   Function *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
 566   llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
 567   Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
 568   return CGF.Builder.CreateExtractValue(Tmp, 0);
 569 }
 570
 571 static Value *emitRangedBuiltin(CodeGenFunction &CGF,
 572                                 unsigned IntrinsicID,
 573                                 int low, int high) {
 574     llvm::MDBuilder MDHelper(CGF.getLLVMContext());
 575     llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
 576     Function *F = CGF.CGM.getIntrinsic(IntrinsicID, {});
 577     llvm::Instruction *Call = CGF.Builder.CreateCall(F);
 578     Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
 579     return Call;
 580 }
 581
 582 namespace {
 583   struct WidthAndSignedness {
 584     unsigned Width;
 585     bool Signed;
 586   };
 587 }
 588
 589 static WidthAndSignedness
 590 getIntegerWidthAndSignedness(const clang::ASTContext &context,
 591                              const clang::QualType Type) {
 592   assert(Type->isIntegerType() && "Given type is not an integer.");
 593   unsigned Width = Type->isBooleanType()  ? 1
 594                    : Type->isExtIntType() ? context.getIntWidth(Type)
 595                                           : context.getTypeInfo(Type).Width;
 596   bool Signed = Type->isSignedIntegerType();
 597   return {Width, Signed};
 598 }
 599
 600 // Given one or more integer types, this function produces an integer type that
 601 // encompasses them: any value in one of the given types could be expressed in
 602 // the encompassing type.
 603 static struct WidthAndSignedness
 604 EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
 605   assert(Types.size() > 0 && "Empty list of types.");
 606
 607   // If any of the given types is signed, we must return a signed type.
 608   bool Signed = false;
 609   for (const auto &Type : Types) {
 610     Signed |= Type.Signed;
 611   }
 612
 613   // The encompassing type must have a width greater than or equal to the width
 614   // of the specified types.  Additionally, if the encompassing type is signed,
 615   // its width must be strictly greater than the width of any unsigned types
 616   // given.
 617   unsigned Width = 0;
 618   for (const auto &Type : Types) {
 619     unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
 620     if (Width < MinWidth) {
 621       Width = MinWidth;
 622     }
 623   }
 624
 625   return {Width, Signed};
 626 }
 627
 628 Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
 629   llvm::Type *DestType = Int8PtrTy;
 630   if (ArgValue->getType() != DestType)
 631     ArgValue =
 632         Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
 633
 634   Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
 635   return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
 636 }
 637
 638 /// Checks if using the result of __builtin_object_size(p, @p From) in place of
 639 /// __builtin_object_size(p, @p To) is correct
 640 static bool areBOSTypesCompatible(int From, int To) {
 641   // Note: Our __builtin_object_size implementation currently treats Type=0 and
 642   // Type=2 identically. Encoding this implementation detail here may make
 643   // improving __builtin_object_size difficult in the future, so it's omitted.
 644   return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
 645 }
 646
 647 static llvm::Value *
 648 getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
 649   return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
 650 }
 651
 652 llvm::Value *
 653 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
 654                                                  llvm::IntegerType *ResType,
 655                                                  llvm::Value *EmittedE,
 656                                                  bool IsDynamic) {
 657   uint64_t ObjectSize;
 658   if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
 659     return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
 660   return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
 661 }
 662
 663 /// Returns a Value corresponding to the size of the given expression.
 664 /// This Value may be either of the following:
 665 ///   - A llvm::Argument (if E is a param with the pass_object_size attribute on
 666 ///     it)
 667 ///   - A call to the @llvm.objectsize intrinsic
 668 ///
 669 /// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
 670 /// and we wouldn't otherwise try to reference a pass_object_size parameter,
 671 /// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
 672 llvm::Value *
 673 CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
 674                                        llvm::IntegerType *ResType,
 675                                        llvm::Value *EmittedE, bool IsDynamic) {
 676   // We need to reference an argument if the pointer is a parameter with the
 677   // pass_object_size attribute.
 678   if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
 679     auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
 680     auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
 681     if (Param != nullptr && PS != nullptr &&
 682         areBOSTypesCompatible(PS->getType(), Type)) {
 683       auto Iter = SizeArguments.find(Param);
 684       assert(Iter != SizeArguments.end());
 685
 686       const ImplicitParamDecl *D = Iter->second;
 687       auto DIter = LocalDeclMap.find(D);
 688       assert(DIter != LocalDeclMap.end());
 689
 690       return EmitLoadOfScalar(DIter->second, /*Volatile=*/false,
 691                               getContext().getSizeType(), E->getBeginLoc());
 692     }
 693   }
 694
 695   // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
 696   // evaluate E for side-effects. In either case, we shouldn't lower to
 697   // @llvm.objectsize.
 698   if (Type == 3 || (!EmittedE && E->HasSideEffects(getContext())))
 699     return getDefaultBuiltinObjectSizeResult(Type, ResType);
 700
 701   Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
 702   assert(Ptr->getType()->isPointerTy() &&
 703          "Non-pointer passed to __builtin_object_size?");
 704
 705   Function *F =
 706       CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
 707
 708   // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
 709   Value *Min = Builder.getInt1((Type & 2) != 0);
 710   // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
 711   Value *NullIsUnknown = Builder.getTrue();
 712   Value *Dynamic = Builder.getInt1(IsDynamic);
 713   return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic});
 714 }
 715
 716 namespace {
 717 /// A struct to generically describe a bit test intrinsic.
 718 struct BitTest {
 719   enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
 720   enum InterlockingKind : uint8_t {
 721     Unlocked,
 722     Sequential,
 723     Acquire,
 724     Release,
 725     NoFence
 726   };
 727
 728   ActionKind Action;
 729   InterlockingKind Interlocking;
 730   bool Is64Bit;
 731
 732   static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
 733 };
 734 } // namespace
 735
 736 BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
 737   switch (BuiltinID) {
 738     // Main portable variants.
 739   case Builtin::BI_bittest:
 740     return {TestOnly, Unlocked, false};
 741   case Builtin::BI_bittestandcomplement:
 742     return {Complement, Unlocked, false};
 743   case Builtin::BI_bittestandreset:
 744     return {Reset, Unlocked, false};
 745   case Builtin::BI_bittestandset:
 746     return {Set, Unlocked, false};
 747   case Builtin::BI_interlockedbittestandreset:
 748     return {Reset, Sequential, false};
 749   case Builtin::BI_interlockedbittestandset:
 750     return {Set, Sequential, false};
 751
 752     // X86-specific 64-bit variants.
 753   case Builtin::BI_bittest64:
 754     return {TestOnly, Unlocked, true};
 755   case Builtin::BI_bittestandcomplement64:
 756     return {Complement, Unlocked, true};
 757   case Builtin::BI_bittestandreset64:
 758     return {Reset, Unlocked, true};
 759   case Builtin::BI_bittestandset64:
 760     return {Set, Unlocked, true};
 761   case Builtin::BI_interlockedbittestandreset64:
 762     return {Reset, Sequential, true};
 763   case Builtin::BI_interlockedbittestandset64:
 764     return {Set, Sequential, true};
 765
 766     // ARM/AArch64-specific ordering variants.
 767   case Builtin::BI_interlockedbittestandset_acq:
 768     return {Set, Acquire, false};
 769   case Builtin::BI_interlockedbittestandset_rel:
 770     return {Set, Release, false};
 771   case Builtin::BI_interlockedbittestandset_nf:
 772     return {Set, NoFence, false};
 773   case Builtin::BI_interlockedbittestandreset_acq:
 774     return {Reset, Acquire, false};
 775   case Builtin::BI_interlockedbittestandreset_rel:
 776     return {Reset, Release, false};
 777   case Builtin::BI_interlockedbittestandreset_nf:
 778     return {Reset, NoFence, false};
 779   }
 780   llvm_unreachable("expected only bittest intrinsics");
 781 }
 782
 783 static char bitActionToX86BTCode(BitTest::ActionKind A) {
 784   switch (A) {
 785   case BitTest::TestOnly:   return '\0';
 786   case BitTest::Complement: return 'c';
 787   case BitTest::Reset:      return 'r';
 788   case BitTest::Set:        return 's';
 789   }
 790   llvm_unreachable("invalid action");
 791 }
 792
 793 static llvm::Value *EmitX86BitTestIntrinsic(CodeGenFunction &CGF,
 794                                             BitTest BT,
 795                                             const CallExpr *E, Value *BitBase,
 796                                             Value *BitPos) {
 797   char Action = bitActionToX86BTCode(BT.Action);
 798   char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
 799
 800   // Build the assembly.
 801   SmallString<64> Asm;
 802   raw_svector_ostream AsmOS(Asm);
 803   if (BT.Interlocking != BitTest::Unlocked)
 804     AsmOS << "lock ";
 805   AsmOS << "bt";
 806   if (Action)
 807     AsmOS << Action;
 808   AsmOS << SizeSuffix << " $2, ($1)\n\tsetc ${0:b}";
 809
 810   // Build the constraints. FIXME: We should support immediates when possible.
 811   std::string Constraints = "=r,r,r,~{cc},~{flags},~{fpsr}";
 812   llvm::IntegerType *IntType = llvm::IntegerType::get(
 813       CGF.getLLVMContext(),
 814       CGF.getContext().getTypeSize(E->getArg(1)->getType()));
 815   llvm::Type *IntPtrType = IntType->getPointerTo();
 816   llvm::FunctionType *FTy =
 817       llvm::FunctionType::get(CGF.Int8Ty, {IntPtrType, IntType}, false);
 818
 819   llvm::InlineAsm *IA =
 820       llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
 821   return CGF.Builder.CreateCall(IA, {BitBase, BitPos});
 822 }
 823
 824 static llvm::AtomicOrdering
 825 getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
 826   switch (I) {
 827   case BitTest::Unlocked:   return llvm::AtomicOrdering::NotAtomic;
 828   case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
 829   case BitTest::Acquire:    return llvm::AtomicOrdering::Acquire;
 830   case BitTest::Release:    return llvm::AtomicOrdering::Release;
 831   case BitTest::NoFence:    return llvm::AtomicOrdering::Monotonic;
 832   }
 833   llvm_unreachable("invalid interlocking");
 834 }
 835
 836 /// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
 837 /// bits and a bit position and read and optionally modify the bit at that
 838 /// position. The position index can be arbitrarily large, i.e. it can be larger
 839 /// than 31 or 63, so we need an indexed load in the general case.
 840 static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
 841                                          unsigned BuiltinID,
 842                                          const CallExpr *E) {
 843   Value *BitBase = CGF.EmitScalarExpr(E->getArg(0));
 844   Value *BitPos = CGF.EmitScalarExpr(E->getArg(1));
 845
 846   BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
 847
 848   // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
 849   // indexing operation internally. Use them if possible.
 850   if (CGF.getTarget().getTriple().isX86())
 851     return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
 852
 853   // Otherwise, use generic code to load one byte and test the bit. Use all but
 854   // the bottom three bits as the array index, and the bottom three bits to form
 855   // a mask.
 856   // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
 857   Value *ByteIndex = CGF.Builder.CreateAShr(
 858       BitPos, llvm::ConstantInt::get(BitPos->getType(), 3), "bittest.byteidx");
 859   Value *BitBaseI8 = CGF.Builder.CreatePointerCast(BitBase, CGF.Int8PtrTy);
 860   Address ByteAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BitBaseI8,
 861                                                  ByteIndex, "bittest.byteaddr"),
 862                    CharUnits::One());
 863   Value *PosLow =
 864       CGF.Builder.CreateAnd(CGF.Builder.CreateTrunc(BitPos, CGF.Int8Ty),
 865                             llvm::ConstantInt::get(CGF.Int8Ty, 0x7));
 866
 867   // The updating instructions will need a mask.
 868   Value *Mask = nullptr;
 869   if (BT.Action != BitTest::TestOnly) {
 870     Mask = CGF.Builder.CreateShl(llvm::ConstantInt::get(CGF.Int8Ty, 1), PosLow,
 871                                  "bittest.mask");
 872   }
 873
 874   // Check the action and ordering of the interlocked intrinsics.
 875   llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(BT.Interlocking);
 876
 877   Value *OldByte = nullptr;
 878   if (Ordering != llvm::AtomicOrdering::NotAtomic) {
 879     // Emit a combined atomicrmw load/store operation for the interlocked
 880     // intrinsics.
 881     llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
 882     if (BT.Action == BitTest::Reset) {
 883       Mask = CGF.Builder.CreateNot(Mask);
 884       RMWOp = llvm::AtomicRMWInst::And;
 885     }
 886     OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr.getPointer(), Mask,
 887                                           Ordering);
 888   } else {
 889     // Emit a plain load for the non-interlocked intrinsics.
 890     OldByte = CGF.Builder.CreateLoad(ByteAddr, "bittest.byte");
 891     Value *NewByte = nullptr;
 892     switch (BT.Action) {
 893     case BitTest::TestOnly:
 894       // Don't store anything.
 895       break;
 896     case BitTest::Complement:
 897       NewByte = CGF.Builder.CreateXor(OldByte, Mask);
 898       break;
 899     case BitTest::Reset:
 900       NewByte = CGF.Builder.CreateAnd(OldByte, CGF.Builder.CreateNot(Mask));
 901       break;
 902     case BitTest::Set:
 903       NewByte = CGF.Builder.CreateOr(OldByte, Mask);
 904       break;
 905     }
 906     if (NewByte)
 907       CGF.Builder.CreateStore(NewByte, ByteAddr);
 908   }
 909
 910   // However we loaded the old byte, either by plain load or atomicrmw, shift
 911   // the bit into the low position and mask it to 0 or 1.
 912   Value *ShiftedByte = CGF.Builder.CreateLShr(OldByte, PosLow, "bittest.shr");
 913   return CGF.Builder.CreateAnd(
 914       ShiftedByte, llvm::ConstantInt::get(CGF.Int8Ty, 1), "bittest.res");
 915 }
 916
 917 namespace {
 918 enum class MSVCSetJmpKind {
 919   _setjmpex,
 920   _setjmp3,
 921   _setjmp
 922 };
 923 }
 924
 925 /// MSVC handles setjmp a bit differently on different platforms. On every
 926 /// architecture except 32-bit x86, the frame address is passed. On x86, extra
 927 /// parameters can be passed as variadic arguments, but we always pass none.
 928 static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
 929                                const CallExpr *E) {
 930   llvm::Value *Arg1 = nullptr;
 931   llvm::Type *Arg1Ty = nullptr;
 932   StringRef Name;
 933   bool IsVarArg = false;
 934   if (SJKind == MSVCSetJmpKind::_setjmp3) {
 935     Name = "_setjmp3";
 936     Arg1Ty = CGF.Int32Ty;
 937     Arg1 = llvm::ConstantInt::get(CGF.IntTy, 0);
 938     IsVarArg = true;
 939   } else {
 940     Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
 941     Arg1Ty = CGF.Int8PtrTy;
 942     if (CGF.getTarget().getTriple().getArch() == llvm::Triple::aarch64) {
 943       Arg1 = CGF.Builder.CreateCall(
 944           CGF.CGM.getIntrinsic(Intrinsic::sponentry, CGF.AllocaInt8PtrTy));
 945     } else
 946       Arg1 = CGF.Builder.CreateCall(
 947           CGF.CGM.getIntrinsic(Intrinsic::frameaddress, CGF.AllocaInt8PtrTy),
 948           llvm::ConstantInt::get(CGF.Int32Ty, 0));
 949   }
 950
 951   // Mark the call site and declaration with ReturnsTwice.
 952   llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
 953   llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
 954       CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex,
 955       llvm::Attribute::ReturnsTwice);
 956   llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
 957       llvm::FunctionType::get(CGF.IntTy, ArgTypes, IsVarArg), Name,
 958       ReturnsTwiceAttr, /*Local=*/true);
 959
 960   llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
 961       CGF.EmitScalarExpr(E->getArg(0)), CGF.Int8PtrTy);
 962   llvm::Value *Args[] = {Buf, Arg1};
 963   llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(SetJmpFn, Args);
 964   CB->setAttributes(ReturnsTwiceAttr);
 965   return RValue::get(CB);
 966 }
 967
 968 // Many of MSVC builtins are on x64, ARM and AArch64; to avoid repeating code,
 969 // we handle them here.
 970 enum class CodeGenFunction::MSVCIntrin {
 971   _BitScanForward,
 972   _BitScanReverse,
 973   _InterlockedAnd,
 974   _InterlockedDecrement,
 975   _InterlockedExchange,
 976   _InterlockedExchangeAdd,
 977   _InterlockedExchangeSub,
 978   _InterlockedIncrement,
 979   _InterlockedOr,
 980   _InterlockedXor,
 981   _InterlockedExchangeAdd_acq,
 982   _InterlockedExchangeAdd_rel,
 983   _InterlockedExchangeAdd_nf,
 984   _InterlockedExchange_acq,
 985   _InterlockedExchange_rel,
 986   _InterlockedExchange_nf,
 987   _InterlockedCompareExchange_acq,
 988   _InterlockedCompareExchange_rel,
 989   _InterlockedCompareExchange_nf,
 990   _InterlockedOr_acq,
 991   _InterlockedOr_rel,
 992   _InterlockedOr_nf,
 993   _InterlockedXor_acq,
 994   _InterlockedXor_rel,
 995   _InterlockedXor_nf,
 996   _InterlockedAnd_acq,
 997   _InterlockedAnd_rel,
 998   _InterlockedAnd_nf,
 999   _InterlockedIncrement_acq,
1000   _InterlockedIncrement_rel,
1001   _InterlockedIncrement_nf,
1002   _InterlockedDecrement_acq,
1003   _InterlockedDecrement_rel,
1004   _InterlockedDecrement_nf,
1005   __fastfail,
1006 };
1007
1008 Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
1009                                             const CallExpr *E) {
1010   switch (BuiltinID) {
1011   case MSVCIntrin::_BitScanForward:
1012   case MSVCIntrin::_BitScanReverse: {
1013     Value *ArgValue = EmitScalarExpr(E->getArg(1));
1014
1015     llvm::Type *ArgType = ArgValue->getType();
1016     llvm::Type *IndexType =
1017       EmitScalarExpr(E->getArg(0))->getType()->getPointerElementType();
1018     llvm::Type *ResultType = ConvertType(E->getType());
1019
1020     Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1021     Value *ResZero = llvm::Constant::getNullValue(ResultType);
1022     Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1023
1024     BasicBlock *Begin = Builder.GetInsertBlock();
1025     BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn);
1026     Builder.SetInsertPoint(End);
1027     PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result");
1028
1029     Builder.SetInsertPoint(Begin);
1030     Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
1031     BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn);
1032     Builder.CreateCondBr(IsZero, End, NotZero);
1033     Result->addIncoming(ResZero, Begin);
1034
1035     Builder.SetInsertPoint(NotZero);
1036     Address IndexAddress = EmitPointerWithAlignment(E->getArg(0));
1037
1038     if (BuiltinID == MSVCIntrin::_BitScanForward) {
1039       Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1040       Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1041       ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
1042       Builder.CreateStore(ZeroCount, IndexAddress, false);
1043     } else {
1044       unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1045       Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1046
1047       Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1048       Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1049       ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
1050       Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1051       Builder.CreateStore(Index, IndexAddress, false);
1052     }
1053     Builder.CreateBr(End);
1054     Result->addIncoming(ResOne, NotZero);
1055
1056     Builder.SetInsertPoint(End);
1057     return Result;
1058   }
1059   case MSVCIntrin::_InterlockedAnd:
1060     return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E);
1061   case MSVCIntrin::_InterlockedExchange:
1062     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E);
1063   case MSVCIntrin::_InterlockedExchangeAdd:
1064     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E);
1065   case MSVCIntrin::_InterlockedExchangeSub:
1066     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Sub, E);
1067   case MSVCIntrin::_InterlockedOr:
1068     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E);
1069   case MSVCIntrin::_InterlockedXor:
1070     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E);
1071   case MSVCIntrin::_InterlockedExchangeAdd_acq:
1072     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1073                                  AtomicOrdering::Acquire);
1074   case MSVCIntrin::_InterlockedExchangeAdd_rel:
1075     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1076                                  AtomicOrdering::Release);
1077   case MSVCIntrin::_InterlockedExchangeAdd_nf:
1078     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1079                                  AtomicOrdering::Monotonic);
1080   case MSVCIntrin::_InterlockedExchange_acq:
1081     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1082                                  AtomicOrdering::Acquire);
1083   case MSVCIntrin::_InterlockedExchange_rel:
1084     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1085                                  AtomicOrdering::Release);
1086   case MSVCIntrin::_InterlockedExchange_nf:
1087     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1088                                  AtomicOrdering::Monotonic);
1089   case MSVCIntrin::_InterlockedCompareExchange_acq:
1090     return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
1091   case MSVCIntrin::_InterlockedCompareExchange_rel:
1092     return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Release);
1093   case MSVCIntrin::_InterlockedCompareExchange_nf:
1094     return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Monotonic);
1095   case MSVCIntrin::_InterlockedOr_acq:
1096     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1097                                  AtomicOrdering::Acquire);
1098   case MSVCIntrin::_InterlockedOr_rel:
1099     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1100                                  AtomicOrdering::Release);
1101   case MSVCIntrin::_InterlockedOr_nf:
1102     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1103                                  AtomicOrdering::Monotonic);
1104   case MSVCIntrin::_InterlockedXor_acq:
1105     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1106                                  AtomicOrdering::Acquire);
1107   case MSVCIntrin::_InterlockedXor_rel:
1108     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1109                                  AtomicOrdering::Release);
1110   case MSVCIntrin::_InterlockedXor_nf:
1111     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1112                                  AtomicOrdering::Monotonic);
1113   case MSVCIntrin::_InterlockedAnd_acq:
1114     return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
1115                                  AtomicOrdering::Acquire);
1116   case MSVCIntrin::_InterlockedAnd_rel:
1117     return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
1118                                  AtomicOrdering::Release);
1119   case MSVCIntrin::_InterlockedAnd_nf:
1120     return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
1121                                  AtomicOrdering::Monotonic);
1122   case MSVCIntrin::_InterlockedIncrement_acq:
1123     return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Acquire);
1124   case MSVCIntrin::_InterlockedIncrement_rel:
1125     return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Release);
1126   case MSVCIntrin::_InterlockedIncrement_nf:
1127     return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Monotonic);
1128   case MSVCIntrin::_InterlockedDecrement_acq:
1129     return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Acquire);
1130   case MSVCIntrin::_InterlockedDecrement_rel:
1131     return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Release);
1132   case MSVCIntrin::_InterlockedDecrement_nf:
1133     return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Monotonic);
1134
1135   case MSVCIntrin::_InterlockedDecrement:
1136     return EmitAtomicDecrementValue(*this, E);
1137   case MSVCIntrin::_InterlockedIncrement:
1138     return EmitAtomicIncrementValue(*this, E);
1139
1140   case MSVCIntrin::__fastfail: {
1141     // Request immediate process termination from the kernel. The instruction
1142     // sequences to do this are documented on MSDN:
1143     // https://msdn.microsoft.com/en-us/library/dn774154.aspx
1144     llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
1145     StringRef Asm, Constraints;
1146     switch (ISA) {
1147     default:
1148       ErrorUnsupported(E, "__fastfail call for this architecture");
1149       break;
1150     case llvm::Triple::x86:
1151     case llvm::Triple::x86_64:
1152       Asm = "int $$0x29";
1153       Constraints = "{cx}";
1154       break;
1155     case llvm::Triple::thumb:
1156       Asm = "udf #251";
1157       Constraints = "{r0}";
1158       break;
1159     case llvm::Triple::aarch64:
1160       Asm = "brk #0xF003";
1161       Constraints = "{w0}";
1162     }
1163     llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty}, false);
1164     llvm::InlineAsm *IA =
1165         llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
1166     llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1167         getLLVMContext(), llvm::AttributeList::FunctionIndex,
1168         llvm::Attribute::NoReturn);
1169     llvm::CallInst *CI = Builder.CreateCall(IA, EmitScalarExpr(E->getArg(0)));
1170     CI->setAttributes(NoReturnAttr);
1171     return CI;
1172   }
1173   }
1174   llvm_unreachable("Incorrect MSVC intrinsic!");
1175 }
1176
1177 namespace {
1178 // ARC cleanup for __builtin_os_log_format
1179 struct CallObjCArcUse final : EHScopeStack::Cleanup {
1180   CallObjCArcUse(llvm::Value *object) : object(object) {}
1181   llvm::Value *object;
1182
1183   void Emit(CodeGenFunction &CGF, Flags flags) override {
1184     CGF.EmitARCIntrinsicUse(object);
1185   }
1186 };
1187 }
1188
1189 Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
1190                                                  BuiltinCheckKind Kind) {
1191   assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero)
1192           && "Unsupported builtin check kind");
1193
1194   Value *ArgValue = EmitScalarExpr(E);
1195   if (!SanOpts.has(SanitizerKind::Builtin) || !getTarget().isCLZForZeroUndef())
1196     return ArgValue;
1197
1198   SanitizerScope SanScope(this);
1199   Value *Cond = Builder.CreateICmpNE(
1200       ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
1201   EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
1202             SanitizerHandler::InvalidBuiltin,
1203             {EmitCheckSourceLocation(E->getExprLoc()),
1204              llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
1205             None);
1206   return ArgValue;
1207 }
1208
1209 /// Get the argument type for arguments to os_log_helper.
1210 static CanQualType getOSLogArgType(ASTContext &C, int Size) {
1211   QualType UnsignedTy = C.getIntTypeForBitwidth(Size * 8, /*Signed=*/false);
1212   return C.getCanonicalType(UnsignedTy);
1213 }
1214
1215 llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
1216     const analyze_os_log::OSLogBufferLayout &Layout,
1217     CharUnits BufferAlignment) {
1218   ASTContext &Ctx = getContext();
1219
1220   llvm::SmallString<64> Name;
1221   {
1222     raw_svector_ostream OS(Name);
1223     OS << "__os_log_helper";
1224     OS << "_" << BufferAlignment.getQuantity();
1225     OS << "_" << int(Layout.getSummaryByte());
1226     OS << "_" << int(Layout.getNumArgsByte());
1227     for (const auto &Item : Layout.Items)
1228       OS << "_" << int(Item.getSizeByte()) << "_"
1229          << int(Item.getDescriptorByte());
1230   }
1231
1232   if (llvm::Function *F = CGM.getModule().getFunction(Name))
1233     return F;
1234
1235   llvm::SmallVector<QualType, 4> ArgTys;
1236   FunctionArgList Args;
1237   Args.push_back(ImplicitParamDecl::Create(
1238       Ctx, nullptr, SourceLocation(), &Ctx.Idents.get("buffer"), Ctx.VoidPtrTy,
1239       ImplicitParamDecl::Other));
1240   ArgTys.emplace_back(Ctx.VoidPtrTy);
1241
1242   for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
1243     char Size = Layout.Items[I].getSizeByte();
1244     if (!Size)
1245       continue;
1246
1247     QualType ArgTy = getOSLogArgType(Ctx, Size);
1248     Args.push_back(ImplicitParamDecl::Create(
1249         Ctx, nullptr, SourceLocation(),
1250         &Ctx.Idents.get(std::string("arg") + llvm::to_string(I)), ArgTy,
1251         ImplicitParamDecl::Other));
1252     ArgTys.emplace_back(ArgTy);
1253   }
1254
1255   QualType ReturnTy = Ctx.VoidTy;
1256   QualType FuncionTy = Ctx.getFunctionType(ReturnTy, ArgTys, {});
1257
1258   // The helper function has linkonce_odr linkage to enable the linker to merge
1259   // identical functions. To ensure the merging always happens, 'noinline' is
1260   // attached to the function when compiling with -Oz.
1261   const CGFunctionInfo &FI =
1262       CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, Args);
1263   llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
1264   llvm::Function *Fn = llvm::Function::Create(
1265       FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
1266   Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1267   CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, Fn);
1268   CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Fn);
1269   Fn->setDoesNotThrow();
1270
1271   // Attach 'noinline' at -Oz.
1272   if (CGM.getCodeGenOpts().OptimizeSize == 2)
1273     Fn->addFnAttr(llvm::Attribute::NoInline);
1274
1275   auto NL = ApplyDebugLocation::CreateEmpty(*this);
1276   IdentifierInfo *II = &Ctx.Idents.get(Name);
1277   FunctionDecl *FD = FunctionDecl::Create(
1278       Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(), II,
1279       FuncionTy, nullptr, SC_PrivateExtern, false, false);
1280   // Avoid generating debug location info for the function.
1281   FD->setImplicit();
1282
1283   StartFunction(FD, ReturnTy, Fn, FI, Args);
1284
1285   // Create a scope with an artificial location for the body of this function.
1286   auto AL = ApplyDebugLocation::CreateArtificial(*this);
1287
1288   CharUnits Offset;
1289   Address BufAddr(Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"),
1290                   BufferAlignment);
1291   Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
1292                       Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
1293   Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
1294                       Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs"));
1295
1296   unsigned I = 1;
1297   for (const auto &Item : Layout.Items) {
1298     Builder.CreateStore(
1299         Builder.getInt8(Item.getDescriptorByte()),
1300         Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor"));
1301     Builder.CreateStore(
1302         Builder.getInt8(Item.getSizeByte()),
1303         Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize"));
1304
1305     CharUnits Size = Item.size();
1306     if (!Size.getQuantity())
1307       continue;
1308
1309     Address Arg = GetAddrOfLocalVar(Args[I]);
1310     Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset, "argData");
1311     Addr = Builder.CreateBitCast(Addr, Arg.getPointer()->getType(),
1312                                  "argDataCast");
1313     Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
1314     Offset += Size;
1315     ++I;
1316   }
1317
1318   FinishFunction();
1319
1320   return Fn;
1321 }
1322
1323 RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
1324   assert(E.getNumArgs() >= 2 &&
1325          "__builtin_os_log_format takes at least 2 arguments");
1326   ASTContext &Ctx = getContext();
1327   analyze_os_log::OSLogBufferLayout Layout;
1328   analyze_os_log::computeOSLogBufferLayout(Ctx, &E, Layout);
1329   Address BufAddr = EmitPointerWithAlignment(E.getArg(0));
1330   llvm::SmallVector<llvm::Value *, 4> RetainableOperands;
1331
1332   // Ignore argument 1, the format string. It is not currently used.
1333   CallArgList Args;
1334   Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy);
1335
1336   for (const auto &Item : Layout.Items) {
1337     int Size = Item.getSizeByte();
1338     if (!Size)
1339       continue;
1340
1341     llvm::Value *ArgVal;
1342
1343     if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
1344       uint64_t Val = 0;
1345       for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
1346         Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
1347       ArgVal = llvm::Constant::getIntegerValue(Int64Ty, llvm::APInt(64, Val));
1348     } else if (const Expr *TheExpr = Item.getExpr()) {
1349       ArgVal = EmitScalarExpr(TheExpr, /*Ignore*/ false);
1350
1351       // If a temporary object that requires destruction after the full
1352       // expression is passed, push a lifetime-extended cleanup to extend its
1353       // lifetime to the end of the enclosing block scope.
1354       auto LifetimeExtendObject = [&](const Expr *E) {
1355         E = E->IgnoreParenCasts();
1356         // Extend lifetimes of objects returned by function calls and message
1357         // sends.
1358
1359         // FIXME: We should do this in other cases in which temporaries are
1360         //        created including arguments of non-ARC types (e.g., C++
1361         //        temporaries).
1362         if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
1363           return true;
1364         return false;
1365       };
1366
1367       if (TheExpr->getType()->isObjCRetainableType() &&
1368           getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
1369         assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
1370                "Only scalar can be a ObjC retainable type");
1371         if (!isa<Constant>(ArgVal)) {
1372           CleanupKind Cleanup = getARCCleanupKind();
1373           QualType Ty = TheExpr->getType();
1374           Address Alloca = Address::invalid();
1375           Address Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca);
1376           ArgVal = EmitARCRetain(Ty, ArgVal);
1377           Builder.CreateStore(ArgVal, Addr);
1378           pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty,
1379                                       CodeGenFunction::destroyARCStrongPrecise,
1380                                       Cleanup & EHCleanup);
1381
1382           // Push a clang.arc.use call to ensure ARC optimizer knows that the
1383           // argument has to be alive.
1384           if (CGM.getCodeGenOpts().OptimizationLevel != 0)
1385             pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
1386         }
1387       }
1388     } else {
1389       ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
1390     }
1391
1392     unsigned ArgValSize =
1393         CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
1394     llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
1395                                                      ArgValSize);
1396     ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
1397     CanQualType ArgTy = getOSLogArgType(Ctx, Size);
1398     // If ArgVal has type x86_fp80, zero-extend ArgVal.
1399     ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
1400     Args.add(RValue::get(ArgVal), ArgTy);
1401   }
1402
1403   const CGFunctionInfo &FI =
1404       CGM.getTypes().arrangeBuiltinFunctionCall(Ctx.VoidTy, Args);
1405   llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
1406       Layout, BufAddr.getAlignment());
1407   EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args);
1408   return RValue::get(BufAddr.getPointer());
1409 }
1410
1411 /// Determine if a binop is a checked mixed-sign multiply we can specialize.
1412 static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
1413                                        WidthAndSignedness Op1Info,
1414                                        WidthAndSignedness Op2Info,
1415                                        WidthAndSignedness ResultInfo) {
1416   return BuiltinID == Builtin::BI__builtin_mul_overflow &&
1417          std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
1418          Op1Info.Signed != Op2Info.Signed;
1419 }
1420
1421 /// Emit a checked mixed-sign multiply. This is a cheaper specialization of
1422 /// the generic checked-binop irgen.
1423 static RValue
1424 EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
1425                              WidthAndSignedness Op1Info, const clang::Expr *Op2,
1426                              WidthAndSignedness Op2Info,
1427                              const clang::Expr *ResultArg, QualType ResultQTy,
1428                              WidthAndSignedness ResultInfo) {
1429   assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
1430                                     Op2Info, ResultInfo) &&
1431          "Not a mixed-sign multipliction we can specialize");
1432
1433   // Emit the signed and unsigned operands.
1434   const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
1435   const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
1436   llvm::Value *Signed = CGF.EmitScalarExpr(SignedOp);
1437   llvm::Value *Unsigned = CGF.EmitScalarExpr(UnsignedOp);
1438   unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
1439   unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
1440
1441   // One of the operands may be smaller than the other. If so, [s|z]ext it.
1442   if (SignedOpWidth < UnsignedOpWidth)
1443     Signed = CGF.Builder.CreateSExt(Signed, Unsigned->getType(), "op.sext");
1444   if (UnsignedOpWidth < SignedOpWidth)
1445     Unsigned = CGF.Builder.CreateZExt(Unsigned, Signed->getType(), "op.zext");
1446
1447   llvm::Type *OpTy = Signed->getType();
1448   llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
1449   Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
1450   llvm::Type *ResTy = ResultPtr.getElementType();
1451   unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
1452
1453   // Take the absolute value of the signed operand.
1454   llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(Signed, Zero);
1455   llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(Zero, Signed);
1456   llvm::Value *AbsSigned =
1457       CGF.Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
1458
1459   // Perform a checked unsigned multiplication.
1460   llvm::Value *UnsignedOverflow;
1461   llvm::Value *UnsignedResult =
1462       EmitOverflowIntrinsic(CGF, llvm::Intrinsic::umul_with_overflow, AbsSigned,
1463                             Unsigned, UnsignedOverflow);
1464
1465   llvm::Value *Overflow, *Result;
1466   if (ResultInfo.Signed) {
1467     // Signed overflow occurs if the result is greater than INT_MAX or lesser
1468     // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
1469     auto IntMax =
1470         llvm::APInt::getSignedMaxValue(ResultInfo.Width).zextOrSelf(OpWidth);
1471     llvm::Value *MaxResult =
1472         CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
1473                               CGF.Builder.CreateZExt(IsNegative, OpTy));
1474     llvm::Value *SignedOverflow =
1475         CGF.Builder.CreateICmpUGT(UnsignedResult, MaxResult);
1476     Overflow = CGF.Builder.CreateOr(UnsignedOverflow, SignedOverflow);
1477
1478     // Prepare the signed result (possibly by negating it).
1479     llvm::Value *NegativeResult = CGF.Builder.CreateNeg(UnsignedResult);
1480     llvm::Value *SignedResult =
1481         CGF.Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
1482     Result = CGF.Builder.CreateTrunc(SignedResult, ResTy);
1483   } else {
1484     // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
1485     llvm::Value *Underflow = CGF.Builder.CreateAnd(
1486         IsNegative, CGF.Builder.CreateIsNotNull(UnsignedResult));
1487     Overflow = CGF.Builder.CreateOr(UnsignedOverflow, Underflow);
1488     if (ResultInfo.Width < OpWidth) {
1489       auto IntMax =
1490           llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
1491       llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
1492           UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
1493       Overflow = CGF.Builder.CreateOr(Overflow, TruncOverflow);
1494     }
1495
1496     // Negate the product if it would be negative in infinite precision.
1497     Result = CGF.Builder.CreateSelect(
1498         IsNegative, CGF.Builder.CreateNeg(UnsignedResult), UnsignedResult);
1499
1500     Result = CGF.Builder.CreateTrunc(Result, ResTy);
1501   }
1502   assert(Overflow && Result && "Missing overflow or result");
1503
1504   bool isVolatile =
1505       ResultArg->getType()->getPointeeType().isVolatileQualified();
1506   CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
1507                           isVolatile);
1508   return RValue::get(Overflow);
1509 }
1510
1511 static llvm::Value *dumpRecord(CodeGenFunction &CGF, QualType RType,
1512                                Value *&RecordPtr, CharUnits Align,
1513                                llvm::FunctionCallee Func, int Lvl) {
1514   ASTContext &Context = CGF.getContext();
1515   RecordDecl *RD = RType->castAs<RecordType>()->getDecl()->getDefinition();
1516   std::string Pad = std::string(Lvl * 4, ' ');
1517
1518   Value *GString =
1519       CGF.Builder.CreateGlobalStringPtr(RType.getAsString() + " {\n");
1520   Value *Res = CGF.Builder.CreateCall(Func, {GString});
1521
1522   static llvm::DenseMap<QualType, const char *> Types;
1523   if (Types.empty()) {
1524     Types[Context.CharTy] = "%c";
1525     Types[Context.BoolTy] = "%d";
1526     Types[Context.SignedCharTy] = "%hhd";
1527     Types[Context.UnsignedCharTy] = "%hhu";
1528     Types[Context.IntTy] = "%d";
1529     Types[Context.UnsignedIntTy] = "%u";
1530     Types[Context.LongTy] = "%ld";
1531     Types[Context.UnsignedLongTy] = "%lu";
1532     Types[Context.LongLongTy] = "%lld";
1533     Types[Context.UnsignedLongLongTy] = "%llu";
1534     Types[Context.ShortTy] = "%hd";
1535     Types[Context.UnsignedShortTy] = "%hu";
1536     Types[Context.VoidPtrTy] = "%p";
1537     Types[Context.FloatTy] = "%f";
1538     Types[Context.DoubleTy] = "%f";
1539     Types[Context.LongDoubleTy] = "%Lf";
1540     Types[Context.getPointerType(Context.CharTy)] = "%s";
1541     Types[Context.getPointerType(Context.getConstType(Context.CharTy))] = "%s";
1542   }
1543
1544   for (const auto *FD : RD->fields()) {
1545     Value *FieldPtr = RecordPtr;
1546     if (RD->isUnion())
1547       FieldPtr = CGF.Builder.CreatePointerCast(
1548           FieldPtr, CGF.ConvertType(Context.getPointerType(FD->getType())));
1549     else
1550       FieldPtr = CGF.Builder.CreateStructGEP(CGF.ConvertType(RType), FieldPtr,
1551                                              FD->getFieldIndex());
1552
1553     GString = CGF.Builder.CreateGlobalStringPtr(
1554         llvm::Twine(Pad)
1555             .concat(FD->getType().getAsString())
1556             .concat(llvm::Twine(' '))
1557             .concat(FD->getNameAsString())
1558             .concat(" : ")
1559             .str());
1560     Value *TmpRes = CGF.Builder.CreateCall(Func, {GString});
1561     Res = CGF.Builder.CreateAdd(Res, TmpRes);
1562
1563     QualType CanonicalType =
1564         FD->getType().getUnqualifiedType().getCanonicalType();
1565
1566     // We check whether we are in a recursive type
1567     if (CanonicalType->isRecordType()) {
1568       TmpRes = dumpRecord(CGF, CanonicalType, FieldPtr, Align, Func, Lvl + 1);
1569       Res = CGF.Builder.CreateAdd(TmpRes, Res);
1570       continue;
1571     }
1572
1573     // We try to determine the best format to print the current field
1574     llvm::Twine Format = Types.find(CanonicalType) == Types.end()
1575                              ? Types[Context.VoidPtrTy]
1576                              : Types[CanonicalType];
1577
1578     Address FieldAddress = Address(FieldPtr, Align);
1579     FieldPtr = CGF.Builder.CreateLoad(FieldAddress);
1580
1581     // FIXME Need to handle bitfield here
1582     GString = CGF.Builder.CreateGlobalStringPtr(
1583         Format.concat(llvm::Twine('\n')).str());
1584     TmpRes = CGF.Builder.CreateCall(Func, {GString, FieldPtr});
1585     Res = CGF.Builder.CreateAdd(Res, TmpRes);
1586   }
1587
1588   GString = CGF.Builder.CreateGlobalStringPtr(Pad + "}\n");
1589   Value *TmpRes = CGF.Builder.CreateCall(Func, {GString});
1590   Res = CGF.Builder.CreateAdd(Res, TmpRes);
1591   return Res;
1592 }
1593
1594 static bool
1595 TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty,
1596                               llvm::SmallPtrSetImpl<const Decl *> &Seen) {
1597   if (const auto *Arr = Ctx.getAsArrayType(Ty))
1598     Ty = Ctx.getBaseElementType(Arr);
1599
1600   const auto *Record = Ty->getAsCXXRecordDecl();
1601   if (!Record)
1602     return false;
1603
1604   // We've already checked this type, or are in the process of checking it.
1605   if (!Seen.insert(Record).second)
1606     return false;
1607
1608   assert(Record->hasDefinition() &&
1609          "Incomplete types should already be diagnosed");
1610
1611   if (Record->isDynamicClass())
1612     return true;
1613
1614   for (FieldDecl *F : Record->fields()) {
1615     if (TypeRequiresBuiltinLaunderImp(Ctx, F->getType(), Seen))
1616       return true;
1617   }
1618   return false;
1619 }
1620
1621 /// Determine if the specified type requires laundering by checking if it is a
1622 /// dynamic class type or contains a subobject which is a dynamic class type.
1623 static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty) {
1624   if (!CGM.getCodeGenOpts().StrictVTablePointers)
1625     return false;
1626   llvm::SmallPtrSet<const Decl *, 16> Seen;
1627   return TypeRequiresBuiltinLaunderImp(CGM.getContext(), Ty, Seen);
1628 }
1629
1630 RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
1631   llvm::Value *Src = EmitScalarExpr(E->getArg(0));
1632   llvm::Value *ShiftAmt = EmitScalarExpr(E->getArg(1));
1633
1634   // The builtin's shift arg may have a different type than the source arg and
1635   // result, but the LLVM intrinsic uses the same type for all values.
1636   llvm::Type *Ty = Src->getType();
1637   ShiftAmt = Builder.CreateIntCast(ShiftAmt, Ty, false);
1638
1639   // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
1640   unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1641   Function *F = CGM.getIntrinsic(IID, Ty);
1642   return RValue::get(Builder.CreateCall(F, { Src, Src, ShiftAmt }));
1643 }
1644
1645 RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
1646                                         const CallExpr *E,
1647                                         ReturnValueSlot ReturnValue) {
1648   const FunctionDecl *FD = GD.getDecl()->getAsFunction();
1649   // See if we can constant fold this builtin.  If so, don't emit it at all.
1650   Expr::EvalResult Result;
1651   if (E->EvaluateAsRValue(Result, CGM.getContext()) &&
1652       !Result.hasSideEffects()) {
1653     if (Result.Val.isInt())
1654       return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
1655                                                 Result.Val.getInt()));
1656     if (Result.Val.isFloat())
1657       return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
1658                                                Result.Val.getFloat()));
1659   }
1660
1661   // There are LLVM math intrinsics/instructions corresponding to math library
1662   // functions except the LLVM op will never set errno while the math library
1663   // might. Also, math builtins have the same semantics as their math library
1664   // twins. Thus, we can transform math library and builtin calls to their
1665   // LLVM counterparts if the call is marked 'const' (known to never set errno).
1666   if (FD->hasAttr<ConstAttr>()) {
1667     switch (BuiltinID) {
1668     case Builtin::BIceil:
1669     case Builtin::BIceilf:
1670     case Builtin::BIceill:
1671     case Builtin::BI__builtin_ceil:
1672     case Builtin::BI__builtin_ceilf:
1673     case Builtin::BI__builtin_ceilf16:
1674     case Builtin::BI__builtin_ceill:
1675       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1676                                    Intrinsic::ceil,
1677                                    Intrinsic::experimental_constrained_ceil));
1678
1679     case Builtin::BIcopysign:
1680     case Builtin::BIcopysignf:
1681     case Builtin::BIcopysignl:
1682     case Builtin::BI__builtin_copysign:
1683     case Builtin::BI__builtin_copysignf:
1684     case Builtin::BI__builtin_copysignf16:
1685     case Builtin::BI__builtin_copysignl:
1686     case Builtin::BI__builtin_copysignf128:
1687       return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::copysign));
1688
1689     case Builtin::BIcos:
1690     case Builtin::BIcosf:
1691     case Builtin::BIcosl:
1692     case Builtin::BI__builtin_cos:
1693     case Builtin::BI__builtin_cosf:
1694     case Builtin::BI__builtin_cosf16:
1695     case Builtin::BI__builtin_cosl:
1696       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1697                                    Intrinsic::cos,
1698                                    Intrinsic::experimental_constrained_cos));
1699
1700     case Builtin::BIexp:
1701     case Builtin::BIexpf:
1702     case Builtin::BIexpl:
1703     case Builtin::BI__builtin_exp:
1704     case Builtin::BI__builtin_expf:
1705     case Builtin::BI__builtin_expf16:
1706     case Builtin::BI__builtin_expl:
1707       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1708                                    Intrinsic::exp,
1709                                    Intrinsic::experimental_constrained_exp));
1710
1711     case Builtin::BIexp2:
1712     case Builtin::BIexp2f:
1713     case Builtin::BIexp2l:
1714     case Builtin::BI__builtin_exp2:
1715     case Builtin::BI__builtin_exp2f:
1716     case Builtin::BI__builtin_exp2f16:
1717     case Builtin::BI__builtin_exp2l:
1718       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1719                                    Intrinsic::exp2,
1720                                    Intrinsic::experimental_constrained_exp2));
1721
1722     case Builtin::BIfabs:
1723     case Builtin::BIfabsf:
1724     case Builtin::BIfabsl:
1725     case Builtin::BI__builtin_fabs:
1726     case Builtin::BI__builtin_fabsf:
1727     case Builtin::BI__builtin_fabsf16:
1728     case Builtin::BI__builtin_fabsl:
1729     case Builtin::BI__builtin_fabsf128:
1730       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::fabs));
1731
1732     case Builtin::BIfloor:
1733     case Builtin::BIfloorf:
1734     case Builtin::BIfloorl:
1735     case Builtin::BI__builtin_floor:
1736     case Builtin::BI__builtin_floorf:
1737     case Builtin::BI__builtin_floorf16:
1738     case Builtin::BI__builtin_floorl:
1739       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1740                                    Intrinsic::floor,
1741                                    Intrinsic::experimental_constrained_floor));
1742
1743     case Builtin::BIfma:
1744     case Builtin::BIfmaf:
1745     case Builtin::BIfmal:
1746     case Builtin::BI__builtin_fma:
1747     case Builtin::BI__builtin_fmaf:
1748     case Builtin::BI__builtin_fmaf16:
1749     case Builtin::BI__builtin_fmal:
1750       return RValue::get(emitTernaryMaybeConstrainedFPBuiltin(*this, E,
1751                                    Intrinsic::fma,
1752                                    Intrinsic::experimental_constrained_fma));
1753
1754     case Builtin::BIfmax:
1755     case Builtin::BIfmaxf:
1756     case Builtin::BIfmaxl:
1757     case Builtin::BI__builtin_fmax:
1758     case Builtin::BI__builtin_fmaxf:
1759     case Builtin::BI__builtin_fmaxf16:
1760     case Builtin::BI__builtin_fmaxl:
1761       return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(*this, E,
1762                                    Intrinsic::maxnum,
1763                                    Intrinsic::experimental_constrained_maxnum));
1764
1765     case Builtin::BIfmin:
1766     case Builtin::BIfminf:
1767     case Builtin::BIfminl:
1768     case Builtin::BI__builtin_fmin:
1769     case Builtin::BI__builtin_fminf:
1770     case Builtin::BI__builtin_fminf16:
1771     case Builtin::BI__builtin_fminl:
1772       return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(*this, E,
1773                                    Intrinsic::minnum,
1774                                    Intrinsic::experimental_constrained_minnum));
1775
1776     // fmod() is a special-case. It maps to the frem instruction rather than an
1777     // LLVM intrinsic.
1778     case Builtin::BIfmod:
1779     case Builtin::BIfmodf:
1780     case Builtin::BIfmodl:
1781     case Builtin::BI__builtin_fmod:
1782     case Builtin::BI__builtin_fmodf:
1783     case Builtin::BI__builtin_fmodf16:
1784     case Builtin::BI__builtin_fmodl: {
1785       Value *Arg1 = EmitScalarExpr(E->getArg(0));
1786       Value *Arg2 = EmitScalarExpr(E->getArg(1));
1787       return RValue::get(Builder.CreateFRem(Arg1, Arg2, "fmod"));
1788     }
1789
1790     case Builtin::BIlog:
1791     case Builtin::BIlogf:
1792     case Builtin::BIlogl:
1793     case Builtin::BI__builtin_log:
1794     case Builtin::BI__builtin_logf:
1795     case Builtin::BI__builtin_logf16:
1796     case Builtin::BI__builtin_logl:
1797       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1798                                    Intrinsic::log,
1799                                    Intrinsic::experimental_constrained_log));
1800
1801     case Builtin::BIlog10:
1802     case Builtin::BIlog10f:
1803     case Builtin::BIlog10l:
1804     case Builtin::BI__builtin_log10:
1805     case Builtin::BI__builtin_log10f:
1806     case Builtin::BI__builtin_log10f16:
1807     case Builtin::BI__builtin_log10l:
1808       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1809                                    Intrinsic::log10,
1810                                    Intrinsic::experimental_constrained_log10));
1811
1812     case Builtin::BIlog2:
1813     case Builtin::BIlog2f:
1814     case Builtin::BIlog2l:
1815     case Builtin::BI__builtin_log2:
1816     case Builtin::BI__builtin_log2f:
1817     case Builtin::BI__builtin_log2f16:
1818     case Builtin::BI__builtin_log2l:
1819       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1820                                    Intrinsic::log2,
1821                                    Intrinsic::experimental_constrained_log2));
1822
1823     case Builtin::BInearbyint:
1824     case Builtin::BInearbyintf:
1825     case Builtin::BInearbyintl:
1826     case Builtin::BI__builtin_nearbyint:
1827     case Builtin::BI__builtin_nearbyintf:
1828     case Builtin::BI__builtin_nearbyintl:
1829       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1830                                 Intrinsic::nearbyint,
1831                                 Intrinsic::experimental_constrained_nearbyint));
1832
1833     case Builtin::BIpow:
1834     case Builtin::BIpowf:
1835     case Builtin::BIpowl:
1836     case Builtin::BI__builtin_pow:
1837     case Builtin::BI__builtin_powf:
1838     case Builtin::BI__builtin_powf16:
1839     case Builtin::BI__builtin_powl:
1840       return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(*this, E,
1841                                    Intrinsic::pow,
1842                                    Intrinsic::experimental_constrained_pow));
1843
1844     case Builtin::BIrint:
1845     case Builtin::BIrintf:
1846     case Builtin::BIrintl:
1847     case Builtin::BI__builtin_rint:
1848     case Builtin::BI__builtin_rintf:
1849     case Builtin::BI__builtin_rintf16:
1850     case Builtin::BI__builtin_rintl:
1851       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1852                                    Intrinsic::rint,
1853                                    Intrinsic::experimental_constrained_rint));
1854
1855     case Builtin::BIround:
1856     case Builtin::BIroundf:
1857     case Builtin::BIroundl:
1858     case Builtin::BI__builtin_round:
1859     case Builtin::BI__builtin_roundf:
1860     case Builtin::BI__builtin_roundf16:
1861     case Builtin::BI__builtin_roundl:
1862       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1863                                    Intrinsic::round,
1864                                    Intrinsic::experimental_constrained_round));
1865
1866     case Builtin::BIsin:
1867     case Builtin::BIsinf:
1868     case Builtin::BIsinl:
1869     case Builtin::BI__builtin_sin:
1870     case Builtin::BI__builtin_sinf:
1871     case Builtin::BI__builtin_sinf16:
1872     case Builtin::BI__builtin_sinl:
1873       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1874                                    Intrinsic::sin,
1875                                    Intrinsic::experimental_constrained_sin));
1876
1877     case Builtin::BIsqrt:
1878     case Builtin::BIsqrtf:
1879     case Builtin::BIsqrtl:
1880     case Builtin::BI__builtin_sqrt:
1881     case Builtin::BI__builtin_sqrtf:
1882     case Builtin::BI__builtin_sqrtf16:
1883     case Builtin::BI__builtin_sqrtl:
1884       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1885                                    Intrinsic::sqrt,
1886                                    Intrinsic::experimental_constrained_sqrt));
1887
1888     case Builtin::BItrunc:
1889     case Builtin::BItruncf:
1890     case Builtin::BItruncl:
1891     case Builtin::BI__builtin_trunc:
1892     case Builtin::BI__builtin_truncf:
1893     case Builtin::BI__builtin_truncf16:
1894     case Builtin::BI__builtin_truncl:
1895       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
1896                                    Intrinsic::trunc,
1897                                    Intrinsic::experimental_constrained_trunc));
1898
1899     case Builtin::BIlround:
1900     case Builtin::BIlroundf:
1901     case Builtin::BIlroundl:
1902     case Builtin::BI__builtin_lround:
1903     case Builtin::BI__builtin_lroundf:
1904     case Builtin::BI__builtin_lroundl:
1905       return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
1906           *this, E, Intrinsic::lround,
1907           Intrinsic::experimental_constrained_lround));
1908
1909     case Builtin::BIllround:
1910     case Builtin::BIllroundf:
1911     case Builtin::BIllroundl:
1912     case Builtin::BI__builtin_llround:
1913     case Builtin::BI__builtin_llroundf:
1914     case Builtin::BI__builtin_llroundl:
1915       return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
1916           *this, E, Intrinsic::llround,
1917           Intrinsic::experimental_constrained_llround));
1918
1919     case Builtin::BIlrint:
1920     case Builtin::BIlrintf:
1921     case Builtin::BIlrintl:
1922     case Builtin::BI__builtin_lrint:
1923     case Builtin::BI__builtin_lrintf:
1924     case Builtin::BI__builtin_lrintl:
1925       return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
1926           *this, E, Intrinsic::lrint,
1927           Intrinsic::experimental_constrained_lrint));
1928
1929     case Builtin::BIllrint:
1930     case Builtin::BIllrintf:
1931     case Builtin::BIllrintl:
1932     case Builtin::BI__builtin_llrint:
1933     case Builtin::BI__builtin_llrintf:
1934     case Builtin::BI__builtin_llrintl:
1935       return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
1936           *this, E, Intrinsic::llrint,
1937           Intrinsic::experimental_constrained_llrint));
1938
1939     default:
1940       break;
1941     }
1942   }
1943
1944   switch (BuiltinID) {
1945   default: break;
1946   case Builtin::BI__builtin___CFStringMakeConstantString:
1947   case Builtin::BI__builtin___NSStringMakeConstantString:
1948     return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
1949   case Builtin::BI__builtin_stdarg_start:
1950   case Builtin::BI__builtin_va_start:
1951   case Builtin::BI__va_start:
1952   case Builtin::BI__builtin_va_end:
1953     return RValue::get(
1954         EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
1955                            ? EmitScalarExpr(E->getArg(0))
1956                            : EmitVAListRef(E->getArg(0)).getPointer(),
1957                        BuiltinID != Builtin::BI__builtin_va_end));
1958   case Builtin::BI__builtin_va_copy: {
1959     Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
1960     Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
1961
1962     llvm::Type *Type = Int8PtrTy;
1963
1964     DstPtr = Builder.CreateBitCast(DstPtr, Type);
1965     SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
1966     return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy),
1967                                           {DstPtr, SrcPtr}));
1968   }
1969   case Builtin::BI__builtin_abs:
1970   case Builtin::BI__builtin_labs:
1971   case Builtin::BI__builtin_llabs: {
1972     // X < 0 ? -X : X
1973     // The negation has 'nsw' because abs of INT_MIN is undefined.
1974     Value *ArgValue = EmitScalarExpr(E->getArg(0));
1975     Value *NegOp = Builder.CreateNSWNeg(ArgValue, "neg");
1976     Constant *Zero = llvm::Constant::getNullValue(ArgValue->getType());
1977     Value *CmpResult = Builder.CreateICmpSLT(ArgValue, Zero, "abscond");
1978     Value *Result = Builder.CreateSelect(CmpResult, NegOp, ArgValue, "abs");
1979     return RValue::get(Result);
1980   }
1981   case Builtin::BI__builtin_conj:
1982   case Builtin::BI__builtin_conjf:
1983   case Builtin::BI__builtin_conjl:
1984   case Builtin::BIconj:
1985   case Builtin::BIconjf:
1986   case Builtin::BIconjl: {
1987     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
1988     Value *Real = ComplexVal.first;
1989     Value *Imag = ComplexVal.second;
1990     Imag = Builder.CreateFNeg(Imag, "neg");
1991     return RValue::getComplex(std::make_pair(Real, Imag));
1992   }
1993   case Builtin::BI__builtin_creal:
1994   case Builtin::BI__builtin_crealf:
1995   case Builtin::BI__builtin_creall:
1996   case Builtin::BIcreal:
1997   case Builtin::BIcrealf:
1998   case Builtin::BIcreall: {
1999     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
2000     return RValue::get(ComplexVal.first);
2001   }
2002
2003   case Builtin::BI__builtin_dump_struct: {
2004     llvm::Type *LLVMIntTy = getTypes().ConvertType(getContext().IntTy);
2005     llvm::FunctionType *LLVMFuncType = llvm::FunctionType::get(
2006         LLVMIntTy, {llvm::Type::getInt8PtrTy(getLLVMContext())}, true);
2007
2008     Value *Func = EmitScalarExpr(E->getArg(1)->IgnoreImpCasts());
2009     CharUnits Arg0Align = EmitPointerWithAlignment(E->getArg(0)).getAlignment();
2010
2011     const Expr *Arg0 = E->getArg(0)->IgnoreImpCasts();
2012     QualType Arg0Type = Arg0->getType()->getPointeeType();
2013
2014     Value *RecordPtr = EmitScalarExpr(Arg0);
2015     Value *Res = dumpRecord(*this, Arg0Type, RecordPtr, Arg0Align,
2016                             {LLVMFuncType, Func}, 0);
2017     return RValue::get(Res);
2018   }
2019
2020   case Builtin::BI__builtin_preserve_access_index: {
2021     // Only enabled preserved access index region when debuginfo
2022     // is available as debuginfo is needed to preserve user-level
2023     // access pattern.
2024     if (!getDebugInfo()) {
2025       CGM.Error(E->getExprLoc(), "using builtin_preserve_access_index() without -g");
2026       return RValue::get(EmitScalarExpr(E->getArg(0)));
2027     }
2028
2029     // Nested builtin_preserve_access_index() not supported
2030     if (IsInPreservedAIRegion) {
2031       CGM.Error(E->getExprLoc(), "nested builtin_preserve_access_index() not supported");
2032       return RValue::get(EmitScalarExpr(E->getArg(0)));
2033     }
2034
2035     IsInPreservedAIRegion = true;
2036     Value *Res = EmitScalarExpr(E->getArg(0));
2037     IsInPreservedAIRegion = false;
2038     return RValue::get(Res);
2039   }
2040
2041   case Builtin::BI__builtin_cimag:
2042   case Builtin::BI__builtin_cimagf:
2043   case Builtin::BI__builtin_cimagl:
2044   case Builtin::BIcimag:
2045   case Builtin::BIcimagf:
2046   case Builtin::BIcimagl: {
2047     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
2048     return RValue::get(ComplexVal.second);
2049   }
2050
2051   case Builtin::BI__builtin_clrsb:
2052   case Builtin::BI__builtin_clrsbl:
2053   case Builtin::BI__builtin_clrsbll: {
2054     // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
2055     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2056
2057     llvm::Type *ArgType = ArgValue->getType();
2058     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
2059
2060     llvm::Type *ResultType = ConvertType(E->getType());
2061     Value *Zero = llvm::Constant::getNullValue(ArgType);
2062     Value *IsNeg = Builder.CreateICmpSLT(ArgValue, Zero, "isneg");
2063     Value *Inverse = Builder.CreateNot(ArgValue, "not");
2064     Value *Tmp = Builder.CreateSelect(IsNeg, Inverse, ArgValue);
2065     Value *Ctlz = Builder.CreateCall(F, {Tmp, Builder.getFalse()});
2066     Value *Result = Builder.CreateSub(Ctlz, llvm::ConstantInt::get(ArgType, 1));
2067     Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2068                                    "cast");
2069     return RValue::get(Result);
2070   }
2071   case Builtin::BI__builtin_ctzs:
2072   case Builtin::BI__builtin_ctz:
2073   case Builtin::BI__builtin_ctzl:
2074   case Builtin::BI__builtin_ctzll: {
2075     Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero);
2076
2077     llvm::Type *ArgType = ArgValue->getType();
2078     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
2079
2080     llvm::Type *ResultType = ConvertType(E->getType());
2081     Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
2082     Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
2083     if (Result->getType() != ResultType)
2084       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2085                                      "cast");
2086     return RValue::get(Result);
2087   }
2088   case Builtin::BI__builtin_clzs:
2089   case Builtin::BI__builtin_clz:
2090   case Builtin::BI__builtin_clzl:
2091   case Builtin::BI__builtin_clzll: {
2092     Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero);
2093
2094     llvm::Type *ArgType = ArgValue->getType();
2095     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
2096
2097     llvm::Type *ResultType = ConvertType(E->getType());
2098     Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
2099     Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
2100     if (Result->getType() != ResultType)
2101       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2102                                      "cast");
2103     return RValue::get(Result);
2104   }
2105   case Builtin::BI__builtin_ffs:
2106   case Builtin::BI__builtin_ffsl:
2107   case Builtin::BI__builtin_ffsll: {
2108     // ffs(x) -> x ? cttz(x) + 1 : 0
2109     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2110
2111     llvm::Type *ArgType = ArgValue->getType();
2112     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
2113
2114     llvm::Type *ResultType = ConvertType(E->getType());
2115     Value *Tmp =
2116         Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
2117                           llvm::ConstantInt::get(ArgType, 1));
2118     Value *Zero = llvm::Constant::getNullValue(ArgType);
2119     Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
2120     Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
2121     if (Result->getType() != ResultType)
2122       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2123                                      "cast");
2124     return RValue::get(Result);
2125   }
2126   case Builtin::BI__builtin_parity:
2127   case Builtin::BI__builtin_parityl:
2128   case Builtin::BI__builtin_parityll: {
2129     // parity(x) -> ctpop(x) & 1
2130     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2131
2132     llvm::Type *ArgType = ArgValue->getType();
2133     Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
2134
2135     llvm::Type *ResultType = ConvertType(E->getType());
2136     Value *Tmp = Builder.CreateCall(F, ArgValue);
2137     Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
2138     if (Result->getType() != ResultType)
2139       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2140                                      "cast");
2141     return RValue::get(Result);
2142   }
2143   case Builtin::BI__lzcnt16:
2144   case Builtin::BI__lzcnt:
2145   case Builtin::BI__lzcnt64: {
2146     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2147
2148     llvm::Type *ArgType = ArgValue->getType();
2149     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
2150
2151     llvm::Type *ResultType = ConvertType(E->getType());
2152     Value *Result = Builder.CreateCall(F, {ArgValue, Builder.getFalse()});
2153     if (Result->getType() != ResultType)
2154       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2155                                      "cast");
2156     return RValue::get(Result);
2157   }
2158   case Builtin::BI__popcnt16:
2159   case Builtin::BI__popcnt:
2160   case Builtin::BI__popcnt64:
2161   case Builtin::BI__builtin_popcount:
2162   case Builtin::BI__builtin_popcountl:
2163   case Builtin::BI__builtin_popcountll: {
2164     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2165
2166     llvm::Type *ArgType = ArgValue->getType();
2167     Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
2168
2169     llvm::Type *ResultType = ConvertType(E->getType());
2170     Value *Result = Builder.CreateCall(F, ArgValue);
2171     if (Result->getType() != ResultType)
2172       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2173                                      "cast");
2174     return RValue::get(Result);
2175   }
2176   case Builtin::BI__builtin_unpredictable: {
2177     // Always return the argument of __builtin_unpredictable. LLVM does not
2178     // handle this builtin. Metadata for this builtin should be added directly
2179     // to instructions such as branches or switches that use it.
2180     return RValue::get(EmitScalarExpr(E->getArg(0)));
2181   }
2182   case Builtin::BI__builtin_expect: {
2183     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2184     llvm::Type *ArgType = ArgValue->getType();
2185
2186     Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
2187     // Don't generate llvm.expect on -O0 as the backend won't use it for
2188     // anything.
2189     // Note, we still IRGen ExpectedValue because it could have side-effects.
2190     if (CGM.getCodeGenOpts().OptimizationLevel == 0)
2191       return RValue::get(ArgValue);
2192
2193     Function *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
2194     Value *Result =
2195         Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
2196     return RValue::get(Result);
2197   }
2198   case Builtin::BI__builtin_expect_with_probability: {
2199     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2200     llvm::Type *ArgType = ArgValue->getType();
2201
2202     Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
2203     llvm::APFloat Probability(0.0);
2204     const Expr *ProbArg = E->getArg(2);
2205     bool EvalSucceed = ProbArg->EvaluateAsFloat(Probability, CGM.getContext());
2206     assert(EvalSucceed && "probability should be able to evaluate as float");
2207     (void)EvalSucceed;
2208     bool LoseInfo = false;
2209     Probability.convert(llvm::APFloat::IEEEdouble(),
2210                         llvm::RoundingMode::Dynamic, &LoseInfo);
2211     llvm::Type *Ty = ConvertType(ProbArg->getType());
2212     Constant *Confidence = ConstantFP::get(Ty, Probability);
2213     // Don't generate llvm.expect.with.probability on -O0 as the backend
2214     // won't use it for anything.
2215     // Note, we still IRGen ExpectedValue because it could have side-effects.
2216     if (CGM.getCodeGenOpts().OptimizationLevel == 0)
2217       return RValue::get(ArgValue);
2218
2219     Function *FnExpect =
2220         CGM.getIntrinsic(Intrinsic::expect_with_probability, ArgType);
2221     Value *Result = Builder.CreateCall(
2222         FnExpect, {ArgValue, ExpectedValue, Confidence}, "expval");
2223     return RValue::get(Result);
2224   }
2225   case Builtin::BI__builtin_assume_aligned: {
2226     const Expr *Ptr = E->getArg(0);
2227     Value *PtrValue = EmitScalarExpr(Ptr);
2228     Value *OffsetValue =
2229       (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr;
2230
2231     Value *AlignmentValue = EmitScalarExpr(E->getArg(1));
2232     ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
2233     if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
2234       AlignmentCI = ConstantInt::get(AlignmentCI->getType(),
2235                                      llvm::Value::MaximumAlignment);
2236
2237     emitAlignmentAssumption(PtrValue, Ptr,
2238                             /*The expr loc is sufficient.*/ SourceLocation(),
2239                             AlignmentCI, OffsetValue);
2240     return RValue::get(PtrValue);
2241   }
2242   case Builtin::BI__assume:
2243   case Builtin::BI__builtin_assume: {
2244     if (E->getArg(0)->HasSideEffects(getContext()))
2245       return RValue::get(nullptr);
2246
2247     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2248     Function *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
2249     return RValue::get(Builder.CreateCall(FnAssume, ArgValue));
2250   }
2251   case Builtin::BI__builtin_bswap16:
2252   case Builtin::BI__builtin_bswap32:
2253   case Builtin::BI__builtin_bswap64: {
2254     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap));
2255   }
2256   case Builtin::BI__builtin_bitreverse8:
2257   case Builtin::BI__builtin_bitreverse16:
2258   case Builtin::BI__builtin_bitreverse32:
2259   case Builtin::BI__builtin_bitreverse64: {
2260     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse));
2261   }
2262   case Builtin::BI__builtin_rotateleft8:
2263   case Builtin::BI__builtin_rotateleft16:
2264   case Builtin::BI__builtin_rotateleft32:
2265   case Builtin::BI__builtin_rotateleft64:
2266   case Builtin::BI_rotl8: // Microsoft variants of rotate left
2267   case Builtin::BI_rotl16:
2268   case Builtin::BI_rotl:
2269   case Builtin::BI_lrotl:
2270   case Builtin::BI_rotl64:
2271     return emitRotate(E, false);
2272
2273   case Builtin::BI__builtin_rotateright8:
2274   case Builtin::BI__builtin_rotateright16:
2275   case Builtin::BI__builtin_rotateright32:
2276   case Builtin::BI__builtin_rotateright64:
2277   case Builtin::BI_rotr8: // Microsoft variants of rotate right
2278   case Builtin::BI_rotr16:
2279   case Builtin::BI_rotr:
2280   case Builtin::BI_lrotr:
2281   case Builtin::BI_rotr64:
2282     return emitRotate(E, true);
2283
2284   case Builtin::BI__builtin_constant_p: {
2285     llvm::Type *ResultType = ConvertType(E->getType());
2286
2287     const Expr *Arg = E->getArg(0);
2288     QualType ArgType = Arg->getType();
2289     // FIXME: The allowance for Obj-C pointers and block pointers is historical
2290     // and likely a mistake.
2291     if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
2292         !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
2293       // Per the GCC documentation, only numeric constants are recognized after
2294       // inlining.
2295       return RValue::get(ConstantInt::get(ResultType, 0));
2296
2297     if (Arg->HasSideEffects(getContext()))
2298       // The argument is unevaluated, so be conservative if it might have
2299       // side-effects.
2300       return RValue::get(ConstantInt::get(ResultType, 0));
2301
2302     Value *ArgValue = EmitScalarExpr(Arg);
2303     if (ArgType->isObjCObjectPointerType()) {
2304       // Convert Objective-C objects to id because we cannot distinguish between
2305       // LLVM types for Obj-C classes as they are opaque.
2306       ArgType = CGM.getContext().getObjCIdType();
2307       ArgValue = Builder.CreateBitCast(ArgValue, ConvertType(ArgType));
2308     }
2309     Function *F =
2310         CGM.getIntrinsic(Intrinsic::is_constant, ConvertType(ArgType));
2311     Value *Result = Builder.CreateCall(F, ArgValue);
2312     if (Result->getType() != ResultType)
2313       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/false);
2314     return RValue::get(Result);
2315   }
2316   case Builtin::BI__builtin_dynamic_object_size:
2317   case Builtin::BI__builtin_object_size: {
2318     unsigned Type =
2319         E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
2320     auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
2321
2322     // We pass this builtin onto the optimizer so that it can figure out the
2323     // object size in more complex cases.
2324     bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
2325     return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType,
2326                                              /*EmittedE=*/nullptr, IsDynamic));
2327   }
2328   case Builtin::BI__builtin_prefetch: {
2329     Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
2330     // FIXME: Technically these constants should of type 'int', yes?
2331     RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
2332       llvm::ConstantInt::get(Int32Ty, 0);
2333     Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
2334       llvm::ConstantInt::get(Int32Ty, 3);
2335     Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
2336     Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
2337     return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data}));
2338   }
2339   case Builtin::BI__builtin_readcyclecounter: {
2340     Function *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
2341     return RValue::get(Builder.CreateCall(F));
2342   }
2343   case Builtin::BI__builtin___clear_cache: {
2344     Value *Begin = EmitScalarExpr(E->getArg(0));
2345     Value *End = EmitScalarExpr(E->getArg(1));
2346     Function *F = CGM.getIntrinsic(Intrinsic::clear_cache);
2347     return RValue::get(Builder.CreateCall(F, {Begin, End}));
2348   }
2349   case Builtin::BI__builtin_trap:
2350     return RValue::get(EmitTrapCall(Intrinsic::trap));
2351   case Builtin::BI__debugbreak:
2352     return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
2353   case Builtin::BI__builtin_unreachable: {
2354     EmitUnreachable(E->getExprLoc());
2355
2356     // We do need to preserve an insertion point.
2357     EmitBlock(createBasicBlock("unreachable.cont"));
2358
2359     return RValue::get(nullptr);
2360   }
2361
2362   case Builtin::BI__builtin_powi:
2363   case Builtin::BI__builtin_powif:
2364   case Builtin::BI__builtin_powil:
2365     return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(
2366         *this, E, Intrinsic::powi, Intrinsic::experimental_constrained_powi));
2367
2368   case Builtin::BI__builtin_isgreater:
2369   case Builtin::BI__builtin_isgreaterequal:
2370   case Builtin::BI__builtin_isless:
2371   case Builtin::BI__builtin_islessequal:
2372   case Builtin::BI__builtin_islessgreater:
2373   case Builtin::BI__builtin_isunordered: {
2374     // Ordered comparisons: we know the arguments to these are matching scalar
2375     // floating point values.
2376     Value *LHS = EmitScalarExpr(E->getArg(0));
2377     Value *RHS = EmitScalarExpr(E->getArg(1));
2378
2379     switch (BuiltinID) {
2380     default: llvm_unreachable("Unknown ordered comparison");
2381     case Builtin::BI__builtin_isgreater:
2382       LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
2383       break;
2384     case Builtin::BI__builtin_isgreaterequal:
2385       LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
2386       break;
2387     case Builtin::BI__builtin_isless:
2388       LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
2389       break;
2390     case Builtin::BI__builtin_islessequal:
2391       LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
2392       break;
2393     case Builtin::BI__builtin_islessgreater:
2394       LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
2395       break;
2396     case Builtin::BI__builtin_isunordered:
2397       LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
2398       break;
2399     }
2400     // ZExt bool to int type.
2401     return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
2402   }
2403   case Builtin::BI__builtin_isnan: {
2404     Value *V = EmitScalarExpr(E->getArg(0));
2405     V = Builder.CreateFCmpUNO(V, V, "cmp");
2406     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
2407   }
2408
2409   case Builtin::BI__builtin_matrix_transpose: {
2410     const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
2411     Value *MatValue = EmitScalarExpr(E->getArg(0));
2412     MatrixBuilder<CGBuilderTy> MB(Builder);
2413     Value *Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
2414                                              MatrixTy->getNumColumns());
2415     return RValue::get(Result);
2416   }
2417
2418   case Builtin::BI__builtin_matrix_column_major_load: {
2419     MatrixBuilder<CGBuilderTy> MB(Builder);
2420     // Emit everything that isn't dependent on the first parameter type
2421     Value *Stride = EmitScalarExpr(E->getArg(3));
2422     const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
2423     auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
2424     assert(PtrTy && "arg0 must be of pointer type");
2425     bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
2426
2427     Address Src = EmitPointerWithAlignment(E->getArg(0));
2428     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(),
2429                         E->getArg(0)->getExprLoc(), FD, 0);
2430     Value *Result = MB.CreateColumnMajorLoad(
2431         Src.getPointer(), Align(Src.getAlignment().getQuantity()), Stride,
2432         IsVolatile, ResultTy->getNumRows(), ResultTy->getNumColumns(),
2433         "matrix");
2434     return RValue::get(Result);
2435   }
2436
2437   case Builtin::BI__builtin_matrix_column_major_store: {
2438     MatrixBuilder<CGBuilderTy> MB(Builder);
2439     Value *Matrix = EmitScalarExpr(E->getArg(0));
2440     Address Dst = EmitPointerWithAlignment(E->getArg(1));
2441     Value *Stride = EmitScalarExpr(E->getArg(2));
2442
2443     const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
2444     auto *PtrTy = E->getArg(1)->getType()->getAs<PointerType>();
2445     assert(PtrTy && "arg1 must be of pointer type");
2446     bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
2447
2448     EmitNonNullArgCheck(RValue::get(Dst.getPointer()), E->getArg(1)->getType(),
2449                         E->getArg(1)->getExprLoc(), FD, 0);
2450     Value *Result = MB.CreateColumnMajorStore(
2451         Matrix, Dst.getPointer(), Align(Dst.getAlignment().getQuantity()),
2452         Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
2453     return RValue::get(Result);
2454   }
2455
2456   case Builtin::BIfinite:
2457   case Builtin::BI__finite:
2458   case Builtin::BIfinitef:
2459   case Builtin::BI__finitef:
2460   case Builtin::BIfinitel:
2461   case Builtin::BI__finitel:
2462   case Builtin::BI__builtin_isinf:
2463   case Builtin::BI__builtin_isfinite: {
2464     // isinf(x)    --> fabs(x) == infinity
2465     // isfinite(x) --> fabs(x) != infinity
2466     // x != NaN via the ordered compare in either case.
2467     Value *V = EmitScalarExpr(E->getArg(0));
2468     Value *Fabs = EmitFAbs(*this, V);
2469     Constant *Infinity = ConstantFP::getInfinity(V->getType());
2470     CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
2471                                   ? CmpInst::FCMP_OEQ
2472                                   : CmpInst::FCMP_ONE;
2473     Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity, "cmpinf");
2474     return RValue::get(Builder.CreateZExt(FCmp, ConvertType(E->getType())));
2475   }
2476
2477   case Builtin::BI__builtin_isinf_sign: {
2478     // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
2479     Value *Arg = EmitScalarExpr(E->getArg(0));
2480     Value *AbsArg = EmitFAbs(*this, Arg);
2481     Value *IsInf = Builder.CreateFCmpOEQ(
2482         AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
2483     Value *IsNeg = EmitSignBit(*this, Arg);
2484
2485     llvm::Type *IntTy = ConvertType(E->getType());
2486     Value *Zero = Constant::getNullValue(IntTy);
2487     Value *One = ConstantInt::get(IntTy, 1);
2488     Value *NegativeOne = ConstantInt::get(IntTy, -1);
2489     Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
2490     Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
2491     return RValue::get(Result);
2492   }
2493
2494   case Builtin::BI__builtin_isnormal: {
2495     // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
2496     Value *V = EmitScalarExpr(E->getArg(0));
2497     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
2498
2499     Value *Abs = EmitFAbs(*this, V);
2500     Value *IsLessThanInf =
2501       Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
2502     APFloat Smallest = APFloat::getSmallestNormalized(
2503                    getContext().getFloatTypeSemantics(E->getArg(0)->getType()));
2504     Value *IsNormal =
2505       Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
2506                             "isnormal");
2507     V = Builder.CreateAnd(Eq, IsLessThanInf, "and");
2508     V = Builder.CreateAnd(V, IsNormal, "and");
2509     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
2510   }
2511
2512   case Builtin::BI__builtin_flt_rounds: {
2513     Function *F = CGM.getIntrinsic(Intrinsic::flt_rounds);
2514
2515     llvm::Type *ResultType = ConvertType(E->getType());
2516     Value *Result = Builder.CreateCall(F);
2517     if (Result->getType() != ResultType)
2518       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2519                                      "cast");
2520     return RValue::get(Result);
2521   }
2522
2523   case Builtin::BI__builtin_fpclassify: {
2524     Value *V = EmitScalarExpr(E->getArg(5));
2525     llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
2526
2527     // Create Result
2528     BasicBlock *Begin = Builder.GetInsertBlock();
2529     BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
2530     Builder.SetInsertPoint(End);
2531     PHINode *Result =
2532       Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
2533                         "fpclassify_result");
2534
2535     // if (V==0) return FP_ZERO
2536     Builder.SetInsertPoint(Begin);
2537     Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
2538                                           "iszero");
2539     Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
2540     BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
2541     Builder.CreateCondBr(IsZero, End, NotZero);
2542     Result->addIncoming(ZeroLiteral, Begin);
2543
2544     // if (V != V) return FP_NAN
2545     Builder.SetInsertPoint(NotZero);
2546     Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
2547     Value *NanLiteral = EmitScalarExpr(E->getArg(0));
2548     BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
2549     Builder.CreateCondBr(IsNan, End, NotNan);
2550     Result->addIncoming(NanLiteral, NotZero);
2551
2552     // if (fabs(V) == infinity) return FP_INFINITY
2553     Builder.SetInsertPoint(NotNan);
2554     Value *VAbs = EmitFAbs(*this, V);
2555     Value *IsInf =
2556       Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
2557                             "isinf");
2558     Value *InfLiteral = EmitScalarExpr(E->getArg(1));
2559     BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
2560     Builder.CreateCondBr(IsInf, End, NotInf);
2561     Result->addIncoming(InfLiteral, NotNan);
2562
2563     // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
2564     Builder.SetInsertPoint(NotInf);
2565     APFloat Smallest = APFloat::getSmallestNormalized(
2566         getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
2567     Value *IsNormal =
2568       Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
2569                             "isnormal");
2570     Value *NormalResult =
2571       Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
2572                            EmitScalarExpr(E->getArg(3)));
2573     Builder.CreateBr(End);
2574     Result->addIncoming(NormalResult, NotInf);
2575
2576     // return Result
2577     Builder.SetInsertPoint(End);
2578     return RValue::get(Result);
2579   }
2580
2581   case Builtin::BIalloca:
2582   case Builtin::BI_alloca:
2583   case Builtin::BI__builtin_alloca: {
2584     Value *Size = EmitScalarExpr(E->getArg(0));
2585     const TargetInfo &TI = getContext().getTargetInfo();
2586     // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
2587     const Align SuitableAlignmentInBytes =
2588         CGM.getContext()
2589             .toCharUnitsFromBits(TI.getSuitableAlign())
2590             .getAsAlign();
2591     AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
2592     AI->setAlignment(SuitableAlignmentInBytes);
2593     initializeAlloca(*this, AI, Size, SuitableAlignmentInBytes);
2594     return RValue::get(AI);
2595   }
2596
2597   case Builtin::BI__builtin_alloca_with_align: {
2598     Value *Size = EmitScalarExpr(E->getArg(0));
2599     Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
2600     auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
2601     unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
2602     const Align AlignmentInBytes =
2603         CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getAsAlign();
2604     AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
2605     AI->setAlignment(AlignmentInBytes);
2606     initializeAlloca(*this, AI, Size, AlignmentInBytes);
2607     return RValue::get(AI);
2608   }
2609
2610   case Builtin::BIbzero:
2611   case Builtin::BI__builtin_bzero: {
2612     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2613     Value *SizeVal = EmitScalarExpr(E->getArg(1));
2614     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
2615                         E->getArg(0)->getExprLoc(), FD, 0);
2616     Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
2617     return RValue::get(nullptr);
2618   }
2619   case Builtin::BImemcpy:
2620   case Builtin::BI__builtin_memcpy:
2621   case Builtin::BImempcpy:
2622   case Builtin::BI__builtin_mempcpy: {
2623     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2624     Address Src = EmitPointerWithAlignment(E->getArg(1));
2625     Value *SizeVal = EmitScalarExpr(E->getArg(2));
2626     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
2627                         E->getArg(0)->getExprLoc(), FD, 0);
2628     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
2629                         E->getArg(1)->getExprLoc(), FD, 1);
2630     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
2631     if (BuiltinID == Builtin::BImempcpy ||
2632         BuiltinID == Builtin::BI__builtin_mempcpy)
2633       return RValue::get(Builder.CreateInBoundsGEP(Dest.getPointer(), SizeVal));
2634     else
2635       return RValue::get(Dest.getPointer());
2636   }
2637
2638   case Builtin::BI__builtin_memcpy_inline: {
2639     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2640     Address Src = EmitPointerWithAlignment(E->getArg(1));
2641     uint64_t Size =
2642         E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
2643     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
2644                         E->getArg(0)->getExprLoc(), FD, 0);
2645     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
2646                         E->getArg(1)->getExprLoc(), FD, 1);
2647     Builder.CreateMemCpyInline(Dest, Src, Size);
2648     return RValue::get(nullptr);
2649   }
2650
2651   case Builtin::BI__builtin_char_memchr:
2652     BuiltinID = Builtin::BI__builtin_memchr;
2653     break;
2654
2655   case Builtin::BI__builtin___memcpy_chk: {
2656     // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
2657     Expr::EvalResult SizeResult, DstSizeResult;
2658     if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
2659         !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
2660       break;
2661     llvm::APSInt Size = SizeResult.Val.getInt();
2662     llvm::APSInt DstSize = DstSizeResult.Val.getInt();
2663     if (Size.ugt(DstSize))
2664       break;
2665     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2666     Address Src = EmitPointerWithAlignment(E->getArg(1));
2667     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
2668     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
2669     return RValue::get(Dest.getPointer());
2670   }
2671
2672   case Builtin::BI__builtin_objc_memmove_collectable: {
2673     Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
2674     Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
2675     Value *SizeVal = EmitScalarExpr(E->getArg(2));
2676     CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
2677                                                   DestAddr, SrcAddr, SizeVal);
2678     return RValue::get(DestAddr.getPointer());
2679   }
2680
2681   case Builtin::BI__builtin___memmove_chk: {
2682     // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
2683     Expr::EvalResult SizeResult, DstSizeResult;
2684     if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
2685         !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
2686       break;
2687     llvm::APSInt Size = SizeResult.Val.getInt();
2688     llvm::APSInt DstSize = DstSizeResult.Val.getInt();
2689     if (Size.ugt(DstSize))
2690       break;
2691     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2692     Address Src = EmitPointerWithAlignment(E->getArg(1));
2693     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
2694     Builder.CreateMemMove(Dest, Src, SizeVal, false);
2695     return RValue::get(Dest.getPointer());
2696   }
2697
2698   case Builtin::BImemmove:
2699   case Builtin::BI__builtin_memmove: {
2700     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2701     Address Src = EmitPointerWithAlignment(E->getArg(1));
2702     Value *SizeVal = EmitScalarExpr(E->getArg(2));
2703     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
2704                         E->getArg(0)->getExprLoc(), FD, 0);
2705     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
2706                         E->getArg(1)->getExprLoc(), FD, 1);
2707     Builder.CreateMemMove(Dest, Src, SizeVal, false);
2708     return RValue::get(Dest.getPointer());
2709   }
2710   case Builtin::BImemset:
2711   case Builtin::BI__builtin_memset: {
2712     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2713     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
2714                                          Builder.getInt8Ty());
2715     Value *SizeVal = EmitScalarExpr(E->getArg(2));
2716     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
2717                         E->getArg(0)->getExprLoc(), FD, 0);
2718     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
2719     return RValue::get(Dest.getPointer());
2720   }
2721   case Builtin::BI__builtin___memset_chk: {
2722     // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
2723     Expr::EvalResult SizeResult, DstSizeResult;
2724     if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
2725         !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
2726       break;
2727     llvm::APSInt Size = SizeResult.Val.getInt();
2728     llvm::APSInt DstSize = DstSizeResult.Val.getInt();
2729     if (Size.ugt(DstSize))
2730       break;
2731     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2732     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
2733                                          Builder.getInt8Ty());
2734     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
2735     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
2736     return RValue::get(Dest.getPointer());
2737   }
2738   case Builtin::BI__builtin_wmemcmp: {
2739     // The MSVC runtime library does not provide a definition of wmemcmp, so we
2740     // need an inline implementation.
2741     if (!getTarget().getTriple().isOSMSVCRT())
2742       break;
2743
2744     llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
2745
2746     Value *Dst = EmitScalarExpr(E->getArg(0));
2747     Value *Src = EmitScalarExpr(E->getArg(1));
2748     Value *Size = EmitScalarExpr(E->getArg(2));
2749
2750     BasicBlock *Entry = Builder.GetInsertBlock();
2751     BasicBlock *CmpGT = createBasicBlock("wmemcmp.gt");
2752     BasicBlock *CmpLT = createBasicBlock("wmemcmp.lt");
2753     BasicBlock *Next = createBasicBlock("wmemcmp.next");
2754     BasicBlock *Exit = createBasicBlock("wmemcmp.exit");
2755     Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
2756     Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
2757
2758     EmitBlock(CmpGT);
2759     PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
2760     DstPhi->addIncoming(Dst, Entry);
2761     PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
2762     SrcPhi->addIncoming(Src, Entry);
2763     PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
2764     SizePhi->addIncoming(Size, Entry);
2765     CharUnits WCharAlign =
2766         getContext().getTypeAlignInChars(getContext().WCharTy);
2767     Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
2768     Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
2769     Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
2770     Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
2771
2772     EmitBlock(CmpLT);
2773     Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
2774     Builder.CreateCondBr(DstLtSrc, Exit, Next);
2775
2776     EmitBlock(Next);
2777     Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
2778     Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
2779     Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
2780     Value *NextSizeEq0 =
2781         Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
2782     Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
2783     DstPhi->addIncoming(NextDst, Next);
2784     SrcPhi->addIncoming(NextSrc, Next);
2785     SizePhi->addIncoming(NextSize, Next);
2786
2787     EmitBlock(Exit);
2788     PHINode *Ret = Builder.CreatePHI(IntTy, 4);
2789     Ret->addIncoming(ConstantInt::get(IntTy, 0), Entry);
2790     Ret->addIncoming(ConstantInt::get(IntTy, 1), CmpGT);
2791     Ret->addIncoming(ConstantInt::get(IntTy, -1), CmpLT);
2792     Ret->addIncoming(ConstantInt::get(IntTy, 0), Next);
2793     return RValue::get(Ret);
2794   }
2795   case Builtin::BI__builtin_dwarf_cfa: {
2796     // The offset in bytes from the first argument to the CFA.
2797     //
2798     // Why on earth is this in the frontend?  Is there any reason at
2799     // all that the backend can't reasonably determine this while
2800     // lowering llvm.eh.dwarf.cfa()?
2801     //
2802     // TODO: If there's a satisfactory reason, add a target hook for
2803     // this instead of hard-coding 0, which is correct for most targets.
2804     int32_t Offset = 0;
2805
2806     Function *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
2807     return RValue::get(Builder.CreateCall(F,
2808                                       llvm::ConstantInt::get(Int32Ty, Offset)));
2809   }
2810   case Builtin::BI__builtin_return_address: {
2811     Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
2812                                                    getContext().UnsignedIntTy);
2813     Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
2814     return RValue::get(Builder.CreateCall(F, Depth));
2815   }
2816   case Builtin::BI_ReturnAddress: {
2817     Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
2818     return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
2819   }
2820   case Builtin::BI__builtin_frame_address: {
2821     Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
2822                                                    getContext().UnsignedIntTy);
2823     Function *F = CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy);
2824     return RValue::get(Builder.CreateCall(F, Depth));
2825   }
2826   case Builtin::BI__builtin_extract_return_addr: {
2827     Value *Address = EmitScalarExpr(E->getArg(0));
2828     Value *Result = getTargetHooks().decodeReturnAddress(*this, Address);
2829     return RValue::get(Result);
2830   }
2831   case Builtin::BI__builtin_frob_return_addr: {
2832     Value *Address = EmitScalarExpr(E->getArg(0));
2833     Value *Result = getTargetHooks().encodeReturnAddress(*this, Address);
2834     return RValue::get(Result);
2835   }
2836   case Builtin::BI__builtin_dwarf_sp_column: {
2837     llvm::IntegerType *Ty
2838       = cast<llvm::IntegerType>(ConvertType(E->getType()));
2839     int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
2840     if (Column == -1) {
2841       CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
2842       return RValue::get(llvm::UndefValue::get(Ty));
2843     }
2844     return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
2845   }
2846   case Builtin::BI__builtin_init_dwarf_reg_size_table: {
2847     Value *Address = EmitScalarExpr(E->getArg(0));
2848     if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
2849       CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
2850     return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
2851   }
2852   case Builtin::BI__builtin_eh_return: {
2853     Value *Int = EmitScalarExpr(E->getArg(0));
2854     Value *Ptr = EmitScalarExpr(E->getArg(1));
2855
2856     llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
2857     assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
2858            "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
2859     Function *F =
2860         CGM.getIntrinsic(IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
2861                                                     : Intrinsic::eh_return_i64);
2862     Builder.CreateCall(F, {Int, Ptr});
2863     Builder.CreateUnreachable();
2864
2865     // We do need to preserve an insertion point.
2866     EmitBlock(createBasicBlock("builtin_eh_return.cont"));
2867
2868     return RValue::get(nullptr);
2869   }
2870   case Builtin::BI__builtin_unwind_init: {
2871     Function *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
2872     return RValue::get(Builder.CreateCall(F));
2873   }
2874   case Builtin::BI__builtin_extend_pointer: {
2875     // Extends a pointer to the size of an _Unwind_Word, which is
2876     // uint64_t on all platforms.  Generally this gets poked into a
2877     // register and eventually used as an address, so if the
2878     // addressing registers are wider than pointers and the platform
2879     // doesn't implicitly ignore high-order bits when doing
2880     // addressing, we need to make sure we zext / sext based on
2881     // the platform's expectations.
2882     //
2883     // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
2884
2885     // Cast the pointer to intptr_t.
2886     Value *Ptr = EmitScalarExpr(E->getArg(0));
2887     Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
2888
2889     // If that's 64 bits, we're done.
2890     if (IntPtrTy->getBitWidth() == 64)
2891       return RValue::get(Result);
2892
2893     // Otherwise, ask the codegen data what to do.
2894     if (getTargetHooks().extendPointerWithSExt())
2895       return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
2896     else
2897       return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
2898   }
2899   case Builtin::BI__builtin_setjmp: {
2900     // Buffer is a void**.
2901     Address Buf = EmitPointerWithAlignment(E->getArg(0));
2902
2903     // Store the frame pointer to the setjmp buffer.
2904     Value *FrameAddr = Builder.CreateCall(
2905         CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy),
2906         ConstantInt::get(Int32Ty, 0));
2907     Builder.CreateStore(FrameAddr, Buf);
2908
2909     // Store the stack pointer to the setjmp buffer.
2910     Value *StackAddr =
2911         Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
2912     Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2);
2913     Builder.CreateStore(StackAddr, StackSaveSlot);
2914
2915     // Call LLVM's EH setjmp, which is lightweight.
2916     Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
2917     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
2918     return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
2919   }
2920   case Builtin::BI__builtin_longjmp: {
2921     Value *Buf = EmitScalarExpr(E->getArg(0));
2922     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
2923
2924     // Call LLVM's EH longjmp, which is lightweight.
2925     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
2926
2927     // longjmp doesn't return; mark this as unreachable.
2928     Builder.CreateUnreachable();
2929
2930     // We do need to preserve an insertion point.
2931     EmitBlock(createBasicBlock("longjmp.cont"));
2932
2933     return RValue::get(nullptr);
2934   }
2935   case Builtin::BI__builtin_launder: {
2936     const Expr *Arg = E->getArg(0);
2937     QualType ArgTy = Arg->getType()->getPointeeType();
2938     Value *Ptr = EmitScalarExpr(Arg);
2939     if (TypeRequiresBuiltinLaunder(CGM, ArgTy))
2940       Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
2941
2942     return RValue::get(Ptr);
2943   }
2944   case Builtin::BI__sync_fetch_and_add:
2945   case Builtin::BI__sync_fetch_and_sub:
2946   case Builtin::BI__sync_fetch_and_or:
2947   case Builtin::BI__sync_fetch_and_and:
2948   case Builtin::BI__sync_fetch_and_xor:
2949   case Builtin::BI__sync_fetch_and_nand:
2950   case Builtin::BI__sync_add_and_fetch:
2951   case Builtin::BI__sync_sub_and_fetch:
2952   case Builtin::BI__sync_and_and_fetch:
2953   case Builtin::BI__sync_or_and_fetch:
2954   case Builtin::BI__sync_xor_and_fetch:
2955   case Builtin::BI__sync_nand_and_fetch:
2956   case Builtin::BI__sync_val_compare_and_swap:
2957   case Builtin::BI__sync_bool_compare_and_swap:
2958   case Builtin::BI__sync_lock_test_and_set:
2959   case Builtin::BI__sync_lock_release:
2960   case Builtin::BI__sync_swap:
2961     llvm_unreachable("Shouldn't make it through sema");
2962   case Builtin::BI__sync_fetch_and_add_1:
2963   case Builtin::BI__sync_fetch_and_add_2:
2964   case Builtin::BI__sync_fetch_and_add_4:
2965   case Builtin::BI__sync_fetch_and_add_8:
2966   case Builtin::BI__sync_fetch_and_add_16:
2967     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
2968   case Builtin::BI__sync_fetch_and_sub_1:
2969   case Builtin::BI__sync_fetch_and_sub_2:
2970   case Builtin::BI__sync_fetch_and_sub_4:
2971   case Builtin::BI__sync_fetch_and_sub_8:
2972   case Builtin::BI__sync_fetch_and_sub_16:
2973     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
2974   case Builtin::BI__sync_fetch_and_or_1:
2975   case Builtin::BI__sync_fetch_and_or_2:
2976   case Builtin::BI__sync_fetch_and_or_4:
2977   case Builtin::BI__sync_fetch_and_or_8:
2978   case Builtin::BI__sync_fetch_and_or_16:
2979     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
2980   case Builtin::BI__sync_fetch_and_and_1:
2981   case Builtin::BI__sync_fetch_and_and_2:
2982   case Builtin::BI__sync_fetch_and_and_4:
2983   case Builtin::BI__sync_fetch_and_and_8:
2984   case Builtin::BI__sync_fetch_and_and_16:
2985     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
2986   case Builtin::BI__sync_fetch_and_xor_1:
2987   case Builtin::BI__sync_fetch_and_xor_2:
2988   case Builtin::BI__sync_fetch_and_xor_4:
2989   case Builtin::BI__sync_fetch_and_xor_8:
2990   case Builtin::BI__sync_fetch_and_xor_16:
2991     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
2992   case Builtin::BI__sync_fetch_and_nand_1:
2993   case Builtin::BI__sync_fetch_and_nand_2:
2994   case Builtin::BI__sync_fetch_and_nand_4:
2995   case Builtin::BI__sync_fetch_and_nand_8:
2996   case Builtin::BI__sync_fetch_and_nand_16:
2997     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E);
2998
2999   // Clang extensions: not overloaded yet.
3000   case Builtin::BI__sync_fetch_and_min:
3001     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
3002   case Builtin::BI__sync_fetch_and_max:
3003     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
3004   case Builtin::BI__sync_fetch_and_umin:
3005     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
3006   case Builtin::BI__sync_fetch_and_umax:
3007     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
3008
3009   case Builtin::BI__sync_add_and_fetch_1:
3010   case Builtin::BI__sync_add_and_fetch_2:
3011   case Builtin::BI__sync_add_and_fetch_4:
3012   case Builtin::BI__sync_add_and_fetch_8:
3013   case Builtin::BI__sync_add_and_fetch_16:
3014     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
3015                                 llvm::Instruction::Add);
3016   case Builtin::BI__sync_sub_and_fetch_1:
3017   case Builtin::BI__sync_sub_and_fetch_2:
3018   case Builtin::BI__sync_sub_and_fetch_4:
3019   case Builtin::BI__sync_sub_and_fetch_8:
3020   case Builtin::BI__sync_sub_and_fetch_16:
3021     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
3022                                 llvm::Instruction::Sub);
3023   case Builtin::BI__sync_and_and_fetch_1:
3024   case Builtin::BI__sync_and_and_fetch_2:
3025   case Builtin::BI__sync_and_and_fetch_4:
3026   case Builtin::BI__sync_and_and_fetch_8:
3027   case Builtin::BI__sync_and_and_fetch_16:
3028     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E,
3029                                 llvm::Instruction::And);
3030   case Builtin::BI__sync_or_and_fetch_1:
3031   case Builtin::BI__sync_or_and_fetch_2:
3032   case Builtin::BI__sync_or_and_fetch_4:
3033   case Builtin::BI__sync_or_and_fetch_8:
3034   case Builtin::BI__sync_or_and_fetch_16:
3035     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
3036                                 llvm::Instruction::Or);
3037   case Builtin::BI__sync_xor_and_fetch_1:
3038   case Builtin::BI__sync_xor_and_fetch_2:
3039   case Builtin::BI__sync_xor_and_fetch_4:
3040   case Builtin::BI__sync_xor_and_fetch_8:
3041   case Builtin::BI__sync_xor_and_fetch_16:
3042     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
3043                                 llvm::Instruction::Xor);
3044   case Builtin::BI__sync_nand_and_fetch_1:
3045   case Builtin::BI__sync_nand_and_fetch_2:
3046   case Builtin::BI__sync_nand_and_fetch_4:
3047   case Builtin::BI__sync_nand_and_fetch_8:
3048   case Builtin::BI__sync_nand_and_fetch_16:
3049     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E,
3050                                 llvm::Instruction::And, true);
3051
3052   case Builtin::BI__sync_val_compare_and_swap_1:
3053   case Builtin::BI__sync_val_compare_and_swap_2:
3054   case Builtin::BI__sync_val_compare_and_swap_4:
3055   case Builtin::BI__sync_val_compare_and_swap_8:
3056   case Builtin::BI__sync_val_compare_and_swap_16:
3057     return RValue::get(MakeAtomicCmpXchgValue(*this, E, false));
3058
3059   case Builtin::BI__sync_bool_compare_and_swap_1:
3060   case Builtin::BI__sync_bool_compare_and_swap_2:
3061   case Builtin::BI__sync_bool_compare_and_swap_4:
3062   case Builtin::BI__sync_bool_compare_and_swap_8:
3063   case Builtin::BI__sync_bool_compare_and_swap_16:
3064     return RValue::get(MakeAtomicCmpXchgValue(*this, E, true));
3065
3066   case Builtin::BI__sync_swap_1:
3067   case Builtin::BI__sync_swap_2:
3068   case Builtin::BI__sync_swap_4:
3069   case Builtin::BI__sync_swap_8:
3070   case Builtin::BI__sync_swap_16:
3071     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
3072
3073   case Builtin::BI__sync_lock_test_and_set_1:
3074   case Builtin::BI__sync_lock_test_and_set_2:
3075   case Builtin::BI__sync_lock_test_and_set_4:
3076   case Builtin::BI__sync_lock_test_and_set_8:
3077   case Builtin::BI__sync_lock_test_and_set_16:
3078     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
3079
3080   case Builtin::BI__sync_lock_release_1:
3081   case Builtin::BI__sync_lock_release_2:
3082   case Builtin::BI__sync_lock_release_4:
3083   case Builtin::BI__sync_lock_release_8:
3084   case Builtin::BI__sync_lock_release_16: {
3085     Value *Ptr = EmitScalarExpr(E->getArg(0));
3086     QualType ElTy = E->getArg(0)->getType()->getPointeeType();
3087     CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
3088     llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
3089                                              StoreSize.getQuantity() * 8);
3090     Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
3091     llvm::StoreInst *Store =
3092       Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
3093                                  StoreSize);
3094     Store->setAtomic(llvm::AtomicOrdering::Release);
3095     return RValue::get(nullptr);
3096   }
3097
3098   case Builtin::BI__sync_synchronize: {
3099     // We assume this is supposed to correspond to a C++0x-style
3100     // sequentially-consistent fence (i.e. this is only usable for
3101     // synchronization, not device I/O or anything like that). This intrinsic
3102     // is really badly designed in the sense that in theory, there isn't
3103     // any way to safely use it... but in practice, it mostly works
3104     // to use it with non-atomic loads and stores to get acquire/release
3105     // semantics.
3106     Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
3107     return RValue::get(nullptr);
3108   }
3109
3110   case Builtin::BI__builtin_nontemporal_load:
3111     return RValue::get(EmitNontemporalLoad(*this, E));
3112   case Builtin::BI__builtin_nontemporal_store:
3113     return RValue::get(EmitNontemporalStore(*this, E));
3114   case Builtin::BI__c11_atomic_is_lock_free:
3115   case Builtin::BI__atomic_is_lock_free: {
3116     // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
3117     // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
3118     // _Atomic(T) is always properly-aligned.
3119     const char *LibCallName = "__atomic_is_lock_free";
3120     CallArgList Args;
3121     Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
3122              getContext().getSizeType());
3123     if (BuiltinID == Builtin::BI__atomic_is_lock_free)
3124       Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
3125                getContext().VoidPtrTy);
3126     else
3127       Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
3128                getContext().VoidPtrTy);
3129     const CGFunctionInfo &FuncInfo =
3130         CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args);
3131     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
3132     llvm::FunctionCallee Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
3133     return EmitCall(FuncInfo, CGCallee::forDirect(Func),
3134                     ReturnValueSlot(), Args);
3135   }
3136
3137   case Builtin::BI__atomic_test_and_set: {
3138     // Look at the argument type to determine whether this is a volatile
3139     // operation. The parameter type is always volatile.
3140     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
3141     bool Volatile =
3142         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
3143
3144     Value *Ptr = EmitScalarExpr(E->getArg(0));
3145     unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
3146     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
3147     Value *NewVal = Builder.getInt8(1);
3148     Value *Order = EmitScalarExpr(E->getArg(1));
3149     if (isa<llvm::ConstantInt>(Order)) {
3150       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
3151       AtomicRMWInst *Result = nullptr;
3152       switch (ord) {
3153       case 0:  // memory_order_relaxed
3154       default: // invalid order
3155         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
3156                                          llvm::AtomicOrdering::Monotonic);
3157         break;
3158       case 1: // memory_order_consume
3159       case 2: // memory_order_acquire
3160         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
3161                                          llvm::AtomicOrdering::Acquire);
3162         break;
3163       case 3: // memory_order_release
3164         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
3165                                          llvm::AtomicOrdering::Release);
3166         break;
3167       case 4: // memory_order_acq_rel
3168
3169         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
3170                                          llvm::AtomicOrdering::AcquireRelease);
3171         break;
3172       case 5: // memory_order_seq_cst
3173         Result = Builder.CreateAtomicRMW(
3174             llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
3175             llvm::AtomicOrdering::SequentiallyConsistent);
3176         break;
3177       }
3178       Result->setVolatile(Volatile);
3179       return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
3180     }
3181
3182     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
3183
3184     llvm::BasicBlock *BBs[5] = {
3185       createBasicBlock("monotonic", CurFn),
3186       createBasicBlock("acquire", CurFn),
3187       createBasicBlock("release", CurFn),
3188       createBasicBlock("acqrel", CurFn),
3189       createBasicBlock("seqcst", CurFn)
3190     };
3191     llvm::AtomicOrdering Orders[5] = {
3192         llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
3193         llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
3194         llvm::AtomicOrdering::SequentiallyConsistent};
3195
3196     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
3197     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
3198
3199     Builder.SetInsertPoint(ContBB);
3200     PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set");
3201
3202     for (unsigned i = 0; i < 5; ++i) {
3203       Builder.SetInsertPoint(BBs[i]);
3204       AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
3205                                                    Ptr, NewVal, Orders[i]);
3206       RMW->setVolatile(Volatile);
3207       Result->addIncoming(RMW, BBs[i]);
3208       Builder.CreateBr(ContBB);
3209     }
3210
3211     SI->addCase(Builder.getInt32(0), BBs[0]);
3212     SI->addCase(Builder.getInt32(1), BBs[1]);
3213     SI->addCase(Builder.getInt32(2), BBs[1]);
3214     SI->addCase(Builder.getInt32(3), BBs[2]);
3215     SI->addCase(Builder.getInt32(4), BBs[3]);
3216     SI->addCase(Builder.getInt32(5), BBs[4]);
3217
3218     Builder.SetInsertPoint(ContBB);
3219     return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
3220   }
3221
3222   case Builtin::BI__atomic_clear: {
3223     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
3224     bool Volatile =
3225         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
3226
3227     Address Ptr = EmitPointerWithAlignment(E->getArg(0));
3228     unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace();
3229     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
3230     Value *NewVal = Builder.getInt8(0);
3231     Value *Order = EmitScalarExpr(E->getArg(1));
3232     if (isa<llvm::ConstantInt>(Order)) {
3233       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
3234       StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
3235       switch (ord) {
3236       case 0:  // memory_order_relaxed
3237       default: // invalid order
3238         Store->setOrdering(llvm::AtomicOrdering::Monotonic);
3239         break;
3240       case 3:  // memory_order_release
3241         Store->setOrdering(llvm::AtomicOrdering::Release);
3242         break;
3243       case 5:  // memory_order_seq_cst
3244         Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
3245         break;
3246       }
3247       return RValue::get(nullptr);
3248     }
3249
3250     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
3251
3252     llvm::BasicBlock *BBs[3] = {
3253       createBasicBlock("monotonic", CurFn),
3254       createBasicBlock("release", CurFn),
3255       createBasicBlock("seqcst", CurFn)
3256     };
3257     llvm::AtomicOrdering Orders[3] = {
3258         llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
3259         llvm::AtomicOrdering::SequentiallyConsistent};
3260
3261     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
3262     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
3263
3264     for (unsigned i = 0; i < 3; ++i) {
3265       Builder.SetInsertPoint(BBs[i]);
3266       StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
3267       Store->setOrdering(Orders[i]);
3268       Builder.CreateBr(ContBB);
3269     }
3270
3271     SI->addCase(Builder.getInt32(0), BBs[0]);
3272     SI->addCase(Builder.getInt32(3), BBs[1]);
3273     SI->addCase(Builder.getInt32(5), BBs[2]);
3274
3275     Builder.SetInsertPoint(ContBB);
3276     return RValue::get(nullptr);
3277   }
3278
3279   case Builtin::BI__atomic_thread_fence:
3280   case Builtin::BI__atomic_signal_fence:
3281   case Builtin::BI__c11_atomic_thread_fence:
3282   case Builtin::BI__c11_atomic_signal_fence: {
3283     llvm::SyncScope::ID SSID;
3284     if (BuiltinID == Builtin::BI__atomic_signal_fence ||
3285         BuiltinID == Builtin::BI__c11_atomic_signal_fence)
3286       SSID = llvm::SyncScope::SingleThread;
3287     else
3288       SSID = llvm::SyncScope::System;
3289     Value *Order = EmitScalarExpr(E->getArg(0));
3290     if (isa<llvm::ConstantInt>(Order)) {
3291       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
3292       switch (ord) {
3293       case 0:  // memory_order_relaxed
3294       default: // invalid order
3295         break;
3296       case 1:  // memory_order_consume
3297       case 2:  // memory_order_acquire
3298         Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
3299         break;
3300       case 3:  // memory_order_release
3301         Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
3302         break;
3303       case 4:  // memory_order_acq_rel
3304         Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
3305         break;
3306       case 5:  // memory_order_seq_cst
3307         Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
3308         break;
3309       }
3310       return RValue::get(nullptr);
3311     }
3312
3313     llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
3314     AcquireBB = createBasicBlock("acquire", CurFn);
3315     ReleaseBB = createBasicBlock("release", CurFn);
3316     AcqRelBB = createBasicBlock("acqrel", CurFn);
3317     SeqCstBB = createBasicBlock("seqcst", CurFn);
3318     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
3319
3320     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
3321     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
3322
3323     Builder.SetInsertPoint(AcquireBB);
3324     Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
3325     Builder.CreateBr(ContBB);
3326     SI->addCase(Builder.getInt32(1), AcquireBB);
3327     SI->addCase(Builder.getInt32(2), AcquireBB);
3328
3329     Builder.SetInsertPoint(ReleaseBB);
3330     Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
3331     Builder.CreateBr(ContBB);
3332     SI->addCase(Builder.getInt32(3), ReleaseBB);
3333
3334     Builder.SetInsertPoint(AcqRelBB);
3335     Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
3336     Builder.CreateBr(ContBB);
3337     SI->addCase(Builder.getInt32(4), AcqRelBB);
3338
3339     Builder.SetInsertPoint(SeqCstBB);
3340     Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
3341     Builder.CreateBr(ContBB);
3342     SI->addCase(Builder.getInt32(5), SeqCstBB);
3343
3344     Builder.SetInsertPoint(ContBB);
3345     return RValue::get(nullptr);
3346   }
3347
3348   case Builtin::BI__builtin_signbit:
3349   case Builtin::BI__builtin_signbitf:
3350   case Builtin::BI__builtin_signbitl: {
3351     return RValue::get(
3352         Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))),
3353                            ConvertType(E->getType())));
3354   }
3355   case Builtin::BI__warn_memset_zero_len:
3356     return RValue::getIgnored();
3357   case Builtin::BI__annotation: {
3358     // Re-encode each wide string to UTF8 and make an MDString.
3359     SmallVector<Metadata *, 1> Strings;
3360     for (const Expr *Arg : E->arguments()) {
3361       const auto *Str = cast<StringLiteral>(Arg->IgnoreParenCasts());
3362       assert(Str->getCharByteWidth() == 2);
3363       StringRef WideBytes = Str->getBytes();
3364       std::string StrUtf8;
3365       if (!convertUTF16ToUTF8String(
3366               makeArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
3367         CGM.ErrorUnsupported(E, "non-UTF16 __annotation argument");
3368         continue;
3369       }
3370       Strings.push_back(llvm::MDString::get(getLLVMContext(), StrUtf8));
3371     }
3372
3373     // Build and MDTuple of MDStrings and emit the intrinsic call.
3374     llvm::Function *F =
3375         CGM.getIntrinsic(llvm::Intrinsic::codeview_annotation, {});
3376     MDTuple *StrTuple = MDTuple::get(getLLVMContext(), Strings);
3377     Builder.CreateCall(F, MetadataAsValue::get(getLLVMContext(), StrTuple));
3378     return RValue::getIgnored();
3379   }
3380   case Builtin::BI__builtin_annotation: {
3381     llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
3382     llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
3383                                       AnnVal->getType());
3384
3385     // Get the annotation string, go through casts. Sema requires this to be a
3386     // non-wide string literal, potentially casted, so the cast<> is safe.
3387     const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts();
3388     StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
3389     return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc()));
3390   }
3391   case Builtin::BI__builtin_addcb:
3392   case Builtin::BI__builtin_addcs:
3393   case Builtin::BI__builtin_addc:
3394   case Builtin::BI__builtin_addcl:
3395   case Builtin::BI__builtin_addcll:
3396   case Builtin::BI__builtin_subcb:
3397   case Builtin::BI__builtin_subcs:
3398   case Builtin::BI__builtin_subc:
3399   case Builtin::BI__builtin_subcl:
3400   case Builtin::BI__builtin_subcll: {
3401
3402     // We translate all of these builtins from expressions of the form:
3403     //   int x = ..., y = ..., carryin = ..., carryout, result;
3404     //   result = __builtin_addc(x, y, carryin, &carryout);
3405     //
3406     // to LLVM IR of the form:
3407     //
3408     //   %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
3409     //   %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
3410     //   %carry1 = extractvalue {i32, i1} %tmp1, 1
3411     //   %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
3412     //                                                       i32 %carryin)
3413     //   %result = extractvalue {i32, i1} %tmp2, 0
3414     //   %carry2 = extractvalue {i32, i1} %tmp2, 1
3415     //   %tmp3 = or i1 %carry1, %carry2
3416     //   %tmp4 = zext i1 %tmp3 to i32
3417     //   store i32 %tmp4, i32* %carryout
3418
3419     // Scalarize our inputs.
3420     llvm::Value *X = EmitScalarExpr(E->getArg(0));
3421     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
3422     llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
3423     Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3));
3424
3425     // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
3426     llvm::Intrinsic::ID IntrinsicId;
3427     switch (BuiltinID) {
3428     default: llvm_unreachable("Unknown multiprecision builtin id.");
3429     case Builtin::BI__builtin_addcb:
3430     case Builtin::BI__builtin_addcs:
3431     case Builtin::BI__builtin_addc:
3432     case Builtin::BI__builtin_addcl:
3433     case Builtin::BI__builtin_addcll:
3434       IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
3435       break;
3436     case Builtin::BI__builtin_subcb:
3437     case Builtin::BI__builtin_subcs:
3438     case Builtin::BI__builtin_subc:
3439     case Builtin::BI__builtin_subcl:
3440     case Builtin::BI__builtin_subcll:
3441       IntrinsicId = llvm::Intrinsic::usub_with_overflow;
3442       break;
3443     }
3444
3445     // Construct our resulting LLVM IR expression.
3446     llvm::Value *Carry1;
3447     llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId,
3448                                               X, Y, Carry1);
3449     llvm::Value *Carry2;
3450     llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId,
3451                                               Sum1, Carryin, Carry2);
3452     llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
3453                                                X->getType());
3454     Builder.CreateStore(CarryOut, CarryOutPtr);
3455     return RValue::get(Sum2);
3456   }
3457
3458   case Builtin::BI__builtin_add_overflow:
3459   case Builtin::BI__builtin_sub_overflow:
3460   case Builtin::BI__builtin_mul_overflow: {
3461     const clang::Expr *LeftArg = E->getArg(0);
3462     const clang::Expr *RightArg = E->getArg(1);
3463     const clang::Expr *ResultArg = E->getArg(2);
3464
3465     clang::QualType ResultQTy =
3466         ResultArg->getType()->castAs<PointerType>()->getPointeeType();
3467
3468     WidthAndSignedness LeftInfo =
3469         getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType());
3470     WidthAndSignedness RightInfo =
3471         getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType());
3472     WidthAndSignedness ResultInfo =
3473         getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy);
3474
3475     // Handle mixed-sign multiplication as a special case, because adding
3476     // runtime or backend support for our generic irgen would be too expensive.
3477     if (isSpecialMixedSignMultiply(BuiltinID, LeftInfo, RightInfo, ResultInfo))
3478       return EmitCheckedMixedSignMultiply(*this, LeftArg, LeftInfo, RightArg,
3479                                           RightInfo, ResultArg, ResultQTy,
3480                                           ResultInfo);
3481
3482     WidthAndSignedness EncompassingInfo =
3483         EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo});
3484
3485     llvm::Type *EncompassingLLVMTy =
3486         llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
3487
3488     llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
3489
3490     llvm::Intrinsic::ID IntrinsicId;
3491     switch (BuiltinID) {
3492     default:
3493       llvm_unreachable("Unknown overflow builtin id.");
3494     case Builtin::BI__builtin_add_overflow:
3495       IntrinsicId = EncompassingInfo.Signed
3496                         ? llvm::Intrinsic::sadd_with_overflow
3497                         : llvm::Intrinsic::uadd_with_overflow;
3498       break;
3499     case Builtin::BI__builtin_sub_overflow:
3500       IntrinsicId = EncompassingInfo.Signed
3501                         ? llvm::Intrinsic::ssub_with_overflow
3502                         : llvm::Intrinsic::usub_with_overflow;
3503       break;
3504     case Builtin::BI__builtin_mul_overflow:
3505       IntrinsicId = EncompassingInfo.Signed
3506                         ? llvm::Intrinsic::smul_with_overflow
3507                         : llvm::Intrinsic::umul_with_overflow;
3508       break;
3509     }
3510
3511     llvm::Value *Left = EmitScalarExpr(LeftArg);
3512     llvm::Value *Right = EmitScalarExpr(RightArg);
3513     Address ResultPtr = EmitPointerWithAlignment(ResultArg);
3514
3515     // Extend each operand to the encompassing type.
3516     Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
3517     Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
3518
3519     // Perform the operation on the extended values.
3520     llvm::Value *Overflow, *Result;
3521     Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow);
3522
3523     if (EncompassingInfo.Width > ResultInfo.Width) {
3524       // The encompassing type is wider than the result type, so we need to
3525       // truncate it.
3526       llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy);
3527
3528       // To see if the truncation caused an overflow, we will extend
3529       // the result and then compare it to the original result.
3530       llvm::Value *ResultTruncExt = Builder.CreateIntCast(
3531           ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
3532       llvm::Value *TruncationOverflow =
3533           Builder.CreateICmpNE(Result, ResultTruncExt);
3534
3535       Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
3536       Result = ResultTrunc;
3537     }
3538
3539     // Finally, store the result using the pointer.
3540     bool isVolatile =
3541       ResultArg->getType()->getPointeeType().isVolatileQualified();
3542     Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
3543
3544     return RValue::get(Overflow);
3545   }
3546
3547   case Builtin::BI__builtin_uadd_overflow:
3548   case Builtin::BI__builtin_uaddl_overflow:
3549   case Builtin::BI__builtin_uaddll_overflow:
3550   case Builtin::BI__builtin_usub_overflow:
3551   case Builtin::BI__builtin_usubl_overflow:
3552   case Builtin::BI__builtin_usubll_overflow:
3553   case Builtin::BI__builtin_umul_overflow:
3554   case Builtin::BI__builtin_umull_overflow:
3555   case Builtin::BI__builtin_umulll_overflow:
3556   case Builtin::BI__builtin_sadd_overflow:
3557   case Builtin::BI__builtin_saddl_overflow:
3558   case Builtin::BI__builtin_saddll_overflow:
3559   case Builtin::BI__builtin_ssub_overflow:
3560   case Builtin::BI__builtin_ssubl_overflow:
3561   case Builtin::BI__builtin_ssubll_overflow:
3562   case Builtin::BI__builtin_smul_overflow:
3563   case Builtin::BI__builtin_smull_overflow:
3564   case Builtin::BI__builtin_smulll_overflow: {
3565
3566     // We translate all of these builtins directly to the relevant llvm IR node.
3567
3568     // Scalarize our inputs.
3569     llvm::Value *X = EmitScalarExpr(E->getArg(0));
3570     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
3571     Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2));
3572
3573     // Decide which of the overflow intrinsics we are lowering to:
3574     llvm::Intrinsic::ID IntrinsicId;
3575     switch (BuiltinID) {
3576     default: llvm_unreachable("Unknown overflow builtin id.");
3577     case Builtin::BI__builtin_uadd_overflow:
3578     case Builtin::BI__builtin_uaddl_overflow:
3579     case Builtin::BI__builtin_uaddll_overflow:
3580       IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
3581       break;
3582     case Builtin::BI__builtin_usub_overflow:
3583     case Builtin::BI__builtin_usubl_overflow:
3584     case Builtin::BI__builtin_usubll_overflow:
3585       IntrinsicId = llvm::Intrinsic::usub_with_overflow;
3586       break;
3587     case Builtin::BI__builtin_umul_overflow:
3588     case Builtin::BI__builtin_umull_overflow:
3589     case Builtin::BI__builtin_umulll_overflow:
3590       IntrinsicId = llvm::Intrinsic::umul_with_overflow;
3591       break;
3592     case Builtin::BI__builtin_sadd_overflow:
3593     case Builtin::BI__builtin_saddl_overflow:
3594     case Builtin::BI__builtin_saddll_overflow:
3595       IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
3596       break;
3597     case Builtin::BI__builtin_ssub_overflow:
3598     case Builtin::BI__builtin_ssubl_overflow:
3599     case Builtin::BI__builtin_ssubll_overflow:
3600       IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
3601       break;
3602     case Builtin::BI__builtin_smul_overflow:
3603     case Builtin::BI__builtin_smull_overflow:
3604     case Builtin::BI__builtin_smulll_overflow:
3605       IntrinsicId = llvm::Intrinsic::smul_with_overflow;
3606       break;
3607     }
3608
3609
3610     llvm::Value *Carry;
3611     llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry);
3612     Builder.CreateStore(Sum, SumOutPtr);
3613
3614     return RValue::get(Carry);
3615   }
3616   case Builtin::BI__builtin_addressof:
3617     return RValue::get(EmitLValue(E->getArg(0)).getPointer(*this));
3618   case Builtin::BI__builtin_operator_new:
3619     return EmitBuiltinNewDeleteCall(
3620         E->getCallee()->getType()->castAs<FunctionProtoType>(), E, false);
3621   case Builtin::BI__builtin_operator_delete:
3622     return EmitBuiltinNewDeleteCall(
3623         E->getCallee()->getType()->castAs<FunctionProtoType>(), E, true);
3624
3625   case Builtin::BI__builtin_is_aligned:
3626     return EmitBuiltinIsAligned(E);
3627   case Builtin::BI__builtin_align_up:
3628     return EmitBuiltinAlignTo(E, true);
3629   case Builtin::BI__builtin_align_down:
3630     return EmitBuiltinAlignTo(E, false);
3631
3632   case Builtin::BI__noop:
3633     // __noop always evaluates to an integer literal zero.
3634     return RValue::get(ConstantInt::get(IntTy, 0));
3635   case Builtin::BI__builtin_call_with_static_chain: {
3636     const CallExpr *Call = cast<CallExpr>(E->getArg(0));
3637     const Expr *Chain = E->getArg(1);
3638     return EmitCall(Call->getCallee()->getType(),
3639                     EmitCallee(Call->getCallee()), Call, ReturnValue,
3640                     EmitScalarExpr(Chain));
3641   }
3642   case Builtin::BI_InterlockedExchange8:
3643   case Builtin::BI_InterlockedExchange16:
3644   case Builtin::BI_InterlockedExchange:
3645   case Builtin::BI_InterlockedExchangePointer:
3646     return RValue::get(
3647         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
3648   case Builtin::BI_InterlockedCompareExchangePointer:
3649   case Builtin::BI_InterlockedCompareExchangePointer_nf: {
3650     llvm::Type *RTy;
3651     llvm::IntegerType *IntType =
3652       IntegerType::get(getLLVMContext(),
3653                        getContext().getTypeSize(E->getType()));
3654     llvm::Type *IntPtrType = IntType->getPointerTo();
3655
3656     llvm::Value *Destination =
3657       Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType);
3658
3659     llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
3660     RTy = Exchange->getType();
3661     Exchange = Builder.CreatePtrToInt(Exchange, IntType);
3662
3663     llvm::Value *Comparand =
3664       Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
3665
3666     auto Ordering =
3667       BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
3668       AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
3669
3670     auto Result = Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
3671                                               Ordering, Ordering);
3672     Result->setVolatile(true);
3673
3674     return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
3675                                                                          0),
3676                                               RTy));
3677   }
3678   case Builtin::BI_InterlockedCompareExchange8:
3679   case Builtin::BI_InterlockedCompareExchange16:
3680   case Builtin::BI_InterlockedCompareExchange:
3681   case Builtin::BI_InterlockedCompareExchange64:
3682     return RValue::get(EmitAtomicCmpXchgForMSIntrin(*this, E));
3683   case Builtin::BI_InterlockedIncrement16:
3684   case Builtin::BI_InterlockedIncrement:
3685     return RValue::get(
3686         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E));
3687   case Builtin::BI_InterlockedDecrement16:
3688   case Builtin::BI_InterlockedDecrement:
3689     return RValue::get(
3690         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E));
3691   case Builtin::BI_InterlockedAnd8:
3692   case Builtin::BI_InterlockedAnd16:
3693   case Builtin::BI_InterlockedAnd:
3694     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E));
3695   case Builtin::BI_InterlockedExchangeAdd8:
3696   case Builtin::BI_InterlockedExchangeAdd16:
3697   case Builtin::BI_InterlockedExchangeAdd:
3698     return RValue::get(
3699         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E));
3700   case Builtin::BI_InterlockedExchangeSub8:
3701   case Builtin::BI_InterlockedExchangeSub16:
3702   case Builtin::BI_InterlockedExchangeSub:
3703     return RValue::get(
3704         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E));
3705   case Builtin::BI_InterlockedOr8:
3706   case Builtin::BI_InterlockedOr16:
3707   case Builtin::BI_InterlockedOr:
3708     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E));
3709   case Builtin::BI_InterlockedXor8:
3710   case Builtin::BI_InterlockedXor16:
3711   case Builtin::BI_InterlockedXor:
3712     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E));
3713
3714   case Builtin::BI_bittest64:
3715   case Builtin::BI_bittest:
3716   case Builtin::BI_bittestandcomplement64:
3717   case Builtin::BI_bittestandcomplement:
3718   case Builtin::BI_bittestandreset64:
3719   case Builtin::BI_bittestandreset:
3720   case Builtin::BI_bittestandset64:
3721   case Builtin::BI_bittestandset:
3722   case Builtin::BI_interlockedbittestandreset:
3723   case Builtin::BI_interlockedbittestandreset64:
3724   case Builtin::BI_interlockedbittestandset64:
3725   case Builtin::BI_interlockedbittestandset:
3726   case Builtin::BI_interlockedbittestandset_acq:
3727   case Builtin::BI_interlockedbittestandset_rel:
3728   case Builtin::BI_interlockedbittestandset_nf:
3729   case Builtin::BI_interlockedbittestandreset_acq:
3730   case Builtin::BI_interlockedbittestandreset_rel:
3731   case Builtin::BI_interlockedbittestandreset_nf:
3732     return RValue::get(EmitBitTestIntrinsic(*this, BuiltinID, E));
3733
3734     // These builtins exist to emit regular volatile loads and stores not
3735     // affected by the -fms-volatile setting.
3736   case Builtin::BI__iso_volatile_load8:
3737   case Builtin::BI__iso_volatile_load16:
3738   case Builtin::BI__iso_volatile_load32:
3739   case Builtin::BI__iso_volatile_load64:
3740     return RValue::get(EmitISOVolatileLoad(*this, E));
3741   case Builtin::BI__iso_volatile_store8:
3742   case Builtin::BI__iso_volatile_store16:
3743   case Builtin::BI__iso_volatile_store32:
3744   case Builtin::BI__iso_volatile_store64:
3745     return RValue::get(EmitISOVolatileStore(*this, E));
3746
3747   case Builtin::BI__exception_code:
3748   case Builtin::BI_exception_code:
3749     return RValue::get(EmitSEHExceptionCode());
3750   case Builtin::BI__exception_info:
3751   case Builtin::BI_exception_info:
3752     return RValue::get(EmitSEHExceptionInfo());
3753   case Builtin::BI__abnormal_termination:
3754   case Builtin::BI_abnormal_termination:
3755     return RValue::get(EmitSEHAbnormalTermination());
3756   case Builtin::BI_setjmpex:
3757     if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
3758         E->getArg(0)->getType()->isPointerType())
3759       return EmitMSVCRTSetJmp(*this, MSVCSetJmpKind::_setjmpex, E);
3760     break;
3761   case Builtin::BI_setjmp:
3762     if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
3763         E->getArg(0)->getType()->isPointerType()) {
3764       if (getTarget().getTriple().getArch() == llvm::Triple::x86)
3765         return EmitMSVCRTSetJmp(*this, MSVCSetJmpKind::_setjmp3, E);
3766       else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
3767         return EmitMSVCRTSetJmp(*this, MSVCSetJmpKind::_setjmpex, E);
3768       return EmitMSVCRTSetJmp(*this, MSVCSetJmpKind::_setjmp, E);
3769     }
3770     break;
3771
3772   case Builtin::BI__GetExceptionInfo: {
3773     if (llvm::GlobalVariable *GV =
3774             CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))
3775       return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy));
3776     break;
3777   }
3778
3779   case Builtin::BI__fastfail:
3780     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::__fastfail, E));
3781
3782   case Builtin::BI__builtin_coro_size: {
3783     auto & Context = getContext();
3784     auto SizeTy = Context.getSizeType();
3785     auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
3786     Function *F = CGM.getIntrinsic(Intrinsic::coro_size, T);
3787     return RValue::get(Builder.CreateCall(F));
3788   }
3789
3790   case Builtin::BI__builtin_coro_id:
3791     return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
3792   case Builtin::BI__builtin_coro_promise:
3793     return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
3794   case Builtin::BI__builtin_coro_resume:
3795     return EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
3796   case Builtin::BI__builtin_coro_frame:
3797     return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
3798   case Builtin::BI__builtin_coro_noop:
3799     return EmitCoroutineIntrinsic(E, Intrinsic::coro_noop);
3800   case Builtin::BI__builtin_coro_free:
3801     return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
3802   case Builtin::BI__builtin_coro_destroy:
3803     return EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
3804   case Builtin::BI__builtin_coro_done:
3805     return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
3806   case Builtin::BI__builtin_coro_alloc:
3807     return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
3808   case Builtin::BI__builtin_coro_begin:
3809     return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
3810   case Builtin::BI__builtin_coro_end:
3811     return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
3812   case Builtin::BI__builtin_coro_suspend:
3813     return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
3814   case Builtin::BI__builtin_coro_param:
3815     return EmitCoroutineIntrinsic(E, Intrinsic::coro_param);
3816
3817   // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
3818   case Builtin::BIread_pipe:
3819   case Builtin::BIwrite_pipe: {
3820     Value *Arg0 = EmitScalarExpr(E->getArg(0)),
3821           *Arg1 = EmitScalarExpr(E->getArg(1));
3822     CGOpenCLRuntime OpenCLRT(CGM);
3823     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
3824     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
3825
3826     // Type of the generic packet parameter.
3827     unsigned GenericAS =
3828         getContext().getTargetAddressSpace(LangAS::opencl_generic);
3829     llvm::Type *I8PTy = llvm::PointerType::get(
3830         llvm::Type::getInt8Ty(getLLVMContext()), GenericAS);
3831
3832     // Testing which overloaded version we should generate the call for.
3833     if (2U == E->getNumArgs()) {
3834       const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
3835                                                              : "__write_pipe_2";
3836       // Creating a generic function type to be able to call with any builtin or
3837       // user defined type.
3838       llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
3839       llvm::FunctionType *FTy = llvm::FunctionType::get(
3840           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3841       Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
3842       return RValue::get(
3843           Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3844                              {Arg0, BCast, PacketSize, PacketAlign}));
3845     } else {
3846       assert(4 == E->getNumArgs() &&
3847              "Illegal number of parameters to pipe function");
3848       const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
3849                                                              : "__write_pipe_4";
3850
3851       llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
3852                               Int32Ty, Int32Ty};
3853       Value *Arg2 = EmitScalarExpr(E->getArg(2)),
3854             *Arg3 = EmitScalarExpr(E->getArg(3));
3855       llvm::FunctionType *FTy = llvm::FunctionType::get(
3856           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3857       Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
3858       // We know the third argument is an integer type, but we may need to cast
3859       // it to i32.
3860       if (Arg2->getType() != Int32Ty)
3861         Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
3862       return RValue::get(Builder.CreateCall(
3863           CGM.CreateRuntimeFunction(FTy, Name),
3864           {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
3865     }
3866   }
3867   // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
3868   // functions
3869   case Builtin::BIreserve_read_pipe:
3870   case Builtin::BIreserve_write_pipe:
3871   case Builtin::BIwork_group_reserve_read_pipe:
3872   case Builtin::BIwork_group_reserve_write_pipe:
3873   case Builtin::BIsub_group_reserve_read_pipe:
3874   case Builtin::BIsub_group_reserve_write_pipe: {
3875     // Composing the mangled name for the function.
3876     const char *Name;
3877     if (BuiltinID == Builtin::BIreserve_read_pipe)
3878       Name = "__reserve_read_pipe";
3879     else if (BuiltinID == Builtin::BIreserve_write_pipe)
3880       Name = "__reserve_write_pipe";
3881     else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
3882       Name = "__work_group_reserve_read_pipe";
3883     else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
3884       Name = "__work_group_reserve_write_pipe";
3885     else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
3886       Name = "__sub_group_reserve_read_pipe";
3887     else
3888       Name = "__sub_group_reserve_write_pipe";
3889
3890     Value *Arg0 = EmitScalarExpr(E->getArg(0)),
3891           *Arg1 = EmitScalarExpr(E->getArg(1));
3892     llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
3893     CGOpenCLRuntime OpenCLRT(CGM);
3894     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
3895     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
3896
3897     // Building the generic function prototype.
3898     llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
3899     llvm::FunctionType *FTy = llvm::FunctionType::get(
3900         ReservedIDTy, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3901     // We know the second argument is an integer type, but we may need to cast
3902     // it to i32.
3903     if (Arg1->getType() != Int32Ty)
3904       Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
3905     return RValue::get(
3906         Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3907                            {Arg0, Arg1, PacketSize, PacketAlign}));
3908   }
3909   // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
3910   // functions
3911   case Builtin::BIcommit_read_pipe:
3912   case Builtin::BIcommit_write_pipe:
3913   case Builtin::BIwork_group_commit_read_pipe:
3914   case Builtin::BIwork_group_commit_write_pipe:
3915   case Builtin::BIsub_group_commit_read_pipe:
3916   case Builtin::BIsub_group_commit_write_pipe: {
3917     const char *Name;
3918     if (BuiltinID == Builtin::BIcommit_read_pipe)
3919       Name = "__commit_read_pipe";
3920     else if (BuiltinID == Builtin::BIcommit_write_pipe)
3921       Name = "__commit_write_pipe";
3922     else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
3923       Name = "__work_group_commit_read_pipe";
3924     else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
3925       Name = "__work_group_commit_write_pipe";
3926     else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
3927       Name = "__sub_group_commit_read_pipe";
3928     else
3929       Name = "__sub_group_commit_write_pipe";
3930
3931     Value *Arg0 = EmitScalarExpr(E->getArg(0)),
3932           *Arg1 = EmitScalarExpr(E->getArg(1));
3933     CGOpenCLRuntime OpenCLRT(CGM);
3934     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
3935     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
3936
3937     // Building the generic function prototype.
3938     llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
3939     llvm::FunctionType *FTy =
3940         llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
3941                                 llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3942
3943     return RValue::get(
3944         Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3945                            {Arg0, Arg1, PacketSize, PacketAlign}));
3946   }
3947   // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
3948   case Builtin::BIget_pipe_num_packets:
3949   case Builtin::BIget_pipe_max_packets: {
3950     const char *BaseName;
3951     const auto *PipeTy = E->getArg(0)->getType()->castAs<PipeType>();
3952     if (BuiltinID == Builtin::BIget_pipe_num_packets)
3953       BaseName = "__get_pipe_num_packets";
3954     else
3955       BaseName = "__get_pipe_max_packets";
3956     std::string Name = std::string(BaseName) +
3957                        std::string(PipeTy->isReadOnly() ? "_ro" : "_wo");
3958
3959     // Building the generic function prototype.
3960     Value *Arg0 = EmitScalarExpr(E->getArg(0));
3961     CGOpenCLRuntime OpenCLRT(CGM);
3962     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
3963     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
3964     llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
3965     llvm::FunctionType *FTy = llvm::FunctionType::get(
3966         Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3967
3968     return RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3969                                           {Arg0, PacketSize, PacketAlign}));
3970   }
3971
3972   // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
3973   case Builtin::BIto_global:
3974   case Builtin::BIto_local:
3975   case Builtin::BIto_private: {
3976     auto Arg0 = EmitScalarExpr(E->getArg(0));
3977     auto NewArgT = llvm::PointerType::get(Int8Ty,
3978       CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic));
3979     auto NewRetT = llvm::PointerType::get(Int8Ty,
3980       CGM.getContext().getTargetAddressSpace(
3981         E->getType()->getPointeeType().getAddressSpace()));
3982     auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT}, false);
3983     llvm::Value *NewArg;
3984     if (Arg0->getType()->getPointerAddressSpace() !=
3985         NewArgT->getPointerAddressSpace())
3986       NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
3987     else
3988       NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
3989     auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
3990     auto NewCall =
3991         Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
3992     return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
3993       ConvertType(E->getType())));
3994   }
3995
3996   // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
3997   // It contains four different overload formats specified in Table 6.13.17.1.
3998   case Builtin::BIenqueue_kernel: {
3999     StringRef Name; // Generated function call name
4000     unsigned NumArgs = E->getNumArgs();
4001
4002     llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
4003     llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
4004         getContext().getTargetAddressSpace(LangAS::opencl_generic));
4005
4006     llvm::Value *Queue = EmitScalarExpr(E->getArg(0));
4007     llvm::Value *Flags = EmitScalarExpr(E->getArg(1));
4008     LValue NDRangeL = EmitAggExprToLValue(E->getArg(2));
4009     llvm::Value *Range = NDRangeL.getAddress(*this).getPointer();
4010     llvm::Type *RangeTy = NDRangeL.getAddress(*this).getType();
4011
4012     if (NumArgs == 4) {
4013       // The most basic form of the call with parameters:
4014       // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
4015       Name = "__enqueue_kernel_basic";
4016       llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, GenericVoidPtrTy,
4017                               GenericVoidPtrTy};
4018       llvm::FunctionType *FTy = llvm::FunctionType::get(
4019           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
4020
4021       auto Info =
4022           CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(3));
4023       llvm::Value *Kernel =
4024           Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
4025       llvm::Value *Block =
4026           Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
4027
4028       AttrBuilder B;
4029       B.addByValAttr(NDRangeL.getAddress(*this).getElementType());
4030       llvm::AttributeList ByValAttrSet =
4031           llvm::AttributeList::get(CGM.getModule().getContext(), 3U, B);
4032
4033       auto RTCall =
4034           Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name, ByValAttrSet),
4035                              {Queue, Flags, Range, Kernel, Block});
4036       RTCall->setAttributes(ByValAttrSet);
4037       return RValue::get(RTCall);
4038     }
4039     assert(NumArgs >= 5 && "Invalid enqueue_kernel signature");
4040
4041     // Create a temporary array to hold the sizes of local pointer arguments
4042     // for the block. \p First is the position of the first size argument.
4043     auto CreateArrayForSizeVar = [=](unsigned First)
4044         -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
4045       llvm::APInt ArraySize(32, NumArgs - First);
4046       QualType SizeArrayTy = getContext().getConstantArrayType(
4047           getContext().getSizeType(), ArraySize, nullptr, ArrayType::Normal,
4048           /*IndexTypeQuals=*/0);
4049       auto Tmp = CreateMemTemp(SizeArrayTy, "block_sizes");
4050       llvm::Value *TmpPtr = Tmp.getPointer();
4051       llvm::Value *TmpSize = EmitLifetimeStart(
4052           CGM.getDataLayout().getTypeAllocSize(Tmp.getElementType()), TmpPtr);
4053       llvm::Value *ElemPtr;
4054       // Each of the following arguments specifies the size of the corresponding
4055       // argument passed to the enqueued block.
4056       auto *Zero = llvm::ConstantInt::get(IntTy, 0);
4057       for (unsigned I = First; I < NumArgs; ++I) {
4058         auto *Index = llvm::ConstantInt::get(IntTy, I - First);
4059         auto *GEP = Builder.CreateGEP(TmpPtr, {Zero, Index});
4060         if (I == First)
4061           ElemPtr = GEP;
4062         auto *V =
4063             Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(I)), SizeTy);
4064         Builder.CreateAlignedStore(
4065             V, GEP, CGM.getDataLayout().getPrefTypeAlign(SizeTy));
4066       }
4067       return std::tie(ElemPtr, TmpSize, TmpPtr);
4068     };
4069
4070     // Could have events and/or varargs.
4071     if (E->getArg(3)->getType()->isBlockPointerType()) {
4072       // No events passed, but has variadic arguments.
4073       Name = "__enqueue_kernel_varargs";
4074       auto Info =
4075           CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(3));
4076       llvm::Value *Kernel =
4077           Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
4078       auto *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
4079       llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
4080       std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
4081
4082       // Create a vector of the arguments, as well as a constant value to
4083       // express to the runtime the number of variadic arguments.
4084       llvm::Value *const Args[] = {Queue,  Flags,
4085                                    Range,  Kernel,
4086                                    Block,  ConstantInt::get(IntTy, NumArgs - 4),
4087                                    ElemPtr};
4088       llvm::Type *const ArgTys[] = {
4089           QueueTy,          IntTy, RangeTy,           GenericVoidPtrTy,
4090           GenericVoidPtrTy, IntTy, ElemPtr->getType()};
4091
4092       llvm::FunctionType *FTy = llvm::FunctionType::get(Int32Ty, ArgTys, false);
4093       auto Call = RValue::get(
4094           Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), Args));
4095       if (TmpSize)
4096         EmitLifetimeEnd(TmpSize, TmpPtr);
4097       return Call;
4098     }
4099     // Any calls now have event arguments passed.
4100     if (NumArgs >= 7) {
4101       llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy);
4102       llvm::PointerType *EventPtrTy = EventTy->getPointerTo(
4103           CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic));
4104
4105       llvm::Value *NumEvents =
4106           Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(3)), Int32Ty);
4107
4108       // Since SemaOpenCLBuiltinEnqueueKernel allows fifth and sixth arguments
4109       // to be a null pointer constant (including `0` literal), we can take it
4110       // into account and emit null pointer directly.
4111       llvm::Value *EventWaitList = nullptr;
4112       if (E->getArg(4)->isNullPointerConstant(
4113               getContext(), Expr::NPC_ValueDependentIsNotNull)) {
4114         EventWaitList = llvm::ConstantPointerNull::get(EventPtrTy);
4115       } else {
4116         EventWaitList = E->getArg(4)->getType()->isArrayType()
4117                         ? EmitArrayToPointerDecay(E->getArg(4)).getPointer()
4118                         : EmitScalarExpr(E->getArg(4));
4119         // Convert to generic address space.
4120         EventWaitList = Builder.CreatePointerCast(EventWaitList, EventPtrTy);
4121       }
4122       llvm::Value *EventRet = nullptr;
4123       if (E->getArg(5)->isNullPointerConstant(
4124               getContext(), Expr::NPC_ValueDependentIsNotNull)) {
4125         EventRet = llvm::ConstantPointerNull::get(EventPtrTy);
4126       } else {
4127         EventRet =
4128             Builder.CreatePointerCast(EmitScalarExpr(E->getArg(5)), EventPtrTy);
4129       }
4130
4131       auto Info =
4132           CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(6));
4133       llvm::Value *Kernel =
4134           Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
4135       llvm::Value *Block =
4136           Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
4137
4138       std::vector<llvm::Type *> ArgTys = {
4139           QueueTy,    Int32Ty,    RangeTy,          Int32Ty,
4140           EventPtrTy, EventPtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
4141
4142       std::vector<llvm::Value *> Args = {Queue,     Flags,         Range,
4143                                          NumEvents, EventWaitList, EventRet,
4144                                          Kernel,    Block};
4145
4146       if (NumArgs == 7) {
4147         // Has events but no variadics.
4148         Name = "__enqueue_kernel_basic_events";
4149         llvm::FunctionType *FTy = llvm::FunctionType::get(
4150             Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
4151         return RValue::get(
4152             Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
4153                                llvm::ArrayRef<llvm::Value *>(Args)));
4154       }
4155       // Has event info and variadics
4156       // Pass the number of variadics to the runtime function too.
4157       Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
4158       ArgTys.push_back(Int32Ty);
4159       Name = "__enqueue_kernel_events_varargs";
4160
4161       llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
4162       std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
4163       Args.push_back(ElemPtr);
4164       ArgTys.push_back(ElemPtr->getType());
4165
4166       llvm::FunctionType *FTy = llvm::FunctionType::get(
4167           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
4168       auto Call =
4169           RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
4170                                          llvm::ArrayRef<llvm::Value *>(Args)));
4171       if (TmpSize)
4172         EmitLifetimeEnd(TmpSize, TmpPtr);
4173       return Call;
4174     }
4175     LLVM_FALLTHROUGH;
4176   }
4177   // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
4178   // parameter.
4179   case Builtin::BIget_kernel_work_group_size: {
4180     llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
4181         getContext().getTargetAddressSpace(LangAS::opencl_generic));
4182     auto Info =
4183         CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
4184     Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
4185     Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
4186     return RValue::get(Builder.CreateCall(
4187         CGM.CreateRuntimeFunction(
4188             llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
4189                                     false),
4190             "__get_kernel_work_group_size_impl"),
4191         {Kernel, Arg}));
4192   }
4193   case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
4194     llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
4195         getContext().getTargetAddressSpace(LangAS::opencl_generic));
4196     auto Info =
4197         CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
4198     Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
4199     Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
4200     return RValue::get(Builder.CreateCall(
4201         CGM.CreateRuntimeFunction(
4202             llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
4203                                     false),
4204             "__get_kernel_preferred_work_group_size_multiple_impl"),
4205         {Kernel, Arg}));
4206   }
4207   case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
4208   case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
4209     llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
4210         getContext().getTargetAddressSpace(LangAS::opencl_generic));
4211     LValue NDRangeL = EmitAggExprToLValue(E->getArg(0));
4212     llvm::Value *NDRange = NDRangeL.getAddress(*this).getPointer();
4213     auto Info =
4214         CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1));
4215     Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
4216     Value *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
4217     const char *Name =
4218         BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
4219             ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
4220             : "__get_kernel_sub_group_count_for_ndrange_impl";
4221     return RValue::get(Builder.CreateCall(
4222         CGM.CreateRuntimeFunction(
4223             llvm::FunctionType::get(
4224                 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
4225                 false),
4226             Name),
4227         {NDRange, Kernel, Block}));
4228   }
4229
4230   case Builtin::BI__builtin_store_half:
4231   case Builtin::BI__builtin_store_halff: {
4232     Value *Val = EmitScalarExpr(E->getArg(0));
4233     Address Address = EmitPointerWithAlignment(E->getArg(1));
4234     Value *HalfVal = Builder.CreateFPTrunc(Val, Builder.getHalfTy());
4235     return RValue::get(Builder.CreateStore(HalfVal, Address));
4236   }
4237   case Builtin::BI__builtin_load_half: {
4238     Address Address = EmitPointerWithAlignment(E->getArg(0));
4239     Value *HalfVal = Builder.CreateLoad(Address);
4240     return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getDoubleTy()));
4241   }
4242   case Builtin::BI__builtin_load_halff: {
4243     Address Address = EmitPointerWithAlignment(E->getArg(0));
4244     Value *HalfVal = Builder.CreateLoad(Address);
4245     return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy()));
4246   }
4247   case Builtin::BIprintf:
4248     if (getTarget().getTriple().isNVPTX())
4249       return EmitNVPTXDevicePrintfCallExpr(E, ReturnValue);
4250     if (getTarget().getTriple().getArch() == Triple::amdgcn &&
4251         getLangOpts().HIP)
4252       return EmitAMDGPUDevicePrintfCallExpr(E, ReturnValue);
4253     break;
4254   case Builtin::BI__builtin_canonicalize:
4255   case Builtin::BI__builtin_canonicalizef:
4256   case Builtin::BI__builtin_canonicalizef16:
4257   case Builtin::BI__builtin_canonicalizel:
4258     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::canonicalize));
4259
4260   case Builtin::BI__builtin_thread_pointer: {
4261     if (!getContext().getTargetInfo().isTLSSupported())
4262       CGM.ErrorUnsupported(E, "__builtin_thread_pointer");
4263     // Fall through - it's already mapped to the intrinsic by GCCBuiltin.
4264     break;
4265   }
4266   case Builtin::BI__builtin_os_log_format:
4267     return emitBuiltinOSLogFormat(*E);
4268
4269   case Builtin::BI__xray_customevent: {
4270     if (!ShouldXRayInstrumentFunction())
4271       return RValue::getIgnored();
4272
4273     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
4274             XRayInstrKind::Custom))
4275       return RValue::getIgnored();
4276
4277     if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
4278       if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
4279         return RValue::getIgnored();
4280
4281     Function *F = CGM.getIntrinsic(Intrinsic::xray_customevent);
4282     auto FTy = F->getFunctionType();
4283     auto Arg0 = E->getArg(0);
4284     auto Arg0Val = EmitScalarExpr(Arg0);
4285     auto Arg0Ty = Arg0->getType();
4286     auto PTy0 = FTy->getParamType(0);
4287     if (PTy0 != Arg0Val->getType()) {
4288       if (Arg0Ty->isArrayType())
4289         Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer();
4290       else
4291         Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
4292     }
4293     auto Arg1 = EmitScalarExpr(E->getArg(1));
4294     auto PTy1 = FTy->getParamType(1);
4295     if (PTy1 != Arg1->getType())
4296       Arg1 = Builder.CreateTruncOrBitCast(Arg1, PTy1);
4297     return RValue::get(Builder.CreateCall(F, {Arg0Val, Arg1}));
4298   }
4299
4300   case Builtin::BI__xray_typedevent: {
4301     // TODO: There should be a way to always emit events even if the current
4302     // function is not instrumented. Losing events in a stream can cripple
4303     // a trace.
4304     if (!ShouldXRayInstrumentFunction())
4305       return RValue::getIgnored();
4306
4307     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
4308             XRayInstrKind::Typed))
4309       return RValue::getIgnored();
4310
4311     if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
4312       if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
4313         return RValue::getIgnored();
4314
4315     Function *F = CGM.getIntrinsic(Intrinsic::xray_typedevent);
4316     auto FTy = F->getFunctionType();
4317     auto Arg0 = EmitScalarExpr(E->getArg(0));
4318     auto PTy0 = FTy->getParamType(0);
4319     if (PTy0 != Arg0->getType())
4320       Arg0 = Builder.CreateTruncOrBitCast(Arg0, PTy0);
4321     auto Arg1 = E->getArg(1);
4322     auto Arg1Val = EmitScalarExpr(Arg1);
4323     auto Arg1Ty = Arg1->getType();
4324     auto PTy1 = FTy->getParamType(1);
4325     if (PTy1 != Arg1Val->getType()) {
4326       if (Arg1Ty->isArrayType())
4327         Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer();
4328       else
4329         Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
4330     }
4331     auto Arg2 = EmitScalarExpr(E->getArg(2));
4332     auto PTy2 = FTy->getParamType(2);
4333     if (PTy2 != Arg2->getType())
4334       Arg2 = Builder.CreateTruncOrBitCast(Arg2, PTy2);
4335     return RValue::get(Builder.CreateCall(F, {Arg0, Arg1Val, Arg2}));
4336   }
4337
4338   case Builtin::BI__builtin_ms_va_start:
4339   case Builtin::BI__builtin_ms_va_end:
4340     return RValue::get(
4341         EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(),
4342                        BuiltinID == Builtin::BI__builtin_ms_va_start));
4343
4344   case Builtin::BI__builtin_ms_va_copy: {
4345     // Lower this manually. We can't reliably determine whether or not any
4346     // given va_copy() is for a Win64 va_list from the calling convention
4347     // alone, because it's legal to do this from a System V ABI function.
4348     // With opaque pointer types, we won't have enough information in LLVM
4349     // IR to determine this from the argument types, either. Best to do it
4350     // now, while we have enough information.
4351     Address DestAddr = EmitMSVAListRef(E->getArg(0));
4352     Address SrcAddr = EmitMSVAListRef(E->getArg(1));
4353
4354     llvm::Type *BPP = Int8PtrPtrTy;
4355
4356     DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"),
4357                        DestAddr.getAlignment());
4358     SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"),
4359                       SrcAddr.getAlignment());
4360
4361     Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val");
4362     return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
4363   }
4364   }
4365
4366   // If this is an alias for a lib function (e.g. __builtin_sin), emit
4367   // the call using the normal call path, but using the unmangled
4368   // version of the function name.
4369   if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
4370     return emitLibraryCall(*this, FD, E,
4371                            CGM.getBuiltinLibFunction(FD, BuiltinID));
4372
4373   // If this is a predefined lib function (e.g. malloc), emit the call
4374   // using exactly the normal call path.
4375   if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
4376     return emitLibraryCall(*this, FD, E,
4377                       cast<llvm::Constant>(EmitScalarExpr(E->getCallee())));
4378
4379   // Check that a call to a target specific builtin has the correct target
4380   // features.
4381   // This is down here to avoid non-target specific builtins, however, if
4382   // generic builtins start to require generic target features then we
4383   // can move this up to the beginning of the function.
4384   checkTargetFeatures(E, FD);
4385
4386   if (unsigned VectorWidth = getContext().BuiltinInfo.getRequiredVectorWidth(BuiltinID))
4387     LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
4388
4389   // See if we have a target specific intrinsic.
4390   const char *Name = getContext().BuiltinInfo.getName(BuiltinID);
4391   Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
4392   StringRef Prefix =
4393       llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
4394   if (!Prefix.empty()) {
4395     IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix.data(), Name);
4396     // NOTE we don't need to perform a compatibility flag check here since the
4397     // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
4398     // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
4399     if (IntrinsicID == Intrinsic::not_intrinsic)
4400       IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
4401   }
4402
4403   if (IntrinsicID != Intrinsic::not_intrinsic) {
4404     SmallVector<Value*, 16> Args;
4405
4406     // Find out if any arguments are required to be integer constant
4407     // expressions.
4408     unsigned ICEArguments = 0;
4409     ASTContext::GetBuiltinTypeError Error;
4410     getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
4411     assert(Error == ASTContext::GE_None && "Should not codegen an error");
4412
4413     Function *F = CGM.getIntrinsic(IntrinsicID);
4414     llvm::FunctionType *FTy = F->getFunctionType();
4415
4416     for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
4417       Value *ArgValue;
4418       // If this is a normal argument, just emit it as a scalar.
4419       if ((ICEArguments & (1 << i)) == 0) {
4420         ArgValue = EmitScalarExpr(E->getArg(i));
4421       } else {
4422         // If this is required to be a constant, constant fold it so that we
4423         // know that the generated intrinsic gets a ConstantInt.
4424         llvm::APSInt Result;
4425         bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
4426         assert(IsConst && "Constant arg isn't actually constant?");
4427         (void)IsConst;
4428         ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
4429       }
4430
4431       // If the intrinsic arg type is different from the builtin arg type
4432       // we need to do a bit cast.
4433       llvm::Type *PTy = FTy->getParamType(i);
4434       if (PTy != ArgValue->getType()) {
4435         // XXX - vector of pointers?
4436         if (auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
4437           if (PtrTy->getAddressSpace() !=
4438               ArgValue->getType()->getPointerAddressSpace()) {
4439             ArgValue = Builder.CreateAddrSpaceCast(
4440               ArgValue,
4441               ArgValue->getType()->getPointerTo(PtrTy->getAddressSpace()));
4442           }
4443         }
4444
4445         assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
4446                "Must be able to losslessly bit cast to param");
4447         ArgValue = Builder.CreateBitCast(ArgValue, PTy);
4448       }
4449
4450       Args.push_back(ArgValue);
4451     }
4452
4453     Value *V = Builder.CreateCall(F, Args);
4454     QualType BuiltinRetType = E->getType();
4455
4456     llvm::Type *RetTy = VoidTy;
4457     if (!BuiltinRetType->isVoidType())
4458       RetTy = ConvertType(BuiltinRetType);
4459
4460     if (RetTy != V->getType()) {
4461       // XXX - vector of pointers?
4462       if (auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
4463         if (PtrTy->getAddressSpace() != V->getType()->getPointerAddressSpace()) {
4464           V = Builder.CreateAddrSpaceCast(
4465             V, V->getType()->getPointerTo(PtrTy->getAddressSpace()));
4466         }
4467       }
4468
4469       assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
4470              "Must be able to losslessly bit cast result type");
4471       V = Builder.CreateBitCast(V, RetTy);
4472     }
4473
4474     return RValue::get(V);
4475   }
4476
4477   // Some target-specific builtins can have aggregate return values, e.g.
4478   // __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force
4479   // ReturnValue to be non-null, so that the target-specific emission code can
4480   // always just emit into it.
4481   TypeEvaluationKind EvalKind = getEvaluationKind(E->getType());
4482   if (EvalKind == TEK_Aggregate && ReturnValue.isNull()) {
4483     Address DestPtr = CreateMemTemp(E->getType(), "agg.tmp");
4484     ReturnValue = ReturnValueSlot(DestPtr, false);
4485   }
4486
4487   // Now see if we can emit a target-specific builtin.
4488   if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E, ReturnValue)) {
4489     switch (EvalKind) {
4490     case TEK_Scalar:
4491       return RValue::get(V);
4492     case TEK_Aggregate:
4493       return RValue::getAggregate(ReturnValue.getValue(),
4494                                   ReturnValue.isVolatile());
4495     case TEK_Complex:
4496       llvm_unreachable("No current target builtin returns complex");
4497     }
4498     llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
4499   }
4500
4501   ErrorUnsupported(E, "builtin function");
4502
4503   // Unknown builtin, for now just dump it out and return undef.
4504   return GetUndefRValue(E->getType());
4505 }
4506
4507 static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
4508                                         unsigned BuiltinID, const CallExpr *E,
4509                                         ReturnValueSlot ReturnValue,
4510                                         llvm::Triple::ArchType Arch) {
4511   switch (Arch) {
4512   case llvm::Triple::arm:
4513   case llvm::Triple::armeb:
4514   case llvm::Triple::thumb:
4515   case llvm::Triple::thumbeb:
4516     return CGF->EmitARMBuiltinExpr(BuiltinID, E, ReturnValue, Arch);
4517   case llvm::Triple::aarch64:
4518   case llvm::Triple::aarch64_32:
4519   case llvm::Triple::aarch64_be:
4520     return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
4521   case llvm::Triple::bpfeb:
4522   case llvm::Triple::bpfel:
4523     return CGF->EmitBPFBuiltinExpr(BuiltinID, E);
4524   case llvm::Triple::x86:
4525   case llvm::Triple::x86_64:
4526     return CGF->EmitX86BuiltinExpr(BuiltinID, E);
4527   case llvm::Triple::ppc:
4528   case llvm::Triple::ppc64:
4529   case llvm::Triple::ppc64le:
4530     return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
4531   case llvm::Triple::r600:
4532   case llvm::Triple::amdgcn:
4533     return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
4534   case llvm::Triple::systemz:
4535     return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
4536   case llvm::Triple::nvptx:
4537   case llvm::Triple::nvptx64:
4538     return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
4539   case llvm::Triple::wasm32:
4540   case llvm::Triple::wasm64:
4541     return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
4542   case llvm::Triple::hexagon:
4543     return CGF->EmitHexagonBuiltinExpr(BuiltinID, E);
4544   default:
4545     return nullptr;
4546   }
4547 }
4548
4549 Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
4550                                               const CallExpr *E,
4551                                               ReturnValueSlot ReturnValue) {
4552   if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) {
4553     assert(getContext().getAuxTargetInfo() && "Missing aux target info");
4554     return EmitTargetArchBuiltinExpr(
4555         this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E,
4556         ReturnValue, getContext().getAuxTargetInfo()->getTriple().getArch());
4557   }
4558
4559   return EmitTargetArchBuiltinExpr(this, BuiltinID, E, ReturnValue,
4560                                    getTarget().getTriple().getArch());
4561 }
4562
4563 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
4564                                      NeonTypeFlags TypeFlags,
4565                                      bool HasLegalHalfType = true,
4566                                      bool V1Ty = false,
4567                                      bool AllowBFloatArgsAndRet = true) {
4568   int IsQuad = TypeFlags.isQuad();
4569   switch (TypeFlags.getEltType()) {
4570   case NeonTypeFlags::Int8:
4571   case NeonTypeFlags::Poly8:
4572     return llvm::FixedVectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
4573   case NeonTypeFlags::Int16:
4574   case NeonTypeFlags::Poly16:
4575     return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
4576   case NeonTypeFlags::BFloat16:
4577     if (AllowBFloatArgsAndRet)
4578       return llvm::FixedVectorType::get(CGF->BFloatTy, V1Ty ? 1 : (4 << IsQuad));
4579     else
4580       return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
4581   case NeonTypeFlags::Float16:
4582     if (HasLegalHalfType)
4583       return llvm::FixedVectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad));
4584     else
4585       return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
4586   case NeonTypeFlags::Int32:
4587     return llvm::FixedVectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
4588   case NeonTypeFlags::Int64:
4589   case NeonTypeFlags::Poly64:
4590     return llvm::FixedVectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
4591   case NeonTypeFlags::Poly128:
4592     // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
4593     // There is a lot of i128 and f128 API missing.
4594     // so we use v16i8 to represent poly128 and get pattern matched.
4595     return llvm::FixedVectorType::get(CGF->Int8Ty, 16);
4596   case NeonTypeFlags::Float32:
4597     return llvm::FixedVectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
4598   case NeonTypeFlags::Float64:
4599     return llvm::FixedVectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
4600   }
4601   llvm_unreachable("Unknown vector element type!");
4602 }
4603
4604 static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
4605                                           NeonTypeFlags IntTypeFlags) {
4606   int IsQuad = IntTypeFlags.isQuad();
4607   switch (IntTypeFlags.getEltType()) {
4608   case NeonTypeFlags::Int16:
4609     return llvm::FixedVectorType::get(CGF->HalfTy, (4 << IsQuad));
4610   case NeonTypeFlags::Int32:
4611     return llvm::FixedVectorType::get(CGF->FloatTy, (2 << IsQuad));
4612   case NeonTypeFlags::Int64:
4613     return llvm::FixedVectorType::get(CGF->DoubleTy, (1 << IsQuad));
4614   default:
4615     llvm_unreachable("Type can't be converted to floating-point!");
4616   }
4617 }
4618
4619 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C,
4620                                       const ElementCount &Count) {
4621   Value *SV = llvm::ConstantVector::getSplat(Count, C);
4622   return Builder.CreateShuffleVector(V, V, SV, "lane");
4623 }
4624
4625 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
4626   ElementCount EC = cast<llvm::VectorType>(V->getType())->getElementCount();
4627   return EmitNeonSplat(V, C, EC);
4628 }
4629
4630 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
4631                                      const char *name,
4632                                      unsigned shift, bool rightshift) {
4633   unsigned j = 0;
4634   for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
4635        ai != ae; ++ai, ++j) {
4636     if (F->isConstrainedFPIntrinsic())
4637       if (ai->getType()->isMetadataTy())
4638         continue;
4639     if (shift > 0 && shift == j)
4640       Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
4641     else
4642       Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
4643   }
4644
4645   if (F->isConstrainedFPIntrinsic())
4646     return Builder.CreateConstrainedFPCall(F, Ops, name);
4647   else
4648     return Builder.CreateCall(F, Ops, name);
4649 }
4650
4651 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
4652                                             bool neg) {
4653   int SV = cast<ConstantInt>(V)->getSExtValue();
4654   return ConstantInt::get(Ty, neg ? -SV : SV);
4655 }
4656
4657 // Right-shift a vector by a constant.
4658 Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
4659                                           llvm::Type *Ty, bool usgn,
4660                                           const char *name) {
4661   llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
4662
4663   int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
4664   int EltSize = VTy->getScalarSizeInBits();
4665
4666   Vec = Builder.CreateBitCast(Vec, Ty);
4667
4668   // lshr/ashr are undefined when the shift amount is equal to the vector
4669   // element size.
4670   if (ShiftAmt == EltSize) {
4671     if (usgn) {
4672       // Right-shifting an unsigned value by its size yields 0.
4673       return llvm::ConstantAggregateZero::get(VTy);
4674     } else {
4675       // Right-shifting a signed value by its size is equivalent
4676       // to a shift of size-1.
4677       --ShiftAmt;
4678       Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
4679     }
4680   }
4681
4682   Shift = EmitNeonShiftVector(Shift, Ty, false);
4683   if (usgn)
4684     return Builder.CreateLShr(Vec, Shift, name);
4685   else
4686     return Builder.CreateAShr(Vec, Shift, name);
4687 }
4688
4689 enum {
4690   AddRetType = (1 << 0),
4691   Add1ArgType = (1 << 1),
4692   Add2ArgTypes = (1 << 2),
4693
4694   VectorizeRetType = (1 << 3),
4695   VectorizeArgTypes = (1 << 4),
4696
4697   InventFloatType = (1 << 5),
4698   UnsignedAlts = (1 << 6),
4699
4700   Use64BitVectors = (1 << 7),
4701   Use128BitVectors = (1 << 8),
4702
4703   Vectorize1ArgType = Add1ArgType | VectorizeArgTypes,
4704   VectorRet = AddRetType | VectorizeRetType,
4705   VectorRetGetArgs01 =
4706       AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes,
4707   FpCmpzModifiers =
4708       AddRetType | VectorizeRetType | Add1ArgType | InventFloatType
4709 };
4710
4711 namespace {
4712 struct ARMVectorIntrinsicInfo {
4713   const char *NameHint;
4714   unsigned BuiltinID;
4715   unsigned LLVMIntrinsic;
4716   unsigned AltLLVMIntrinsic;
4717   uint64_t TypeModifier;
4718
4719   bool operator<(unsigned RHSBuiltinID) const {
4720     return BuiltinID < RHSBuiltinID;
4721   }
4722   bool operator<(const ARMVectorIntrinsicInfo &TE) const {
4723     return BuiltinID < TE.BuiltinID;
4724   }
4725 };
4726 } // end anonymous namespace
4727
4728 #define NEONMAP0(NameBase) \
4729   { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
4730
4731 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
4732   { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
4733       Intrinsic::LLVMIntrinsic, 0, TypeModifier }
4734
4735 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
4736   { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
4737       Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
4738       TypeModifier }
4739
4740 static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
4741   NEONMAP1(__a32_vcvt_bf16_v, arm_neon_vcvtfp2bf, 0),
4742   NEONMAP0(splat_lane_v),
4743   NEONMAP0(splat_laneq_v),
4744   NEONMAP0(splatq_lane_v),
4745   NEONMAP0(splatq_laneq_v),
4746   NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
4747   NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
4748   NEONMAP1(vabs_v, arm_neon_vabs, 0),
4749   NEONMAP1(vabsq_v, arm_neon_vabs, 0),
4750   NEONMAP0(vaddhn_v),
4751   NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
4752   NEONMAP1(vaeseq_v, arm_neon_aese, 0),
4753   NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
4754   NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
4755   NEONMAP1(vbfdot_v, arm_neon_bfdot, 0),
4756   NEONMAP1(vbfdotq_v, arm_neon_bfdot, 0),
4757   NEONMAP1(vbfmlalbq_v, arm_neon_bfmlalb, 0),
4758   NEONMAP1(vbfmlaltq_v, arm_neon_bfmlalt, 0),
4759   NEONMAP1(vbfmmlaq_v, arm_neon_bfmmla, 0),
4760   NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
4761   NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
4762   NEONMAP1(vcadd_rot270_v, arm_neon_vcadd_rot270, Add1ArgType),
4763   NEONMAP1(vcadd_rot90_v, arm_neon_vcadd_rot90, Add1ArgType),
4764   NEONMAP1(vcaddq_rot270_v, arm_neon_vcadd_rot270, Add1ArgType),
4765   NEONMAP1(vcaddq_rot90_v, arm_neon_vcadd_rot90, Add1ArgType),
4766   NEONMAP1(vcage_v, arm_neon_vacge, 0),
4767   NEONMAP1(vcageq_v, arm_neon_vacge, 0),
4768   NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
4769   NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
4770   NEONMAP1(vcale_v, arm_neon_vacge, 0),
4771   NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
4772   NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
4773   NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
4774   NEONMAP0(vceqz_v),
4775   NEONMAP0(vceqzq_v),
4776   NEONMAP0(vcgez_v),
4777   NEONMAP0(vcgezq_v),
4778   NEONMAP0(vcgtz_v),
4779   NEONMAP0(vcgtzq_v),
4780   NEONMAP0(vclez_v),
4781   NEONMAP0(vclezq_v),
4782   NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
4783   NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
4784   NEONMAP0(vcltz_v),
4785   NEONMAP0(vcltzq_v),
4786   NEONMAP1(vclz_v, ctlz, Add1ArgType),
4787   NEONMAP1(vclzq_v, ctlz, Add1ArgType),
4788   NEONMAP1(vcnt_v, ctpop, Add1ArgType),
4789   NEONMAP1(vcntq_v, ctpop, Add1ArgType),
4790   NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
4791   NEONMAP0(vcvt_f16_v),
4792   NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
4793   NEONMAP0(vcvt_f32_v),
4794   NEONMAP2(vcvt_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4795   NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4796   NEONMAP1(vcvt_n_s16_v, arm_neon_vcvtfp2fxs, 0),
4797   NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
4798   NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
4799   NEONMAP1(vcvt_n_u16_v, arm_neon_vcvtfp2fxu, 0),
4800   NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
4801   NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
4802   NEONMAP0(vcvt_s16_v),
4803   NEONMAP0(vcvt_s32_v),
4804   NEONMAP0(vcvt_s64_v),
4805   NEONMAP0(vcvt_u16_v),
4806   NEONMAP0(vcvt_u32_v),
4807   NEONMAP0(vcvt_u64_v),
4808   NEONMAP1(vcvta_s16_v, arm_neon_vcvtas, 0),
4809   NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
4810   NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
4811   NEONMAP1(vcvta_u16_v, arm_neon_vcvtau, 0),
4812   NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
4813   NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
4814   NEONMAP1(vcvtaq_s16_v, arm_neon_vcvtas, 0),
4815   NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
4816   NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
4817   NEONMAP1(vcvtaq_u16_v, arm_neon_vcvtau, 0),
4818   NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
4819   NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
4820   NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
4821   NEONMAP1(vcvtm_s16_v, arm_neon_vcvtms, 0),
4822   NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
4823   NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
4824   NEONMAP1(vcvtm_u16_v, arm_neon_vcvtmu, 0),
4825   NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
4826   NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
4827   NEONMAP1(vcvtmq_s16_v, arm_neon_vcvtms, 0),
4828   NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
4829   NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
4830   NEONMAP1(vcvtmq_u16_v, arm_neon_vcvtmu, 0),
4831   NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
4832   NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
4833   NEONMAP1(vcvtn_s16_v, arm_neon_vcvtns, 0),
4834   NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
4835   NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
4836   NEONMAP1(vcvtn_u16_v, arm_neon_vcvtnu, 0),
4837   NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
4838   NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
4839   NEONMAP1(vcvtnq_s16_v, arm_neon_vcvtns, 0),
4840   NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
4841   NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
4842   NEONMAP1(vcvtnq_u16_v, arm_neon_vcvtnu, 0),
4843   NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
4844   NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
4845   NEONMAP1(vcvtp_s16_v, arm_neon_vcvtps, 0),
4846   NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
4847   NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
4848   NEONMAP1(vcvtp_u16_v, arm_neon_vcvtpu, 0),
4849   NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
4850   NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
4851   NEONMAP1(vcvtpq_s16_v, arm_neon_vcvtps, 0),
4852   NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
4853   NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
4854   NEONMAP1(vcvtpq_u16_v, arm_neon_vcvtpu, 0),
4855   NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
4856   NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
4857   NEONMAP0(vcvtq_f16_v),
4858   NEONMAP0(vcvtq_f32_v),
4859   NEONMAP2(vcvtq_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4860   NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4861   NEONMAP1(vcvtq_n_s16_v, arm_neon_vcvtfp2fxs, 0),
4862   NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
4863   NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
4864   NEONMAP1(vcvtq_n_u16_v, arm_neon_vcvtfp2fxu, 0),
4865   NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
4866   NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
4867   NEONMAP0(vcvtq_s16_v),
4868   NEONMAP0(vcvtq_s32_v),
4869   NEONMAP0(vcvtq_s64_v),
4870   NEONMAP0(vcvtq_u16_v),
4871   NEONMAP0(vcvtq_u32_v),
4872   NEONMAP0(vcvtq_u64_v),
4873   NEONMAP2(vdot_v, arm_neon_udot, arm_neon_sdot, 0),
4874   NEONMAP2(vdotq_v, arm_neon_udot, arm_neon_sdot, 0),
4875   NEONMAP0(vext_v),
4876   NEONMAP0(vextq_v),
4877   NEONMAP0(vfma_v),
4878   NEONMAP0(vfmaq_v),
4879   NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
4880   NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
4881   NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
4882   NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
4883   NEONMAP0(vld1_dup_v),
4884   NEONMAP1(vld1_v, arm_neon_vld1, 0),
4885   NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
4886   NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
4887   NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
4888   NEONMAP0(vld1q_dup_v),
4889   NEONMAP1(vld1q_v, arm_neon_vld1, 0),
4890   NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
4891   NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
4892   NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
4893   NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
4894   NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
4895   NEONMAP1(vld2_v, arm_neon_vld2, 0),
4896   NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
4897   NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
4898   NEONMAP1(vld2q_v, arm_neon_vld2, 0),
4899   NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
4900   NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
4901   NEONMAP1(vld3_v, arm_neon_vld3, 0),
4902   NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
4903   NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
4904   NEONMAP1(vld3q_v, arm_neon_vld3, 0),
4905   NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
4906   NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
4907   NEONMAP1(vld4_v, arm_neon_vld4, 0),
4908   NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
4909   NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
4910   NEONMAP1(vld4q_v, arm_neon_vld4, 0),
4911   NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
4912   NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
4913   NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
4914   NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
4915   NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
4916   NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
4917   NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
4918   NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
4919   NEONMAP2(vmmlaq_v, arm_neon_ummla, arm_neon_smmla, 0),
4920   NEONMAP0(vmovl_v),
4921   NEONMAP0(vmovn_v),
4922   NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
4923   NEONMAP0(vmull_v),
4924   NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
4925   NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
4926   NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
4927   NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
4928   NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
4929   NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
4930   NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
4931   NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
4932   NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
4933   NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
4934   NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
4935   NEONMAP2(vqadd_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
4936   NEONMAP2(vqaddq_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
4937   NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
4938   NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
4939   NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
4940   NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
4941   NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
4942   NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
4943   NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
4944   NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
4945   NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
4946   NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
4947   NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
4948   NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
4949   NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
4950   NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
4951   NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
4952   NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
4953   NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
4954   NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
4955   NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
4956   NEONMAP2(vqsub_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
4957   NEONMAP2(vqsubq_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
4958   NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
4959   NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
4960   NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
4961   NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
4962   NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
4963   NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
4964   NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
4965   NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType),
4966   NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType),
4967   NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType),
4968   NEONMAP0(vrndi_v),
4969   NEONMAP0(vrndiq_v),
4970   NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType),
4971   NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType),
4972   NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType),
4973   NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType),
4974   NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType),
4975   NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType),
4976   NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType),
4977   NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType),
4978   NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType),
4979   NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
4980   NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
4981   NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
4982   NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
4983   NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
4984   NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
4985   NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
4986   NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
4987   NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
4988   NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
4989   NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
4990   NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
4991   NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
4992   NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
4993   NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
4994   NEONMAP0(vshl_n_v),
4995   NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
4996   NEONMAP0(vshll_n_v),
4997   NEONMAP0(vshlq_n_v),
4998   NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
4999   NEONMAP0(vshr_n_v),
5000   NEONMAP0(vshrn_n_v),
5001   NEONMAP0(vshrq_n_v),
5002   NEONMAP1(vst1_v, arm_neon_vst1, 0),
5003   NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
5004   NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
5005   NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
5006   NEONMAP1(vst1q_v, arm_neon_vst1, 0),
5007   NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
5008   NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
5009   NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
5010   NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
5011   NEONMAP1(vst2_v, arm_neon_vst2, 0),
5012   NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
5013   NEONMAP1(vst2q_v, arm_neon_vst2, 0),
5014   NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
5015   NEONMAP1(vst3_v, arm_neon_vst3, 0),
5016   NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
5017   NEONMAP1(vst3q_v, arm_neon_vst3, 0),
5018   NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
5019   NEONMAP1(vst4_v, arm_neon_vst4, 0),
5020   NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
5021   NEONMAP1(vst4q_v, arm_neon_vst4, 0),
5022   NEONMAP0(vsubhn_v),
5023   NEONMAP0(vtrn_v),
5024   NEONMAP0(vtrnq_v),
5025   NEONMAP0(vtst_v),
5026   NEONMAP0(vtstq_v),
5027   NEONMAP1(vusdot_v, arm_neon_usdot, 0),
5028   NEONMAP1(vusdotq_v, arm_neon_usdot, 0),
5029   NEONMAP1(vusmmlaq_v, arm_neon_usmmla, 0),
5030   NEONMAP0(vuzp_v),
5031   NEONMAP0(vuzpq_v),
5032   NEONMAP0(vzip_v),
5033   NEONMAP0(vzipq_v)
5034 };
5035
5036 static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
5037   NEONMAP1(__a64_vcvtq_low_bf16_v, aarch64_neon_bfcvtn, 0),
5038   NEONMAP0(splat_lane_v),
5039   NEONMAP0(splat_laneq_v),
5040   NEONMAP0(splatq_lane_v),
5041   NEONMAP0(splatq_laneq_v),
5042   NEONMAP1(vabs_v, aarch64_neon_abs, 0),
5043   NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
5044   NEONMAP0(vaddhn_v),
5045   NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
5046   NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
5047   NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
5048   NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
5049   NEONMAP1(vbfdot_v, aarch64_neon_bfdot, 0),
5050   NEONMAP1(vbfdotq_v, aarch64_neon_bfdot, 0),
5051   NEONMAP1(vbfmlalbq_v, aarch64_neon_bfmlalb, 0),
5052   NEONMAP1(vbfmlaltq_v, aarch64_neon_bfmlalt, 0),
5053   NEONMAP1(vbfmmlaq_v, aarch64_neon_bfmmla, 0),
5054   NEONMAP1(vcadd_rot270_v, aarch64_neon_vcadd_rot270, Add1ArgType),
5055   NEONMAP1(vcadd_rot90_v, aarch64_neon_vcadd_rot90, Add1ArgType),
5056   NEONMAP1(vcaddq_rot270_v, aarch64_neon_vcadd_rot270, Add1ArgType),
5057   NEONMAP1(vcaddq_rot90_v, aarch64_neon_vcadd_rot90, Add1ArgType),
5058   NEONMAP1(vcage_v, aarch64_neon_facge, 0),
5059   NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
5060   NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
5061   NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
5062   NEONMAP1(vcale_v, aarch64_neon_facge, 0),
5063   NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
5064   NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
5065   NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
5066   NEONMAP0(vceqz_v),
5067   NEONMAP0(vceqzq_v),
5068   NEONMAP0(vcgez_v),
5069   NEONMAP0(vcgezq_v),
5070   NEONMAP0(vcgtz_v),
5071   NEONMAP0(vcgtzq_v),
5072   NEONMAP0(vclez_v),
5073   NEONMAP0(vclezq_v),
5074   NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType),
5075   NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType),
5076   NEONMAP0(vcltz_v),
5077   NEONMAP0(vcltzq_v),
5078   NEONMAP1(vclz_v, ctlz, Add1ArgType),
5079   NEONMAP1(vclzq_v, ctlz, Add1ArgType),
5080   NEONMAP1(vcnt_v, ctpop, Add1ArgType),
5081   NEONMAP1(vcntq_v, ctpop, Add1ArgType),
5082   NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
5083   NEONMAP0(vcvt_f16_v),
5084   NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
5085   NEONMAP0(vcvt_f32_v),
5086   NEONMAP2(vcvt_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
5087   NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
5088   NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
5089   NEONMAP1(vcvt_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
5090   NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
5091   NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
5092   NEONMAP1(vcvt_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
5093   NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
5094   NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
5095   NEONMAP0(vcvtq_f16_v),
5096   NEONMAP0(vcvtq_f32_v),
5097   NEONMAP1(vcvtq_high_bf16_v, aarch64_neon_bfcvtn2, 0),
5098   NEONMAP2(vcvtq_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
5099   NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
5100   NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
5101   NEONMAP1(vcvtq_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
5102   NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
5103   NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
5104   NEONMAP1(vcvtq_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
5105   NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
5106   NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
5107   NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType),
5108   NEONMAP2(vdot_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
5109   NEONMAP2(vdotq_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
5110   NEONMAP0(vext_v),
5111   NEONMAP0(vextq_v),
5112   NEONMAP0(vfma_v),
5113   NEONMAP0(vfmaq_v),
5114   NEONMAP1(vfmlal_high_v, aarch64_neon_fmlal2, 0),
5115   NEONMAP1(vfmlal_low_v, aarch64_neon_fmlal, 0),
5116   NEONMAP1(vfmlalq_high_v, aarch64_neon_fmlal2, 0),
5117   NEONMAP1(vfmlalq_low_v, aarch64_neon_fmlal, 0),
5118   NEONMAP1(vfmlsl_high_v, aarch64_neon_fmlsl2, 0),
5119   NEONMAP1(vfmlsl_low_v, aarch64_neon_fmlsl, 0),
5120   NEONMAP1(vfmlslq_high_v, aarch64_neon_fmlsl2, 0),
5121   NEONMAP1(vfmlslq_low_v, aarch64_neon_fmlsl, 0),
5122   NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
5123   NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
5124   NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
5125   NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
5126   NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
5127   NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
5128   NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
5129   NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
5130   NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
5131   NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
5132   NEONMAP2(vmmlaq_v, aarch64_neon_ummla, aarch64_neon_smmla, 0),
5133   NEONMAP0(vmovl_v),
5134   NEONMAP0(vmovn_v),
5135   NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType),
5136   NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType),
5137   NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType),
5138   NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
5139   NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
5140   NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType),
5141   NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType),
5142   NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType),
5143   NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
5144   NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
5145   NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
5146   NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
5147   NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
5148   NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
5149   NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType),
5150   NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
5151   NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
5152   NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType),
5153   NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType),
5154   NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts),
5155   NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType),
5156   NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType),
5157   NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType),
5158   NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
5159   NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
5160   NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType),
5161   NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
5162   NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
5163   NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType),
5164   NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
5165   NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
5166   NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts),
5167   NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
5168   NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts),
5169   NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
5170   NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
5171   NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
5172   NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
5173   NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
5174   NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType),
5175   NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
5176   NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
5177   NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType),
5178   NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType),
5179   NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
5180   NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
5181   NEONMAP0(vrndi_v),
5182   NEONMAP0(vrndiq_v),
5183   NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
5184   NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
5185   NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
5186   NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
5187   NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
5188   NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
5189   NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType),
5190   NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType),
5191   NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType),
5192   NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
5193   NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
5194   NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
5195   NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
5196   NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
5197   NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
5198   NEONMAP0(vshl_n_v),
5199   NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
5200   NEONMAP0(vshll_n_v),
5201   NEONMAP0(vshlq_n_v),
5202   NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
5203   NEONMAP0(vshr_n_v),
5204   NEONMAP0(vshrn_n_v),
5205   NEONMAP0(vshrq_n_v),
5206   NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
5207   NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
5208   NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
5209   NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
5210   NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
5211   NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
5212   NEONMAP0(vsubhn_v),
5213   NEONMAP0(vtst_v),
5214   NEONMAP0(vtstq_v),
5215   NEONMAP1(vusdot_v, aarch64_neon_usdot, 0),
5216   NEONMAP1(vusdotq_v, aarch64_neon_usdot, 0),
5217   NEONMAP1(vusmmlaq_v, aarch64_neon_usmmla, 0),
5218 };
5219
5220 static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[] = {
5221   NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType),
5222   NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType),
5223   NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType),
5224   NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
5225   NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
5226   NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
5227   NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
5228   NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
5229   NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
5230   NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
5231   NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
5232   NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType),
5233   NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
5234   NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType),
5235   NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
5236   NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
5237   NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
5238   NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
5239   NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
5240   NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
5241   NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
5242   NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
5243   NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
5244   NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
5245   NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
5246   NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
5247   NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
5248   NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
5249   NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
5250   NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
5251   NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
5252   NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
5253   NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
5254   NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
5255   NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
5256   NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
5257   NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
5258   NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
5259   NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
5260   NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
5261   NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
5262   NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
5263   NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
5264   NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
5265   NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
5266   NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
5267   NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
5268   NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
5269   NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
5270   NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
5271   NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
5272   NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
5273   NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
5274   NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
5275   NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
5276   NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
5277   NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
5278   NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
5279   NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
5280   NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
5281   NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
5282   NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
5283   NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
5284   NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
5285   NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
5286   NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
5287   NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
5288   NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
5289   NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
5290   NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
5291   NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
5292   NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType),
5293   NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType),
5294   NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
5295   NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
5296   NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
5297   NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
5298   NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
5299   NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
5300   NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
5301   NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
5302   NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
5303   NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
5304   NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
5305   NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType),
5306   NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
5307   NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType),
5308   NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
5309   NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
5310   NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType),
5311   NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType),
5312   NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
5313   NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
5314   NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType),
5315   NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType),
5316   NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors),
5317   NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType),
5318   NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors),
5319   NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
5320   NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType),
5321   NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType),
5322   NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
5323   NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
5324   NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
5325   NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
5326   NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType),
5327   NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
5328   NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
5329   NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
5330   NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType),
5331   NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
5332   NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType),
5333   NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors),
5334   NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType),
5335   NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
5336   NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
5337   NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType),
5338   NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType),
5339   NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
5340   NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
5341   NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType),
5342   NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType),
5343   NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType),
5344   NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType),
5345   NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
5346   NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
5347   NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
5348   NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
5349   NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType),
5350   NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
5351   NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
5352   NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
5353   NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
5354   NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
5355   NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
5356   NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType),
5357   NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType),
5358   NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
5359   NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
5360   NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
5361   NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
5362   NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType),
5363   NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType),
5364   NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType),
5365   NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType),
5366   NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
5367   NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
5368   NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType),
5369   NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType),
5370   NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType),
5371   NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
5372   NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
5373   NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
5374   NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
5375   NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType),
5376   NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
5377   NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
5378   NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
5379   NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
5380   NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType),
5381   NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType),
5382   NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
5383   NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
5384   NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType),
5385   NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType),
5386   NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType),
5387   NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType),
5388   NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType),
5389   NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType),
5390   NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType),
5391   NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType),
5392   NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType),
5393   NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType),
5394   NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType),
5395   NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType),
5396   NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
5397   NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
5398   NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
5399   NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
5400   NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType),
5401   NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType),
5402   NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType),
5403   NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType),
5404   NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
5405   NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType),
5406   NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
5407   NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType),
5408   NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType),
5409   NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType),
5410   NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
5411   NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType),
5412   NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
5413   NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType),
5414   // FP16 scalar intrinisics go here.
5415   NEONMAP1(vabdh_f16, aarch64_sisd_fabd, Add1ArgType),
5416   NEONMAP1(vcvtah_s32_f16, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
5417   NEONMAP1(vcvtah_s64_f16, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
5418   NEONMAP1(vcvtah_u32_f16, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
5419   NEONMAP1(vcvtah_u64_f16, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
5420   NEONMAP1(vcvth_n_f16_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
5421   NEONMAP1(vcvth_n_f16_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
5422   NEONMAP1(vcvth_n_f16_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
5423   NEONMAP1(vcvth_n_f16_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
5424   NEONMAP1(vcvth_n_s32_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
5425   NEONMAP1(vcvth_n_s64_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
5426   NEONMAP1(vcvth_n_u32_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
5427   NEONMAP1(vcvth_n_u64_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
5428   NEONMAP1(vcvtmh_s32_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
5429   NEONMAP1(vcvtmh_s64_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
5430   NEONMAP1(vcvtmh_u32_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
5431   NEONMAP1(vcvtmh_u64_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
5432   NEONMAP1(vcvtnh_s32_f16, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
5433   NEONMAP1(vcvtnh_s64_f16, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
5434   NEONMAP1(vcvtnh_u32_f16, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
5435   NEONMAP1(vcvtnh_u64_f16, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
5436   NEONMAP1(vcvtph_s32_f16, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
5437   NEONMAP1(vcvtph_s64_f16, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
5438   NEONMAP1(vcvtph_u32_f16, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
5439   NEONMAP1(vcvtph_u64_f16, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
5440   NEONMAP1(vmulxh_f16, aarch64_neon_fmulx, Add1ArgType),
5441   NEONMAP1(vrecpeh_f16, aarch64_neon_frecpe, Add1ArgType),
5442   NEONMAP1(vrecpxh_f16, aarch64_neon_frecpx, Add1ArgType),
5443   NEONMAP1(vrsqrteh_f16, aarch64_neon_frsqrte, Add1ArgType),
5444   NEONMAP1(vrsqrtsh_f16, aarch64_neon_frsqrts, Add1ArgType),
5445 };
5446
5447 #undef NEONMAP0
5448 #undef NEONMAP1
5449 #undef NEONMAP2
5450
5451 #define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier)                         \
5452   {                                                                            \
5453     #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0,   \
5454         TypeModifier                                                           \
5455   }
5456
5457 #define SVEMAP2(NameBase, TypeModifier)                                        \
5458   { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
5459 static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[] = {
5460 #define GET_SVE_LLVM_INTRINSIC_MAP
5461 #include "clang/Basic/arm_sve_builtin_cg.inc"
5462 #undef GET_SVE_LLVM_INTRINSIC_MAP
5463 };
5464
5465 #undef SVEMAP1
5466 #undef SVEMAP2
5467
5468 static bool NEONSIMDIntrinsicsProvenSorted = false;
5469
5470 static bool AArch64SIMDIntrinsicsProvenSorted = false;
5471 static bool AArch64SISDIntrinsicsProvenSorted = false;
5472 static bool AArch64SVEIntrinsicsProvenSorted = false;
5473
5474 static const ARMVectorIntrinsicInfo *
5475 findARMVectorIntrinsicInMap(ArrayRef<ARMVectorIntrinsicInfo> IntrinsicMap,
5476                             unsigned BuiltinID, bool &MapProvenSorted) {
5477
5478 #ifndef NDEBUG
5479   if (!MapProvenSorted) {
5480     assert(llvm::is_sorted(IntrinsicMap));
5481     MapProvenSorted = true;
5482   }
5483 #endif
5484
5485   const ARMVectorIntrinsicInfo *Builtin =
5486       llvm::lower_bound(IntrinsicMap, BuiltinID);
5487
5488   if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
5489     return Builtin;
5490
5491   return nullptr;
5492 }
5493
5494 Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
5495                                                    unsigned Modifier,
5496                                                    llvm::Type *ArgType,
5497                                                    const CallExpr *E) {
5498   int VectorSize = 0;
5499   if (Modifier & Use64BitVectors)
5500     VectorSize = 64;
5501   else if (Modifier & Use128BitVectors)
5502     VectorSize = 128;
5503
5504   // Return type.
5505   SmallVector<llvm::Type *, 3> Tys;
5506   if (Modifier & AddRetType) {
5507     llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
5508     if (Modifier & VectorizeRetType)
5509       Ty = llvm::FixedVectorType::get(
5510           Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
5511
5512     Tys.push_back(Ty);
5513   }
5514
5515   // Arguments.
5516   if (Modifier & VectorizeArgTypes) {
5517     int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
5518     ArgType = llvm::FixedVectorType::get(ArgType, Elts);
5519   }
5520
5521   if (Modifier & (Add1ArgType | Add2ArgTypes))
5522     Tys.push_back(ArgType);
5523
5524   if (Modifier & Add2ArgTypes)
5525     Tys.push_back(ArgType);
5526
5527   if (Modifier & InventFloatType)
5528     Tys.push_back(FloatTy);
5529
5530   return CGM.getIntrinsic(IntrinsicID, Tys);
5531 }
5532
5533 static Value *EmitCommonNeonSISDBuiltinExpr(
5534     CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo,
5535     SmallVectorImpl<Value *> &Ops, const CallExpr *E) {
5536   unsigned BuiltinID = SISDInfo.BuiltinID;
5537   unsigned int Int = SISDInfo.LLVMIntrinsic;
5538   unsigned Modifier = SISDInfo.TypeModifier;
5539   const char *s = SISDInfo.NameHint;
5540
5541   switch (BuiltinID) {
5542   case NEON::BI__builtin_neon_vcled_s64:
5543   case NEON::BI__builtin_neon_vcled_u64:
5544   case NEON::BI__builtin_neon_vcles_f32:
5545   case NEON::BI__builtin_neon_vcled_f64:
5546   case NEON::BI__builtin_neon_vcltd_s64:
5547   case NEON::BI__builtin_neon_vcltd_u64:
5548   case NEON::BI__builtin_neon_vclts_f32:
5549   case NEON::BI__builtin_neon_vcltd_f64:
5550   case NEON::BI__builtin_neon_vcales_f32:
5551   case NEON::BI__builtin_neon_vcaled_f64:
5552   case NEON::BI__builtin_neon_vcalts_f32:
5553   case NEON::BI__builtin_neon_vcaltd_f64:
5554     // Only one direction of comparisons actually exist, cmle is actually a cmge
5555     // with swapped operands. The table gives us the right intrinsic but we
5556     // still need to do the swap.
5557     std::swap(Ops[0], Ops[1]);
5558     break;
5559   }
5560
5561   assert(Int && "Generic code assumes a valid intrinsic");
5562
5563   // Determine the type(s) of this overloaded AArch64 intrinsic.
5564   const Expr *Arg = E->getArg(0);
5565   llvm::Type *ArgTy = CGF.ConvertType(Arg->getType());
5566   Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E);
5567
5568   int j = 0;
5569   ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0);
5570   for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
5571        ai != ae; ++ai, ++j) {
5572     llvm::Type *ArgTy = ai->getType();
5573     if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
5574              ArgTy->getPrimitiveSizeInBits())
5575       continue;
5576
5577     assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
5578     // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
5579     // it before inserting.
5580     Ops[j] = CGF.Builder.CreateTruncOrBitCast(
5581         Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
5582     Ops[j] =
5583         CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
5584   }
5585
5586   Value *Result = CGF.EmitNeonCall(F, Ops, s);
5587   llvm::Type *ResultType = CGF.ConvertType(E->getType());
5588   if (ResultType->getPrimitiveSizeInBits() <
5589       Result->getType()->getPrimitiveSizeInBits())
5590     return CGF.Builder.CreateExtractElement(Result, C0);
5591
5592   return CGF.Builder.CreateBitCast(Result, ResultType, s);
5593 }
5594
5595 Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
5596     unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
5597     const char *NameHint, unsigned Modifier, const CallExpr *E,
5598     SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1,
5599     llvm::Triple::ArchType Arch) {
5600   // Get the last argument, which specifies the vector type.
5601   llvm::APSInt NeonTypeConst;
5602   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
5603   if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext()))
5604     return nullptr;
5605
5606   // Determine the type of this overloaded NEON intrinsic.
5607   NeonTypeFlags Type(NeonTypeConst.getZExtValue());
5608   bool Usgn = Type.isUnsigned();
5609   bool Quad = Type.isQuad();
5610   const bool HasLegalHalfType = getTarget().hasLegalHalfType();
5611   const bool AllowBFloatArgsAndRet =
5612       getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
5613
5614   llvm::VectorType *VTy = GetNeonType(this, Type, HasLegalHalfType, false,
5615                                       AllowBFloatArgsAndRet);
5616   llvm::Type *Ty = VTy;
5617   if (!Ty)
5618     return nullptr;
5619
5620   auto getAlignmentValue32 = [&](Address addr) -> Value* {
5621     return Builder.getInt32(addr.getAlignment().getQuantity());
5622   };
5623
5624   unsigned Int = LLVMIntrinsic;
5625   if ((Modifier & UnsignedAlts) && !Usgn)
5626     Int = AltLLVMIntrinsic;
5627
5628   switch (BuiltinID) {
5629   default: break;
5630   case NEON::BI__builtin_neon_splat_lane_v:
5631   case NEON::BI__builtin_neon_splat_laneq_v:
5632   case NEON::BI__builtin_neon_splatq_lane_v:
5633   case NEON::BI__builtin_neon_splatq_laneq_v: {
5634     auto NumElements = VTy->getElementCount();
5635     if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
5636       NumElements = NumElements * 2;
5637     if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
5638       NumElements = NumElements / 2;
5639
5640     Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
5641     return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
5642   }
5643   case NEON::BI__builtin_neon_vpadd_v:
5644   case NEON::BI__builtin_neon_vpaddq_v:
5645     // We don't allow fp/int overloading of intrinsics.
5646     if (VTy->getElementType()->isFloatingPointTy() &&
5647         Int == Intrinsic::aarch64_neon_addp)
5648       Int = Intrinsic::aarch64_neon_faddp;
5649     break;
5650   case NEON::BI__builtin_neon_vabs_v:
5651   case NEON::BI__builtin_neon_vabsq_v:
5652     if (VTy->getElementType()->isFloatingPointTy())
5653       return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
5654     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs");
5655   case NEON::BI__builtin_neon_vaddhn_v: {
5656     llvm::VectorType *SrcTy =
5657         llvm::VectorType::getExtendedElementVectorType(VTy);
5658
5659     // %sum = add <4 x i32> %lhs, %rhs
5660     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5661     Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
5662     Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
5663
5664     // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
5665     Constant *ShiftAmt =
5666         ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
5667     Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
5668
5669     // %res = trunc <4 x i32> %high to <4 x i16>
5670     return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
5671   }
5672   case NEON::BI__builtin_neon_vcale_v:
5673   case NEON::BI__builtin_neon_vcaleq_v:
5674   case NEON::BI__builtin_neon_vcalt_v:
5675   case NEON::BI__builtin_neon_vcaltq_v:
5676     std::swap(Ops[0], Ops[1]);
5677     LLVM_FALLTHROUGH;
5678   case NEON::BI__builtin_neon_vcage_v:
5679   case NEON::BI__builtin_neon_vcageq_v:
5680   case NEON::BI__builtin_neon_vcagt_v:
5681   case NEON::BI__builtin_neon_vcagtq_v: {
5682     llvm::Type *Ty;
5683     switch (VTy->getScalarSizeInBits()) {
5684     default: llvm_unreachable("unexpected type");
5685     case 32:
5686       Ty = FloatTy;
5687       break;
5688     case 64:
5689       Ty = DoubleTy;
5690       break;
5691     case 16:
5692       Ty = HalfTy;
5693       break;
5694     }
5695     auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
5696     llvm::Type *Tys[] = { VTy, VecFlt };
5697     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5698     return EmitNeonCall(F, Ops, NameHint);
5699   }
5700   case NEON::BI__builtin_neon_vceqz_v:
5701   case NEON::BI__builtin_neon_vceqzq_v:
5702     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
5703                                          ICmpInst::ICMP_EQ, "vceqz");
5704   case NEON::BI__builtin_neon_vcgez_v:
5705   case NEON::BI__builtin_neon_vcgezq_v:
5706     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
5707                                          ICmpInst::ICMP_SGE, "vcgez");
5708   case NEON::BI__builtin_neon_vclez_v:
5709   case NEON::BI__builtin_neon_vclezq_v:
5710     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
5711                                          ICmpInst::ICMP_SLE, "vclez");
5712   case NEON::BI__builtin_neon_vcgtz_v:
5713   case NEON::BI__builtin_neon_vcgtzq_v:
5714     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
5715                                          ICmpInst::ICMP_SGT, "vcgtz");
5716   case NEON::BI__builtin_neon_vcltz_v:
5717   case NEON::BI__builtin_neon_vcltzq_v:
5718     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
5719                                          ICmpInst::ICMP_SLT, "vcltz");
5720   case NEON::BI__builtin_neon_vclz_v:
5721   case NEON::BI__builtin_neon_vclzq_v:
5722     // We generate target-independent intrinsic, which needs a second argument
5723     // for whether or not clz of zero is undefined; on ARM it isn't.
5724     Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
5725     break;
5726   case NEON::BI__builtin_neon_vcvt_f32_v:
5727   case NEON::BI__builtin_neon_vcvtq_f32_v:
5728     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5729     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad),
5730                      HasLegalHalfType);
5731     return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
5732                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
5733   case NEON::BI__builtin_neon_vcvt_f16_v:
5734   case NEON::BI__builtin_neon_vcvtq_f16_v:
5735     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5736     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float16, false, Quad),
5737                      HasLegalHalfType);
5738     return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
5739                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
5740   case NEON::BI__builtin_neon_vcvt_n_f16_v:
5741   case NEON::BI__builtin_neon_vcvt_n_f32_v:
5742   case NEON::BI__builtin_neon_vcvt_n_f64_v:
5743   case NEON::BI__builtin_neon_vcvtq_n_f16_v:
5744   case NEON::BI__builtin_neon_vcvtq_n_f32_v:
5745   case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
5746     llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty };
5747     Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
5748     Function *F = CGM.getIntrinsic(Int, Tys);
5749     return EmitNeonCall(F, Ops, "vcvt_n");
5750   }
5751   case NEON::BI__builtin_neon_vcvt_n_s16_v:
5752   case NEON::BI__builtin_neon_vcvt_n_s32_v:
5753   case NEON::BI__builtin_neon_vcvt_n_u16_v:
5754   case NEON::BI__builtin_neon_vcvt_n_u32_v:
5755   case NEON::BI__builtin_neon_vcvt_n_s64_v:
5756   case NEON::BI__builtin_neon_vcvt_n_u64_v:
5757   case NEON::BI__builtin_neon_vcvtq_n_s16_v:
5758   case NEON::BI__builtin_neon_vcvtq_n_s32_v:
5759   case NEON::BI__builtin_neon_vcvtq_n_u16_v:
5760   case NEON::BI__builtin_neon_vcvtq_n_u32_v:
5761   case NEON::BI__builtin_neon_vcvtq_n_s64_v:
5762   case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
5763     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
5764     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5765     return EmitNeonCall(F, Ops, "vcvt_n");
5766   }
5767   case NEON::BI__builtin_neon_vcvt_s32_v:
5768   case NEON::BI__builtin_neon_vcvt_u32_v:
5769   case NEON::BI__builtin_neon_vcvt_s64_v:
5770   case NEON::BI__builtin_neon_vcvt_u64_v:
5771   case NEON::BI__builtin_neon_vcvt_s16_v:
5772   case NEON::BI__builtin_neon_vcvt_u16_v:
5773   case NEON::BI__builtin_neon_vcvtq_s32_v:
5774   case NEON::BI__builtin_neon_vcvtq_u32_v:
5775   case NEON::BI__builtin_neon_vcvtq_s64_v:
5776   case NEON::BI__builtin_neon_vcvtq_u64_v:
5777   case NEON::BI__builtin_neon_vcvtq_s16_v:
5778   case NEON::BI__builtin_neon_vcvtq_u16_v: {
5779     Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
5780     return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
5781                 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
5782   }
5783   case NEON::BI__builtin_neon_vcvta_s16_v:
5784   case NEON::BI__builtin_neon_vcvta_s32_v:
5785   case NEON::BI__builtin_neon_vcvta_s64_v:
5786   case NEON::BI__builtin_neon_vcvta_u16_v:
5787   case NEON::BI__builtin_neon_vcvta_u32_v:
5788   case NEON::BI__builtin_neon_vcvta_u64_v:
5789   case NEON::BI__builtin_neon_vcvtaq_s16_v:
5790   case NEON::BI__builtin_neon_vcvtaq_s32_v:
5791   case NEON::BI__builtin_neon_vcvtaq_s64_v:
5792   case NEON::BI__builtin_neon_vcvtaq_u16_v:
5793   case NEON::BI__builtin_neon_vcvtaq_u32_v:
5794   case NEON::BI__builtin_neon_vcvtaq_u64_v:
5795   case NEON::BI__builtin_neon_vcvtn_s16_v:
5796   case NEON::BI__builtin_neon_vcvtn_s32_v:
5797   case NEON::BI__builtin_neon_vcvtn_s64_v:
5798   case NEON::BI__builtin_neon_vcvtn_u16_v:
5799   case NEON::BI__builtin_neon_vcvtn_u32_v:
5800   case NEON::BI__builtin_neon_vcvtn_u64_v:
5801   case NEON::BI__builtin_neon_vcvtnq_s16_v:
5802   case NEON::BI__builtin_neon_vcvtnq_s32_v:
5803   case NEON::BI__builtin_neon_vcvtnq_s64_v:
5804   case NEON::BI__builtin_neon_vcvtnq_u16_v:
5805   case NEON::BI__builtin_neon_vcvtnq_u32_v:
5806   case NEON::BI__builtin_neon_vcvtnq_u64_v:
5807   case NEON::BI__builtin_neon_vcvtp_s16_v:
5808   case NEON::BI__builtin_neon_vcvtp_s32_v:
5809   case NEON::BI__builtin_neon_vcvtp_s64_v:
5810   case NEON::BI__builtin_neon_vcvtp_u16_v:
5811   case NEON::BI__builtin_neon_vcvtp_u32_v:
5812   case NEON::BI__builtin_neon_vcvtp_u64_v:
5813   case NEON::BI__builtin_neon_vcvtpq_s16_v:
5814   case NEON::BI__builtin_neon_vcvtpq_s32_v:
5815   case NEON::BI__builtin_neon_vcvtpq_s64_v:
5816   case NEON::BI__builtin_neon_vcvtpq_u16_v:
5817   case NEON::BI__builtin_neon_vcvtpq_u32_v:
5818   case NEON::BI__builtin_neon_vcvtpq_u64_v:
5819   case NEON::BI__builtin_neon_vcvtm_s16_v:
5820   case NEON::BI__builtin_neon_vcvtm_s32_v:
5821   case NEON::BI__builtin_neon_vcvtm_s64_v:
5822   case NEON::BI__builtin_neon_vcvtm_u16_v:
5823   case NEON::BI__builtin_neon_vcvtm_u32_v:
5824   case NEON::BI__builtin_neon_vcvtm_u64_v:
5825   case NEON::BI__builtin_neon_vcvtmq_s16_v:
5826   case NEON::BI__builtin_neon_vcvtmq_s32_v:
5827   case NEON::BI__builtin_neon_vcvtmq_s64_v:
5828   case NEON::BI__builtin_neon_vcvtmq_u16_v:
5829   case NEON::BI__builtin_neon_vcvtmq_u32_v:
5830   case NEON::BI__builtin_neon_vcvtmq_u64_v: {
5831     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
5832     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
5833   }
5834   case NEON::BI__builtin_neon_vcvtx_f32_v: {
5835     llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
5836     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
5837
5838   }
5839   case NEON::BI__builtin_neon_vext_v:
5840   case NEON::BI__builtin_neon_vextq_v: {
5841     int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
5842     SmallVector<int, 16> Indices;
5843     for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
5844       Indices.push_back(i+CV);
5845
5846     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5847     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5848     return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext");
5849   }
5850   case NEON::BI__builtin_neon_vfma_v:
5851   case NEON::BI__builtin_neon_vfmaq_v: {
5852     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5853     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5854     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5855
5856     // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5857     return emitCallMaybeConstrainedFPBuiltin(
5858         *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
5859         {Ops[1], Ops[2], Ops[0]});
5860   }
5861   case NEON::BI__builtin_neon_vld1_v:
5862   case NEON::BI__builtin_neon_vld1q_v: {
5863     llvm::Type *Tys[] = {Ty, Int8PtrTy};
5864     Ops.push_back(getAlignmentValue32(PtrOp0));
5865     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1");
5866   }
5867   case NEON::BI__builtin_neon_vld1_x2_v:
5868   case NEON::BI__builtin_neon_vld1q_x2_v:
5869   case NEON::BI__builtin_neon_vld1_x3_v:
5870   case NEON::BI__builtin_neon_vld1q_x3_v:
5871   case NEON::BI__builtin_neon_vld1_x4_v:
5872   case NEON::BI__builtin_neon_vld1q_x4_v: {
5873     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getElementType());
5874     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5875     llvm::Type *Tys[2] = { VTy, PTy };
5876     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5877     Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
5878     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5879     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5880     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5881   }
5882   case NEON::BI__builtin_neon_vld2_v:
5883   case NEON::BI__builtin_neon_vld2q_v:
5884   case NEON::BI__builtin_neon_vld3_v:
5885   case NEON::BI__builtin_neon_vld3q_v:
5886   case NEON::BI__builtin_neon_vld4_v:
5887   case NEON::BI__builtin_neon_vld4q_v:
5888   case NEON::BI__builtin_neon_vld2_dup_v:
5889   case NEON::BI__builtin_neon_vld2q_dup_v:
5890   case NEON::BI__builtin_neon_vld3_dup_v:
5891   case NEON::BI__builtin_neon_vld3q_dup_v:
5892   case NEON::BI__builtin_neon_vld4_dup_v:
5893   case NEON::BI__builtin_neon_vld4q_dup_v: {
5894     llvm::Type *Tys[] = {Ty, Int8PtrTy};
5895     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5896     Value *Align = getAlignmentValue32(PtrOp1);
5897     Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
5898     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5899     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5900     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5901   }
5902   case NEON::BI__builtin_neon_vld1_dup_v:
5903   case NEON::BI__builtin_neon_vld1q_dup_v: {
5904     Value *V = UndefValue::get(Ty);
5905     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
5906     PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty);
5907     LoadInst *Ld = Builder.CreateLoad(PtrOp0);
5908     llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
5909     Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
5910     return EmitNeonSplat(Ops[0], CI);
5911   }
5912   case NEON::BI__builtin_neon_vld2_lane_v:
5913   case NEON::BI__builtin_neon_vld2q_lane_v:
5914   case NEON::BI__builtin_neon_vld3_lane_v:
5915   case NEON::BI__builtin_neon_vld3q_lane_v:
5916   case NEON::BI__builtin_neon_vld4_lane_v:
5917   case NEON::BI__builtin_neon_vld4q_lane_v: {
5918     llvm::Type *Tys[] = {Ty, Int8PtrTy};
5919     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5920     for (unsigned I = 2; I < Ops.size() - 1; ++I)
5921       Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
5922     Ops.push_back(getAlignmentValue32(PtrOp1));
5923     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
5924     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5925     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5926     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5927   }
5928   case NEON::BI__builtin_neon_vmovl_v: {
5929     llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
5930     Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
5931     if (Usgn)
5932       return Builder.CreateZExt(Ops[0], Ty, "vmovl");
5933     return Builder.CreateSExt(Ops[0], Ty, "vmovl");
5934   }
5935   case NEON::BI__builtin_neon_vmovn_v: {
5936     llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
5937     Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
5938     return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
5939   }
5940   case NEON::BI__builtin_neon_vmull_v:
5941     // FIXME: the integer vmull operations could be emitted in terms of pure
5942     // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
5943     // hoisting the exts outside loops. Until global ISel comes along that can
5944     // see through such movement this leads to bad CodeGen. So we need an
5945     // intrinsic for now.
5946     Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
5947     Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
5948     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
5949   case NEON::BI__builtin_neon_vpadal_v:
5950   case NEON::BI__builtin_neon_vpadalq_v: {
5951     // The source operand type has twice as many elements of half the size.
5952     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
5953     llvm::Type *EltTy =
5954       llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
5955     auto *NarrowTy =
5956         llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
5957     llvm::Type *Tys[2] = { Ty, NarrowTy };
5958     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
5959   }
5960   case NEON::BI__builtin_neon_vpaddl_v:
5961   case NEON::BI__builtin_neon_vpaddlq_v: {
5962     // The source operand type has twice as many elements of half the size.
5963     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
5964     llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
5965     auto *NarrowTy =
5966         llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
5967     llvm::Type *Tys[2] = { Ty, NarrowTy };
5968     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
5969   }
5970   case NEON::BI__builtin_neon_vqdmlal_v:
5971   case NEON::BI__builtin_neon_vqdmlsl_v: {
5972     SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
5973     Ops[1] =
5974         EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal");
5975     Ops.resize(2);
5976     return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
5977   }
5978   case NEON::BI__builtin_neon_vqdmulhq_lane_v:
5979   case NEON::BI__builtin_neon_vqdmulh_lane_v:
5980   case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
5981   case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
5982     auto *RTy = cast<llvm::VectorType>(Ty);
5983     if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
5984         BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
5985       RTy = llvm::FixedVectorType::get(RTy->getElementType(),
5986                                        RTy->getNumElements() * 2);
5987     llvm::Type *Tys[2] = {
5988         RTy, GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
5989                                              /*isQuad*/ false))};
5990     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
5991   }
5992   case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
5993   case NEON::BI__builtin_neon_vqdmulh_laneq_v:
5994   case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
5995   case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
5996     llvm::Type *Tys[2] = {
5997         Ty, GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
5998                                             /*isQuad*/ true))};
5999     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
6000   }
6001   case NEON::BI__builtin_neon_vqshl_n_v:
6002   case NEON::BI__builtin_neon_vqshlq_n_v:
6003     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
6004                         1, false);
6005   case NEON::BI__builtin_neon_vqshlu_n_v:
6006   case NEON::BI__builtin_neon_vqshluq_n_v:
6007     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n",
6008                         1, false);
6009   case NEON::BI__builtin_neon_vrecpe_v:
6010   case NEON::BI__builtin_neon_vrecpeq_v:
6011   case NEON::BI__builtin_neon_vrsqrte_v:
6012   case NEON::BI__builtin_neon_vrsqrteq_v:
6013     Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
6014     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
6015   case NEON::BI__builtin_neon_vrndi_v:
6016   case NEON::BI__builtin_neon_vrndiq_v:
6017     Int = Builder.getIsFPConstrained()
6018               ? Intrinsic::experimental_constrained_nearbyint
6019               : Intrinsic::nearbyint;
6020     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
6021   case NEON::BI__builtin_neon_vrshr_n_v:
6022   case NEON::BI__builtin_neon_vrshrq_n_v:
6023     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n",
6024                         1, true);
6025   case NEON::BI__builtin_neon_vshl_n_v:
6026   case NEON::BI__builtin_neon_vshlq_n_v:
6027     Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
6028     return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
6029                              "vshl_n");
6030   case NEON::BI__builtin_neon_vshll_n_v: {
6031     llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
6032     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
6033     if (Usgn)
6034       Ops[0] = Builder.CreateZExt(Ops[0], VTy);
6035     else
6036       Ops[0] = Builder.CreateSExt(Ops[0], VTy);
6037     Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false);
6038     return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
6039   }
6040   case NEON::BI__builtin_neon_vshrn_n_v: {
6041     llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
6042     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
6043     Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false);
6044     if (Usgn)
6045       Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
6046     else
6047       Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
6048     return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
6049   }
6050   case NEON::BI__builtin_neon_vshr_n_v:
6051   case NEON::BI__builtin_neon_vshrq_n_v:
6052     return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n");
6053   case NEON::BI__builtin_neon_vst1_v:
6054   case NEON::BI__builtin_neon_vst1q_v:
6055   case NEON::BI__builtin_neon_vst2_v:
6056   case NEON::BI__builtin_neon_vst2q_v:
6057   case NEON::BI__builtin_neon_vst3_v:
6058   case NEON::BI__builtin_neon_vst3q_v:
6059   case NEON::BI__builtin_neon_vst4_v:
6060   case NEON::BI__builtin_neon_vst4q_v:
6061   case NEON::BI__builtin_neon_vst2_lane_v:
6062   case NEON::BI__builtin_neon_vst2q_lane_v:
6063   case NEON::BI__builtin_neon_vst3_lane_v:
6064   case NEON::BI__builtin_neon_vst3q_lane_v:
6065   case NEON::BI__builtin_neon_vst4_lane_v:
6066   case NEON::BI__builtin_neon_vst4q_lane_v: {
6067     llvm::Type *Tys[] = {Int8PtrTy, Ty};
6068     Ops.push_back(getAlignmentValue32(PtrOp0));
6069     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
6070   }
6071   case NEON::BI__builtin_neon_vst1_x2_v:
6072   case NEON::BI__builtin_neon_vst1q_x2_v:
6073   case NEON::BI__builtin_neon_vst1_x3_v:
6074   case NEON::BI__builtin_neon_vst1q_x3_v:
6075   case NEON::BI__builtin_neon_vst1_x4_v:
6076   case NEON::BI__builtin_neon_vst1q_x4_v: {
6077     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getElementType());
6078     // TODO: Currently in AArch32 mode the pointer operand comes first, whereas
6079     // in AArch64 it comes last. We may want to stick to one or another.
6080     if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
6081         Arch == llvm::Triple::aarch64_32) {
6082       llvm::Type *Tys[2] = { VTy, PTy };
6083       std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6084       return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "");
6085     }
6086     llvm::Type *Tys[2] = { PTy, VTy };
6087     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "");
6088   }
6089   case NEON::BI__builtin_neon_vsubhn_v: {
6090     llvm::VectorType *SrcTy =
6091         llvm::VectorType::getExtendedElementVectorType(VTy);
6092
6093     // %sum = add <4 x i32> %lhs, %rhs
6094     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
6095     Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
6096     Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
6097
6098     // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
6099     Constant *ShiftAmt =
6100         ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
6101     Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
6102
6103     // %res = trunc <4 x i32> %high to <4 x i16>
6104     return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
6105   }
6106   case NEON::BI__builtin_neon_vtrn_v:
6107   case NEON::BI__builtin_neon_vtrnq_v: {
6108     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
6109     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6110     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6111     Value *SV = nullptr;
6112
6113     for (unsigned vi = 0; vi != 2; ++vi) {
6114       SmallVector<int, 16> Indices;
6115       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6116         Indices.push_back(i+vi);
6117         Indices.push_back(i+e+vi);
6118       }
6119       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6120       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
6121       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6122     }
6123     return SV;
6124   }
6125   case NEON::BI__builtin_neon_vtst_v:
6126   case NEON::BI__builtin_neon_vtstq_v: {
6127     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6128     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6129     Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
6130     Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
6131                                 ConstantAggregateZero::get(Ty));
6132     return Builder.CreateSExt(Ops[0], Ty, "vtst");
6133   }
6134   case NEON::BI__builtin_neon_vuzp_v:
6135   case NEON::BI__builtin_neon_vuzpq_v: {
6136     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
6137     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6138     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6139     Value *SV = nullptr;
6140
6141     for (unsigned vi = 0; vi != 2; ++vi) {
6142       SmallVector<int, 16> Indices;
6143       for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
6144         Indices.push_back(2*i+vi);
6145
6146       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6147       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
6148       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6149     }
6150     return SV;
6151   }
6152   case NEON::BI__builtin_neon_vzip_v:
6153   case NEON::BI__builtin_neon_vzipq_v: {
6154     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
6155     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6156     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6157     Value *SV = nullptr;
6158
6159     for (unsigned vi = 0; vi != 2; ++vi) {
6160       SmallVector<int, 16> Indices;
6161       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6162         Indices.push_back((i + vi*e) >> 1);
6163         Indices.push_back(((i + vi*e) >> 1)+e);
6164       }
6165       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6166       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
6167       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6168     }
6169     return SV;
6170   }
6171   case NEON::BI__builtin_neon_vdot_v:
6172   case NEON::BI__builtin_neon_vdotq_v: {
6173     auto *InputTy =
6174         llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
6175     llvm::Type *Tys[2] = { Ty, InputTy };
6176     Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
6177     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot");
6178   }
6179   case NEON::BI__builtin_neon_vfmlal_low_v:
6180   case NEON::BI__builtin_neon_vfmlalq_low_v: {
6181     auto *InputTy =
6182         llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
6183     llvm::Type *Tys[2] = { Ty, InputTy };
6184     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_low");
6185   }
6186   case NEON::BI__builtin_neon_vfmlsl_low_v:
6187   case NEON::BI__builtin_neon_vfmlslq_low_v: {
6188     auto *InputTy =
6189         llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
6190     llvm::Type *Tys[2] = { Ty, InputTy };
6191     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_low");
6192   }
6193   case NEON::BI__builtin_neon_vfmlal_high_v:
6194   case NEON::BI__builtin_neon_vfmlalq_high_v: {
6195     auto *InputTy =
6196         llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
6197     llvm::Type *Tys[2] = { Ty, InputTy };
6198     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_high");
6199   }
6200   case NEON::BI__builtin_neon_vfmlsl_high_v:
6201   case NEON::BI__builtin_neon_vfmlslq_high_v: {
6202     auto *InputTy =
6203         llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
6204     llvm::Type *Tys[2] = { Ty, InputTy };
6205     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_high");
6206   }
6207   case NEON::BI__builtin_neon_vmmlaq_v: {
6208     auto *InputTy =
6209         llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
6210     llvm::Type *Tys[2] = { Ty, InputTy };
6211     Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
6212     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmmla");
6213   }
6214   case NEON::BI__builtin_neon_vusmmlaq_v: {
6215     auto *InputTy =
6216         llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
6217     llvm::Type *Tys[2] = { Ty, InputTy };
6218     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusmmla");
6219   }
6220   case NEON::BI__builtin_neon_vusdot_v:
6221   case NEON::BI__builtin_neon_vusdotq_v: {
6222     auto *InputTy =
6223         llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
6224     llvm::Type *Tys[2] = { Ty, InputTy };
6225     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusdot");
6226   }
6227   case NEON::BI__builtin_neon_vbfdot_v:
6228   case NEON::BI__builtin_neon_vbfdotq_v: {
6229     llvm::Type *InputTy =
6230         llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
6231     llvm::Type *Tys[2] = { Ty, InputTy };
6232     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfdot");
6233   }
6234   case NEON::BI__builtin_neon_vbfmmlaq_v: {
6235     llvm::Type *InputTy =
6236         llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
6237     llvm::Type *Tys[2] = { Ty, InputTy };
6238     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfmmla");
6239   }
6240   case NEON::BI__builtin_neon_vbfmlalbq_v: {
6241     llvm::Type *InputTy =
6242         llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
6243     llvm::Type *Tys[2] = { Ty, InputTy };
6244     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfmlalb");
6245   }
6246   case NEON::BI__builtin_neon_vbfmlaltq_v: {
6247     llvm::Type *InputTy =
6248         llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
6249     llvm::Type *Tys[2] = { Ty, InputTy };
6250     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfmlalt");
6251   }
6252   case NEON::BI__builtin_neon___a32_vcvt_bf16_v: {
6253     llvm::Type *Tys[1] = { Ty };
6254     Function *F = CGM.getIntrinsic(Int, Tys);
6255     return EmitNeonCall(F, Ops, "vcvtfp2bf");
6256   }
6257
6258   }
6259
6260   assert(Int && "Expected valid intrinsic number");
6261
6262   // Determine the type(s) of this overloaded AArch64 intrinsic.
6263   Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
6264
6265   Value *Result = EmitNeonCall(F, Ops, NameHint);
6266   llvm::Type *ResultType = ConvertType(E->getType());
6267   // AArch64 intrinsic one-element vector type cast to
6268   // scalar type expected by the builtin
6269   return Builder.CreateBitCast(Result, ResultType, NameHint);
6270 }
6271
6272 Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr(
6273     Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp,
6274     const CmpInst::Predicate Ip, const Twine &Name) {
6275   llvm::Type *OTy = Op->getType();
6276
6277   // FIXME: this is utterly horrific. We should not be looking at previous
6278   // codegen context to find out what needs doing. Unfortunately TableGen
6279   // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32
6280   // (etc).
6281   if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
6282     OTy = BI->getOperand(0)->getType();
6283
6284   Op = Builder.CreateBitCast(Op, OTy);
6285   if (OTy->getScalarType()->isFloatingPointTy()) {
6286     Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
6287   } else {
6288     Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
6289   }
6290   return Builder.CreateSExt(Op, Ty, Name);
6291 }
6292
6293 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
6294                                  Value *ExtOp, Value *IndexOp,
6295                                  llvm::Type *ResTy, unsigned IntID,
6296                                  const char *Name) {
6297   SmallVector<Value *, 2> TblOps;
6298   if (ExtOp)
6299     TblOps.push_back(ExtOp);
6300
6301   // Build a vector containing sequential number like (0, 1, 2, ..., 15)
6302   SmallVector<int, 16> Indices;
6303   llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
6304   for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
6305     Indices.push_back(2*i);
6306     Indices.push_back(2*i+1);
6307   }
6308
6309   int PairPos = 0, End = Ops.size() - 1;
6310   while (PairPos < End) {
6311     TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
6312                                                      Ops[PairPos+1], Indices,
6313                                                      Name));
6314     PairPos += 2;
6315   }
6316
6317   // If there's an odd number of 64-bit lookup table, fill the high 64-bit
6318   // of the 128-bit lookup table with zero.
6319   if (PairPos == End) {
6320     Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
6321     TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
6322                                                      ZeroTbl, Indices, Name));
6323   }
6324
6325   Function *TblF;
6326   TblOps.push_back(IndexOp);
6327   TblF = CGF.CGM.getIntrinsic(IntID, ResTy);
6328
6329   return CGF.EmitNeonCall(TblF, TblOps, Name);
6330 }
6331
6332 Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
6333   unsigned Value;
6334   switch (BuiltinID) {
6335   default:
6336     return nullptr;
6337   case ARM::BI__builtin_arm_nop:
6338     Value = 0;
6339     break;
6340   case ARM::BI__builtin_arm_yield:
6341   case ARM::BI__yield:
6342     Value = 1;
6343     break;
6344   case ARM::BI__builtin_arm_wfe:
6345   case ARM::BI__wfe:
6346     Value = 2;
6347     break;
6348   case ARM::BI__builtin_arm_wfi:
6349   case ARM::BI__wfi:
6350     Value = 3;
6351     break;
6352   case ARM::BI__builtin_arm_sev:
6353   case ARM::BI__sev:
6354     Value = 4;
6355     break;
6356   case ARM::BI__builtin_arm_sevl:
6357   case ARM::BI__sevl:
6358     Value = 5;
6359     break;
6360   }
6361
6362   return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
6363                             llvm::ConstantInt::get(Int32Ty, Value));
6364 }
6365
6366 enum SpecialRegisterAccessKind {
6367   NormalRead,
6368   VolatileRead,
6369   Write,
6370 };
6371
6372 // Generates the IR for the read/write special register builtin,
6373 // ValueType is the type of the value that is to be written or read,
6374 // RegisterType is the type of the register being written to or read from.
6375 static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF,
6376                                          const CallExpr *E,
6377                                          llvm::Type *RegisterType,
6378                                          llvm::Type *ValueType,
6379                                          SpecialRegisterAccessKind AccessKind,
6380                                          StringRef SysReg = "") {
6381   // write and register intrinsics only support 32 and 64 bit operations.
6382   assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))
6383           && "Unsupported size for register.");
6384
6385   CodeGen::CGBuilderTy &Builder = CGF.Builder;
6386   CodeGen::CodeGenModule &CGM = CGF.CGM;
6387   LLVMContext &Context = CGM.getLLVMContext();
6388
6389   if (SysReg.empty()) {
6390     const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts();
6391     SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
6392   }
6393
6394   llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
6395   llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
6396   llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
6397
6398   llvm::Type *Types[] = { RegisterType };
6399
6400   bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
6401   assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
6402             && "Can't fit 64-bit value in 32-bit register");
6403
6404   if (AccessKind != Write) {
6405     assert(AccessKind == NormalRead || AccessKind == VolatileRead);
6406     llvm::Function *F = CGM.getIntrinsic(
6407         AccessKind == VolatileRead ? llvm::Intrinsic::read_volatile_register
6408                                    : llvm::Intrinsic::read_register,
6409         Types);
6410     llvm::Value *Call = Builder.CreateCall(F, Metadata);
6411
6412     if (MixedTypes)
6413       // Read into 64 bit register and then truncate result to 32 bit.
6414       return Builder.CreateTrunc(Call, ValueType);
6415
6416     if (ValueType->isPointerTy())
6417       // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*).
6418       return Builder.CreateIntToPtr(Call, ValueType);
6419
6420     return Call;
6421   }
6422
6423   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
6424   llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1));
6425   if (MixedTypes) {
6426     // Extend 32 bit write value to 64 bit to pass to write.
6427     ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
6428     return Builder.CreateCall(F, { Metadata, ArgValue });
6429   }
6430
6431   if (ValueType->isPointerTy()) {
6432     // Have VoidPtrTy ArgValue but want to return an i32/i64.
6433     ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
6434     return Builder.CreateCall(F, { Metadata, ArgValue });
6435   }
6436
6437   return Builder.CreateCall(F, { Metadata, ArgValue });
6438 }
6439
6440 /// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
6441 /// argument that specifies the vector type.
6442 static bool HasExtraNeonArgument(unsigned BuiltinID) {
6443   switch (BuiltinID) {
6444   default: break;
6445   case NEON::BI__builtin_neon_vget_lane_i8:
6446   case NEON::BI__builtin_neon_vget_lane_i16:
6447   case NEON::BI__builtin_neon_vget_lane_bf16:
6448   case NEON::BI__builtin_neon_vget_lane_i32:
6449   case NEON::BI__builtin_neon_vget_lane_i64:
6450   case NEON::BI__builtin_neon_vget_lane_f32:
6451   case NEON::BI__builtin_neon_vgetq_lane_i8:
6452   case NEON::BI__builtin_neon_vgetq_lane_i16:
6453   case NEON::BI__builtin_neon_vgetq_lane_bf16:
6454   case NEON::BI__builtin_neon_vgetq_lane_i32:
6455   case NEON::BI__builtin_neon_vgetq_lane_i64:
6456   case NEON::BI__builtin_neon_vgetq_lane_f32:
6457   case NEON::BI__builtin_neon_vduph_lane_bf16:
6458   case NEON::BI__builtin_neon_vduph_laneq_bf16:
6459   case NEON::BI__builtin_neon_vset_lane_i8:
6460   case NEON::BI__builtin_neon_vset_lane_i16:
6461   case NEON::BI__builtin_neon_vset_lane_bf16:
6462   case NEON::BI__builtin_neon_vset_lane_i32:
6463   case NEON::BI__builtin_neon_vset_lane_i64:
6464   case NEON::BI__builtin_neon_vset_lane_f32:
6465   case NEON::BI__builtin_neon_vsetq_lane_i8:
6466   case NEON::BI__builtin_neon_vsetq_lane_i16:
6467   case NEON::BI__builtin_neon_vsetq_lane_bf16:
6468   case NEON::BI__builtin_neon_vsetq_lane_i32:
6469   case NEON::BI__builtin_neon_vsetq_lane_i64:
6470   case NEON::BI__builtin_neon_vsetq_lane_f32:
6471   case NEON::BI__builtin_neon_vsha1h_u32:
6472   case NEON::BI__builtin_neon_vsha1cq_u32:
6473   case NEON::BI__builtin_neon_vsha1pq_u32:
6474   case NEON::BI__builtin_neon_vsha1mq_u32:
6475   case NEON::BI__builtin_neon_vcvth_bf16_f32:
6476   case clang::ARM::BI_MoveToCoprocessor:
6477   case clang::ARM::BI_MoveToCoprocessor2:
6478     return false;
6479   }
6480   return true;
6481 }
6482
6483 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
6484                                            const CallExpr *E,
6485                                            ReturnValueSlot ReturnValue,
6486                                            llvm::Triple::ArchType Arch) {
6487   if (auto Hint = GetValueForARMHint(BuiltinID))
6488     return Hint;
6489
6490   if (BuiltinID == ARM::BI__emit) {
6491     bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
6492     llvm::FunctionType *FTy =
6493         llvm::FunctionType::get(VoidTy, /*Variadic=*/false);
6494
6495     Expr::EvalResult Result;
6496     if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
6497       llvm_unreachable("Sema will ensure that the parameter is constant");
6498
6499     llvm::APSInt Value = Result.Val.getInt();
6500     uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
6501
6502     llvm::InlineAsm *Emit =
6503         IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "",
6504                                  /*hasSideEffects=*/true)
6505                 : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "",
6506                                  /*hasSideEffects=*/true);
6507
6508     return Builder.CreateCall(Emit);
6509   }
6510
6511   if (BuiltinID == ARM::BI__builtin_arm_dbg) {
6512     Value *Option = EmitScalarExpr(E->getArg(0));
6513     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
6514   }
6515
6516   if (BuiltinID == ARM::BI__builtin_arm_prefetch) {
6517     Value *Address = EmitScalarExpr(E->getArg(0));
6518     Value *RW      = EmitScalarExpr(E->getArg(1));
6519     Value *IsData  = EmitScalarExpr(E->getArg(2));
6520
6521     // Locality is not supported on ARM target
6522     Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
6523
6524     Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
6525     return Builder.CreateCall(F, {Address, RW, Locality, IsData});
6526   }
6527
6528   if (BuiltinID == ARM::BI__builtin_arm_rbit) {
6529     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
6530     return Builder.CreateCall(
6531         CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
6532   }
6533
6534   if (BuiltinID == ARM::BI__builtin_arm_cls) {
6535     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
6536     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls), Arg, "cls");
6537   }
6538   if (BuiltinID == ARM::BI__builtin_arm_cls64) {
6539     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
6540     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls64), Arg,
6541                               "cls");
6542   }
6543
6544   if (BuiltinID == ARM::BI__clear_cache) {
6545     assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
6546     const FunctionDecl *FD = E->getDirectCallee();
6547     Value *Ops[2];
6548     for (unsigned i = 0; i < 2; i++)
6549       Ops[i] = EmitScalarExpr(E->getArg(i));
6550     llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
6551     llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
6552     StringRef Name = FD->getName();
6553     return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
6554   }
6555
6556   if (BuiltinID == ARM::BI__builtin_arm_mcrr ||
6557       BuiltinID == ARM::BI__builtin_arm_mcrr2) {
6558     Function *F;
6559
6560     switch (BuiltinID) {
6561     default: llvm_unreachable("unexpected builtin");
6562     case ARM::BI__builtin_arm_mcrr:
6563       F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
6564       break;
6565     case ARM::BI__builtin_arm_mcrr2:
6566       F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
6567       break;
6568     }
6569
6570     // MCRR{2} instruction has 5 operands but
6571     // the intrinsic has 4 because Rt and Rt2
6572     // are represented as a single unsigned 64
6573     // bit integer in the intrinsic definition
6574     // but internally it's represented as 2 32
6575     // bit integers.
6576
6577     Value *Coproc = EmitScalarExpr(E->getArg(0));
6578     Value *Opc1 = EmitScalarExpr(E->getArg(1));
6579     Value *RtAndRt2 = EmitScalarExpr(E->getArg(2));
6580     Value *CRm = EmitScalarExpr(E->getArg(3));
6581
6582     Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
6583     Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty);
6584     Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
6585     Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty);
6586
6587     return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
6588   }
6589
6590   if (BuiltinID == ARM::BI__builtin_arm_mrrc ||
6591       BuiltinID == ARM::BI__builtin_arm_mrrc2) {
6592     Function *F;
6593
6594     switch (BuiltinID) {
6595     default: llvm_unreachable("unexpected builtin");
6596     case ARM::BI__builtin_arm_mrrc:
6597       F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
6598       break;
6599     case ARM::BI__builtin_arm_mrrc2:
6600       F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
6601       break;
6602     }
6603
6604     Value *Coproc = EmitScalarExpr(E->getArg(0));
6605     Value *Opc1 = EmitScalarExpr(E->getArg(1));
6606     Value *CRm  = EmitScalarExpr(E->getArg(2));
6607     Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
6608
6609     // Returns an unsigned 64 bit integer, represented
6610     // as two 32 bit integers.
6611
6612     Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
6613     Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
6614     Rt = Builder.CreateZExt(Rt, Int64Ty);
6615     Rt1 = Builder.CreateZExt(Rt1, Int64Ty);
6616
6617     Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32);
6618     RtAndRt2 = Builder.CreateShl(Rt, ShiftCast, "shl", true);
6619     RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
6620
6621     return Builder.CreateBitCast(RtAndRt2, ConvertType(E->getType()));
6622   }
6623
6624   if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
6625       ((BuiltinID == ARM::BI__builtin_arm_ldrex ||
6626         BuiltinID == ARM::BI__builtin_arm_ldaex) &&
6627        getContext().getTypeSize(E->getType()) == 64) ||
6628       BuiltinID == ARM::BI__ldrexd) {
6629     Function *F;
6630
6631     switch (BuiltinID) {
6632     default: llvm_unreachable("unexpected builtin");
6633     case ARM::BI__builtin_arm_ldaex:
6634       F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
6635       break;
6636     case ARM::BI__builtin_arm_ldrexd:
6637     case ARM::BI__builtin_arm_ldrex:
6638     case ARM::BI__ldrexd:
6639       F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
6640       break;
6641     }
6642
6643     Value *LdPtr = EmitScalarExpr(E->getArg(0));
6644     Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
6645                                     "ldrexd");
6646
6647     Value *Val0 = Builder.CreateExtractValue(Val, 1);
6648     Value *Val1 = Builder.CreateExtractValue(Val, 0);
6649     Val0 = Builder.CreateZExt(Val0, Int64Ty);
6650     Val1 = Builder.CreateZExt(Val1, Int64Ty);
6651
6652     Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
6653     Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
6654     Val = Builder.CreateOr(Val, Val1);
6655     return Builder.CreateBitCast(Val, ConvertType(E->getType()));
6656   }
6657
6658   if (BuiltinID == ARM::BI__builtin_arm_ldrex ||
6659       BuiltinID == ARM::BI__builtin_arm_ldaex) {
6660     Value *LoadAddr = EmitScalarExpr(E->getArg(0));
6661
6662     QualType Ty = E->getType();
6663     llvm::Type *RealResTy = ConvertType(Ty);
6664     llvm::Type *PtrTy = llvm::IntegerType::get(
6665         getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
6666     LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
6667
6668     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex
6669                                        ? Intrinsic::arm_ldaex
6670                                        : Intrinsic::arm_ldrex,
6671                                    PtrTy);
6672     Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
6673
6674     if (RealResTy->isPointerTy())
6675       return Builder.CreateIntToPtr(Val, RealResTy);
6676     else {
6677       llvm::Type *IntResTy = llvm::IntegerType::get(
6678           getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
6679       Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
6680       return Builder.CreateBitCast(Val, RealResTy);
6681     }
6682   }
6683
6684   if (BuiltinID == ARM::BI__builtin_arm_strexd ||
6685       ((BuiltinID == ARM::BI__builtin_arm_stlex ||
6686         BuiltinID == ARM::BI__builtin_arm_strex) &&
6687        getContext().getTypeSize(E->getArg(0)->getType()) == 64)) {
6688     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
6689                                        ? Intrinsic::arm_stlexd
6690                                        : Intrinsic::arm_strexd);
6691     llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty);
6692
6693     Address Tmp = CreateMemTemp(E->getArg(0)->getType());
6694     Value *Val = EmitScalarExpr(E->getArg(0));
6695     Builder.CreateStore(Val, Tmp);
6696
6697     Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
6698     Val = Builder.CreateLoad(LdPtr);
6699
6700     Value *Arg0 = Builder.CreateExtractValue(Val, 0);
6701     Value *Arg1 = Builder.CreateExtractValue(Val, 1);
6702     Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy);
6703     return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd");
6704   }
6705
6706   if (BuiltinID == ARM::BI__builtin_arm_strex ||
6707       BuiltinID == ARM::BI__builtin_arm_stlex) {
6708     Value *StoreVal = EmitScalarExpr(E->getArg(0));
6709     Value *StoreAddr = EmitScalarExpr(E->getArg(1));
6710
6711     QualType Ty = E->getArg(0)->getType();
6712     llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
6713                                                  getContext().getTypeSize(Ty));
6714     StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
6715
6716     if (StoreVal->getType()->isPointerTy())
6717       StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
6718     else {
6719       llvm::Type *IntTy = llvm::IntegerType::get(
6720           getLLVMContext(),
6721           CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
6722       StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
6723       StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
6724     }
6725
6726     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
6727                                        ? Intrinsic::arm_stlex
6728                                        : Intrinsic::arm_strex,
6729                                    StoreAddr->getType());
6730     return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
6731   }
6732
6733   if (BuiltinID == ARM::BI__builtin_arm_clrex) {
6734     Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
6735     return Builder.CreateCall(F);
6736   }
6737
6738   // CRC32
6739   Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
6740   switch (BuiltinID) {
6741   case ARM::BI__builtin_arm_crc32b:
6742     CRCIntrinsicID = Intrinsic::arm_crc32b; break;
6743   case ARM::BI__builtin_arm_crc32cb:
6744     CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
6745   case ARM::BI__builtin_arm_crc32h:
6746     CRCIntrinsicID = Intrinsic::arm_crc32h; break;
6747   case ARM::BI__builtin_arm_crc32ch:
6748     CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
6749   case ARM::BI__builtin_arm_crc32w:
6750   case ARM::BI__builtin_arm_crc32d:
6751     CRCIntrinsicID = Intrinsic::arm_crc32w; break;
6752   case ARM::BI__builtin_arm_crc32cw:
6753   case ARM::BI__builtin_arm_crc32cd:
6754     CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
6755   }
6756
6757   if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
6758     Value *Arg0 = EmitScalarExpr(E->getArg(0));
6759     Value *Arg1 = EmitScalarExpr(E->getArg(1));
6760
6761     // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w
6762     // intrinsics, hence we need different codegen for these cases.
6763     if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
6764         BuiltinID == ARM::BI__builtin_arm_crc32cd) {
6765       Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
6766       Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
6767       Value *Arg1b = Builder.CreateLShr(Arg1, C1);
6768       Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
6769
6770       Function *F = CGM.getIntrinsic(CRCIntrinsicID);
6771       Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
6772       return Builder.CreateCall(F, {Res, Arg1b});
6773     } else {
6774       Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
6775
6776       Function *F = CGM.getIntrinsic(CRCIntrinsicID);
6777       return Builder.CreateCall(F, {Arg0, Arg1});
6778     }
6779   }
6780
6781   if (BuiltinID == ARM::BI__builtin_arm_rsr ||
6782       BuiltinID == ARM::BI__builtin_arm_rsr64 ||
6783       BuiltinID == ARM::BI__builtin_arm_rsrp ||
6784       BuiltinID == ARM::BI__builtin_arm_wsr ||
6785       BuiltinID == ARM::BI__builtin_arm_wsr64 ||
6786       BuiltinID == ARM::BI__builtin_arm_wsrp) {
6787
6788     SpecialRegisterAccessKind AccessKind = Write;
6789     if (BuiltinID == ARM::BI__builtin_arm_rsr ||
6790         BuiltinID == ARM::BI__builtin_arm_rsr64 ||
6791         BuiltinID == ARM::BI__builtin_arm_rsrp)
6792       AccessKind = VolatileRead;
6793
6794     bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp ||
6795                             BuiltinID == ARM::BI__builtin_arm_wsrp;
6796
6797     bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 ||
6798                    BuiltinID == ARM::BI__builtin_arm_wsr64;
6799
6800     llvm::Type *ValueType;
6801     llvm::Type *RegisterType;
6802     if (IsPointerBuiltin) {
6803       ValueType = VoidPtrTy;
6804       RegisterType = Int32Ty;
6805     } else if (Is64Bit) {
6806       ValueType = RegisterType = Int64Ty;
6807     } else {
6808       ValueType = RegisterType = Int32Ty;
6809     }
6810
6811     return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType,
6812                                       AccessKind);
6813   }
6814
6815   // Deal with MVE builtins
6816   if (Value *Result = EmitARMMVEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
6817     return Result;
6818   // Handle CDE builtins
6819   if (Value *Result = EmitARMCDEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
6820     return Result;
6821
6822   // Find out if any arguments are required to be integer constant
6823   // expressions.
6824   unsigned ICEArguments = 0;
6825   ASTContext::GetBuiltinTypeError Error;
6826   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
6827   assert(Error == ASTContext::GE_None && "Should not codegen an error");
6828
6829   auto getAlignmentValue32 = [&](Address addr) -> Value* {
6830     return Builder.getInt32(addr.getAlignment().getQuantity());
6831   };
6832
6833   Address PtrOp0 = Address::invalid();
6834   Address PtrOp1 = Address::invalid();
6835   SmallVector<Value*, 4> Ops;
6836   bool HasExtraArg = HasExtraNeonArgument(BuiltinID);
6837   unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
6838   for (unsigned i = 0, e = NumArgs; i != e; i++) {
6839     if (i == 0) {
6840       switch (BuiltinID) {
6841       case NEON::BI__builtin_neon_vld1_v:
6842       case NEON::BI__builtin_neon_vld1q_v:
6843       case NEON::BI__builtin_neon_vld1q_lane_v:
6844       case NEON::BI__builtin_neon_vld1_lane_v:
6845       case NEON::BI__builtin_neon_vld1_dup_v:
6846       case NEON::BI__builtin_neon_vld1q_dup_v:
6847       case NEON::BI__builtin_neon_vst1_v:
6848       case NEON::BI__builtin_neon_vst1q_v:
6849       case NEON::BI__builtin_neon_vst1q_lane_v:
6850       case NEON::BI__builtin_neon_vst1_lane_v:
6851       case NEON::BI__builtin_neon_vst2_v:
6852       case NEON::BI__builtin_neon_vst2q_v:
6853       case NEON::BI__builtin_neon_vst2_lane_v:
6854       case NEON::BI__builtin_neon_vst2q_lane_v:
6855       case NEON::BI__builtin_neon_vst3_v:
6856       case NEON::BI__builtin_neon_vst3q_v:
6857       case NEON::BI__builtin_neon_vst3_lane_v:
6858       case NEON::BI__builtin_neon_vst3q_lane_v:
6859       case NEON::BI__builtin_neon_vst4_v:
6860       case NEON::BI__builtin_neon_vst4q_v:
6861       case NEON::BI__builtin_neon_vst4_lane_v:
6862       case NEON::BI__builtin_neon_vst4q_lane_v:
6863         // Get the alignment for the argument in addition to the value;
6864         // we'll use it later.
6865         PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
6866         Ops.push_back(PtrOp0.getPointer());
6867         continue;
6868       }
6869     }
6870     if (i == 1) {
6871       switch (BuiltinID) {
6872       case NEON::BI__builtin_neon_vld2_v:
6873       case NEON::BI__builtin_neon_vld2q_v:
6874       case NEON::BI__builtin_neon_vld3_v:
6875       case NEON::BI__builtin_neon_vld3q_v:
6876       case NEON::BI__builtin_neon_vld4_v:
6877       case NEON::BI__builtin_neon_vld4q_v:
6878       case NEON::BI__builtin_neon_vld2_lane_v:
6879       case NEON::BI__builtin_neon_vld2q_lane_v:
6880       case NEON::BI__builtin_neon_vld3_lane_v:
6881       case NEON::BI__builtin_neon_vld3q_lane_v:
6882       case NEON::BI__builtin_neon_vld4_lane_v:
6883       case NEON::BI__builtin_neon_vld4q_lane_v:
6884       case NEON::BI__builtin_neon_vld2_dup_v:
6885       case NEON::BI__builtin_neon_vld2q_dup_v:
6886       case NEON::BI__builtin_neon_vld3_dup_v:
6887       case NEON::BI__builtin_neon_vld3q_dup_v:
6888       case NEON::BI__builtin_neon_vld4_dup_v:
6889       case NEON::BI__builtin_neon_vld4q_dup_v:
6890         // Get the alignment for the argument in addition to the value;
6891         // we'll use it later.
6892         PtrOp1 = EmitPointerWithAlignment(E->getArg(1));
6893         Ops.push_back(PtrOp1.getPointer());
6894         continue;
6895       }
6896     }
6897
6898     if ((ICEArguments & (1 << i)) == 0) {
6899       Ops.push_back(EmitScalarExpr(E->getArg(i)));
6900     } else {
6901       // If this is required to be a constant, constant fold it so that we know
6902       // that the generated intrinsic gets a ConstantInt.
6903       llvm::APSInt Result;
6904       bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
6905       assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
6906       Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
6907     }
6908   }
6909
6910   switch (BuiltinID) {
6911   default: break;
6912
6913   case NEON::BI__builtin_neon_vget_lane_i8:
6914   case NEON::BI__builtin_neon_vget_lane_i16:
6915   case NEON::BI__builtin_neon_vget_lane_i32:
6916   case NEON::BI__builtin_neon_vget_lane_i64:
6917   case NEON::BI__builtin_neon_vget_lane_bf16:
6918   case NEON::BI__builtin_neon_vget_lane_f32:
6919   case NEON::BI__builtin_neon_vgetq_lane_i8:
6920   case NEON::BI__builtin_neon_vgetq_lane_i16:
6921   case NEON::BI__builtin_neon_vgetq_lane_i32:
6922   case NEON::BI__builtin_neon_vgetq_lane_i64:
6923   case NEON::BI__builtin_neon_vgetq_lane_bf16:
6924   case NEON::BI__builtin_neon_vgetq_lane_f32:
6925   case NEON::BI__builtin_neon_vduph_lane_bf16:
6926   case NEON::BI__builtin_neon_vduph_laneq_bf16:
6927     return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
6928
6929   case NEON::BI__builtin_neon_vrndns_f32: {
6930     Value *Arg = EmitScalarExpr(E->getArg(0));
6931     llvm::Type *Tys[] = {Arg->getType()};
6932     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vrintn, Tys);
6933     return Builder.CreateCall(F, {Arg}, "vrndn"); }
6934
6935   case NEON::BI__builtin_neon_vset_lane_i8:
6936   case NEON::BI__builtin_neon_vset_lane_i16:
6937   case NEON::BI__builtin_neon_vset_lane_i32:
6938   case NEON::BI__builtin_neon_vset_lane_i64:
6939   case NEON::BI__builtin_neon_vset_lane_bf16:
6940   case NEON::BI__builtin_neon_vset_lane_f32:
6941   case NEON::BI__builtin_neon_vsetq_lane_i8:
6942   case NEON::BI__builtin_neon_vsetq_lane_i16:
6943   case NEON::BI__builtin_neon_vsetq_lane_i32:
6944   case NEON::BI__builtin_neon_vsetq_lane_i64:
6945   case NEON::BI__builtin_neon_vsetq_lane_bf16:
6946   case NEON::BI__builtin_neon_vsetq_lane_f32:
6947     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
6948
6949   case NEON::BI__builtin_neon_vsha1h_u32:
6950     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
6951                         "vsha1h");
6952   case NEON::BI__builtin_neon_vsha1cq_u32:
6953     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
6954                         "vsha1h");
6955   case NEON::BI__builtin_neon_vsha1pq_u32:
6956     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
6957                         "vsha1h");
6958   case NEON::BI__builtin_neon_vsha1mq_u32:
6959     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
6960                         "vsha1h");
6961
6962   case NEON::BI__builtin_neon_vcvth_bf16_f32: {
6963     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops,
6964                         "vcvtbfp2bf");
6965   }
6966
6967   // The ARM _MoveToCoprocessor builtins put the input register value as
6968   // the first argument, but the LLVM intrinsic expects it as the third one.
6969   case ARM::BI_MoveToCoprocessor:
6970   case ARM::BI_MoveToCoprocessor2: {
6971     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ?
6972                                    Intrinsic::arm_mcr : Intrinsic::arm_mcr2);
6973     return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
6974                                   Ops[3], Ops[4], Ops[5]});
6975   }
6976   case ARM::BI_BitScanForward:
6977   case ARM::BI_BitScanForward64:
6978     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
6979   case ARM::BI_BitScanReverse:
6980   case ARM::BI_BitScanReverse64:
6981     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
6982
6983   case ARM::BI_InterlockedAnd64:
6984     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
6985   case ARM::BI_InterlockedExchange64:
6986     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
6987   case ARM::BI_InterlockedExchangeAdd64:
6988     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
6989   case ARM::BI_InterlockedExchangeSub64:
6990     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
6991   case ARM::BI_InterlockedOr64:
6992     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
6993   case ARM::BI_InterlockedXor64:
6994     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
6995   case ARM::BI_InterlockedDecrement64:
6996     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
6997   case ARM::BI_InterlockedIncrement64:
6998     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
6999   case ARM::BI_InterlockedExchangeAdd8_acq:
7000   case ARM::BI_InterlockedExchangeAdd16_acq:
7001   case ARM::BI_InterlockedExchangeAdd_acq:
7002   case ARM::BI_InterlockedExchangeAdd64_acq:
7003     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_acq, E);
7004   case ARM::BI_InterlockedExchangeAdd8_rel:
7005   case ARM::BI_InterlockedExchangeAdd16_rel:
7006   case ARM::BI_InterlockedExchangeAdd_rel:
7007   case ARM::BI_InterlockedExchangeAdd64_rel:
7008     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_rel, E);
7009   case ARM::BI_InterlockedExchangeAdd8_nf:
7010   case ARM::BI_InterlockedExchangeAdd16_nf:
7011   case ARM::BI_InterlockedExchangeAdd_nf:
7012   case ARM::BI_InterlockedExchangeAdd64_nf:
7013     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_nf, E);
7014   case ARM::BI_InterlockedExchange8_acq:
7015   case ARM::BI_InterlockedExchange16_acq:
7016   case ARM::BI_InterlockedExchange_acq:
7017   case ARM::BI_InterlockedExchange64_acq:
7018     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_acq, E);
7019   case ARM::BI_InterlockedExchange8_rel:
7020   case ARM::BI_InterlockedExchange16_rel:
7021   case ARM::BI_InterlockedExchange_rel:
7022   case ARM::BI_InterlockedExchange64_rel:
7023     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_rel, E);
7024   case ARM::BI_InterlockedExchange8_nf:
7025   case ARM::BI_InterlockedExchange16_nf:
7026   case ARM::BI_InterlockedExchange_nf:
7027   case ARM::BI_InterlockedExchange64_nf:
7028     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_nf, E);
7029   case ARM::BI_InterlockedCompareExchange8_acq:
7030   case ARM::BI_InterlockedCompareExchange16_acq:
7031   case ARM::BI_InterlockedCompareExchange_acq:
7032   case ARM::BI_InterlockedCompareExchange64_acq:
7033     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_acq, E);
7034   case ARM::BI_InterlockedCompareExchange8_rel:
7035   case ARM::BI_InterlockedCompareExchange16_rel:
7036   case ARM::BI_InterlockedCompareExchange_rel:
7037   case ARM::BI_InterlockedCompareExchange64_rel:
7038     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_rel, E);
7039   case ARM::BI_InterlockedCompareExchange8_nf:
7040   case ARM::BI_InterlockedCompareExchange16_nf:
7041   case ARM::BI_InterlockedCompareExchange_nf:
7042   case ARM::BI_InterlockedCompareExchange64_nf:
7043     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E);
7044   case ARM::BI_InterlockedOr8_acq:
7045   case ARM::BI_InterlockedOr16_acq:
7046   case ARM::BI_InterlockedOr_acq:
7047   case ARM::BI_InterlockedOr64_acq:
7048     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_acq, E);
7049   case ARM::BI_InterlockedOr8_rel:
7050   case ARM::BI_InterlockedOr16_rel:
7051   case ARM::BI_InterlockedOr_rel:
7052   case ARM::BI_InterlockedOr64_rel:
7053     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_rel, E);
7054   case ARM::BI_InterlockedOr8_nf:
7055   case ARM::BI_InterlockedOr16_nf:
7056   case ARM::BI_InterlockedOr_nf:
7057   case ARM::BI_InterlockedOr64_nf:
7058     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_nf, E);
7059   case ARM::BI_InterlockedXor8_acq:
7060   case ARM::BI_InterlockedXor16_acq:
7061   case ARM::BI_InterlockedXor_acq:
7062   case ARM::BI_InterlockedXor64_acq:
7063     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_acq, E);
7064   case ARM::BI_InterlockedXor8_rel:
7065   case ARM::BI_InterlockedXor16_rel:
7066   case ARM::BI_InterlockedXor_rel:
7067   case ARM::BI_InterlockedXor64_rel:
7068     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_rel, E);
7069   case ARM::BI_InterlockedXor8_nf:
7070   case ARM::BI_InterlockedXor16_nf:
7071   case ARM::BI_InterlockedXor_nf:
7072   case ARM::BI_InterlockedXor64_nf:
7073     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_nf, E);
7074   case ARM::BI_InterlockedAnd8_acq:
7075   case ARM::BI_InterlockedAnd16_acq:
7076   case ARM::BI_InterlockedAnd_acq:
7077   case ARM::BI_InterlockedAnd64_acq:
7078     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_acq, E);
7079   case ARM::BI_InterlockedAnd8_rel:
7080   case ARM::BI_InterlockedAnd16_rel:
7081   case ARM::BI_InterlockedAnd_rel:
7082   case ARM::BI_InterlockedAnd64_rel:
7083     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_rel, E);
7084   case ARM::BI_InterlockedAnd8_nf:
7085   case ARM::BI_InterlockedAnd16_nf:
7086   case ARM::BI_InterlockedAnd_nf:
7087   case ARM::BI_InterlockedAnd64_nf:
7088     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_nf, E);
7089   case ARM::BI_InterlockedIncrement16_acq:
7090   case ARM::BI_InterlockedIncrement_acq:
7091   case ARM::BI_InterlockedIncrement64_acq:
7092     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_acq, E);
7093   case ARM::BI_InterlockedIncrement16_rel:
7094   case ARM::BI_InterlockedIncrement_rel:
7095   case ARM::BI_InterlockedIncrement64_rel:
7096     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_rel, E);
7097   case ARM::BI_InterlockedIncrement16_nf:
7098   case ARM::BI_InterlockedIncrement_nf:
7099   case ARM::BI_InterlockedIncrement64_nf:
7100     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_nf, E);
7101   case ARM::BI_InterlockedDecrement16_acq:
7102   case ARM::BI_InterlockedDecrement_acq:
7103   case ARM::BI_InterlockedDecrement64_acq:
7104     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_acq, E);
7105   case ARM::BI_InterlockedDecrement16_rel:
7106   case ARM::BI_InterlockedDecrement_rel:
7107   case ARM::BI_InterlockedDecrement64_rel:
7108     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_rel, E);
7109   case ARM::BI_InterlockedDecrement16_nf:
7110   case ARM::BI_InterlockedDecrement_nf:
7111   case ARM::BI_InterlockedDecrement64_nf:
7112     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_nf, E);
7113   }
7114
7115   // Get the last argument, which specifies the vector type.
7116   assert(HasExtraArg);
7117   llvm::APSInt Result;
7118   const Expr *Arg = E->getArg(E->getNumArgs()-1);
7119   if (!Arg->isIntegerConstantExpr(Result, getContext()))
7120     return nullptr;
7121
7122   if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
7123       BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
7124     // Determine the overloaded type of this builtin.
7125     llvm::Type *Ty;
7126     if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
7127       Ty = FloatTy;
7128     else
7129       Ty = DoubleTy;
7130
7131     // Determine whether this is an unsigned conversion or not.
7132     bool usgn = Result.getZExtValue() == 1;
7133     unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
7134
7135     // Call the appropriate intrinsic.
7136     Function *F = CGM.getIntrinsic(Int, Ty);
7137     return Builder.CreateCall(F, Ops, "vcvtr");
7138   }
7139
7140   // Determine the type of this overloaded NEON intrinsic.
7141   NeonTypeFlags Type(Result.getZExtValue());
7142   bool usgn = Type.isUnsigned();
7143   bool rightShift = false;
7144
7145   llvm::VectorType *VTy = GetNeonType(this, Type,
7146                                       getTarget().hasLegalHalfType(),
7147                                       false,
7148                                       getTarget().hasBFloat16Type());
7149   llvm::Type *Ty = VTy;
7150   if (!Ty)
7151     return nullptr;
7152
7153   // Many NEON builtins have identical semantics and uses in ARM and
7154   // AArch64. Emit these in a single function.
7155   auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap);
7156   const ARMVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap(
7157       IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
7158   if (Builtin)
7159     return EmitCommonNeonBuiltinExpr(
7160         Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
7161         Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
7162
7163   unsigned Int;
7164   switch (BuiltinID) {
7165   default: return nullptr;
7166   case NEON::BI__builtin_neon_vld1q_lane_v:
7167     // Handle 64-bit integer elements as a special case.  Use shuffles of
7168     // one-element vectors to avoid poor code for i64 in the backend.
7169     if (VTy->getElementType()->isIntegerTy(64)) {
7170       // Extract the other lane.
7171       Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7172       int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
7173       Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
7174       Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
7175       // Load the value as a one-element vector.
7176       Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
7177       llvm::Type *Tys[] = {Ty, Int8PtrTy};
7178       Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
7179       Value *Align = getAlignmentValue32(PtrOp0);
7180       Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
7181       // Combine them.
7182       int Indices[] = {1 - Lane, Lane};
7183       return Builder.CreateShuffleVector(Ops[1], Ld, Indices, "vld1q_lane");
7184     }
7185     LLVM_FALLTHROUGH;
7186   case NEON::BI__builtin_neon_vld1_lane_v: {
7187     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7188     PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType());
7189     Value *Ld = Builder.CreateLoad(PtrOp0);
7190     return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
7191   }
7192   case NEON::BI__builtin_neon_vqrshrn_n_v:
7193     Int =
7194       usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
7195     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
7196                         1, true);
7197   case NEON::BI__builtin_neon_vqrshrun_n_v:
7198     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
7199                         Ops, "vqrshrun_n", 1, true);
7200   case NEON::BI__builtin_neon_vqshrn_n_v:
7201     Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
7202     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
7203                         1, true);
7204   case NEON::BI__builtin_neon_vqshrun_n_v:
7205     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
7206                         Ops, "vqshrun_n", 1, true);
7207   case NEON::BI__builtin_neon_vrecpe_v:
7208   case NEON::BI__builtin_neon_vrecpeq_v:
7209     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
7210                         Ops, "vrecpe");
7211   case NEON::BI__builtin_neon_vrshrn_n_v:
7212     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
7213                         Ops, "vrshrn_n", 1, true);
7214   case NEON::BI__builtin_neon_vrsra_n_v:
7215   case NEON::BI__builtin_neon_vrsraq_n_v:
7216     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7217     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7218     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
7219     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
7220     Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
7221     return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
7222   case NEON::BI__builtin_neon_vsri_n_v:
7223   case NEON::BI__builtin_neon_vsriq_n_v:
7224     rightShift = true;
7225     LLVM_FALLTHROUGH;
7226   case NEON::BI__builtin_neon_vsli_n_v:
7227   case NEON::BI__builtin_neon_vsliq_n_v:
7228     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
7229     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
7230                         Ops, "vsli_n");
7231   case NEON::BI__builtin_neon_vsra_n_v:
7232   case NEON::BI__builtin_neon_vsraq_n_v:
7233     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7234     Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
7235     return Builder.CreateAdd(Ops[0], Ops[1]);
7236   case NEON::BI__builtin_neon_vst1q_lane_v:
7237     // Handle 64-bit integer elements as a special case.  Use a shuffle to get
7238     // a one-element vector and avoid poor code for i64 in the backend.
7239     if (VTy->getElementType()->isIntegerTy(64)) {
7240       Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7241       Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
7242       Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
7243       Ops[2] = getAlignmentValue32(PtrOp0);
7244       llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
7245       return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
7246                                                  Tys), Ops);
7247     }
7248     LLVM_FALLTHROUGH;
7249   case NEON::BI__builtin_neon_vst1_lane_v: {
7250     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7251     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
7252     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
7253     auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty));
7254     return St;
7255   }
7256   case NEON::BI__builtin_neon_vtbl1_v:
7257     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
7258                         Ops, "vtbl1");
7259   case NEON::BI__builtin_neon_vtbl2_v:
7260     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
7261                         Ops, "vtbl2");
7262   case NEON::BI__builtin_neon_vtbl3_v:
7263     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
7264                         Ops, "vtbl3");
7265   case NEON::BI__builtin_neon_vtbl4_v:
7266     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
7267                         Ops, "vtbl4");
7268   case NEON::BI__builtin_neon_vtbx1_v:
7269     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
7270                         Ops, "vtbx1");
7271   case NEON::BI__builtin_neon_vtbx2_v:
7272     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
7273                         Ops, "vtbx2");
7274   case NEON::BI__builtin_neon_vtbx3_v:
7275     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
7276                         Ops, "vtbx3");
7277   case NEON::BI__builtin_neon_vtbx4_v:
7278     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
7279                         Ops, "vtbx4");
7280   }
7281 }
7282
7283 template<typename Integer>
7284 static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context) {
7285   llvm::APSInt IntVal;
7286   bool IsConst = E->isIntegerConstantExpr(IntVal, Context);
7287   assert(IsConst && "Sema should have checked this was a constant");
7288   (void)IsConst;
7289   return IntVal.getExtValue();
7290 }
7291
7292 static llvm::Value *SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V,
7293                                      llvm::Type *T, bool Unsigned) {
7294   // Helper function called by Tablegen-constructed ARM MVE builtin codegen,
7295   // which finds it convenient to specify signed/unsigned as a boolean flag.
7296   return Unsigned ? Builder.CreateZExt(V, T) : Builder.CreateSExt(V, T);
7297 }
7298
7299 static llvm::Value *MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V,
7300                                     uint32_t Shift, bool Unsigned) {
7301   // MVE helper function for integer shift right. This must handle signed vs
7302   // unsigned, and also deal specially with the case where the shift count is
7303   // equal to the lane size. In LLVM IR, an LShr with that parameter would be
7304   // undefined behavior, but in MVE it's legal, so we must convert it to code
7305   // that is not undefined in IR.
7306   unsigned LaneBits = cast<llvm::VectorType>(V->getType())
7307                           ->getElementType()
7308                           ->getPrimitiveSizeInBits();
7309   if (Shift == LaneBits) {
7310     // An unsigned shift of the full lane size always generates zero, so we can
7311     // simply emit a zero vector. A signed shift of the full lane size does the
7312     // same thing as shifting by one bit fewer.
7313     if (Unsigned)
7314       return llvm::Constant::getNullValue(V->getType());
7315     else
7316       --Shift;
7317   }
7318   return Unsigned ? Builder.CreateLShr(V, Shift) : Builder.CreateAShr(V, Shift);
7319 }
7320
7321 static llvm::Value *ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V) {
7322   // MVE-specific helper function for a vector splat, which infers the element
7323   // count of the output vector by knowing that MVE vectors are all 128 bits
7324   // wide.
7325   unsigned Elements = 128 / V->getType()->getPrimitiveSizeInBits();
7326   return Builder.CreateVectorSplat(Elements, V);
7327 }
7328
7329 static llvm::Value *ARMMVEVectorReinterpret(CGBuilderTy &Builder,
7330                                             CodeGenFunction *CGF,
7331                                             llvm::Value *V,
7332                                             llvm::Type *DestType) {
7333   // Convert one MVE vector type into another by reinterpreting its in-register
7334   // format.
7335   //
7336   // Little-endian, this is identical to a bitcast (which reinterprets the
7337   // memory format). But big-endian, they're not necessarily the same, because
7338   // the register and memory formats map to each other differently depending on
7339   // the lane size.
7340   //
7341   // We generate a bitcast whenever we can (if we're little-endian, or if the
7342   // lane sizes are the same anyway). Otherwise we fall back to an IR intrinsic
7343   // that performs the different kind of reinterpretation.
7344   if (CGF->getTarget().isBigEndian() &&
7345       V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
7346     return Builder.CreateCall(
7347         CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vreinterpretq,
7348                               {DestType, V->getType()}),
7349         V);
7350   } else {
7351     return Builder.CreateBitCast(V, DestType);
7352   }
7353 }
7354
7355 static llvm::Value *VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd) {
7356   // Make a shufflevector that extracts every other element of a vector (evens
7357   // or odds, as desired).
7358   SmallVector<int, 16> Indices;
7359   unsigned InputElements =
7360       cast<llvm::VectorType>(V->getType())->getNumElements();
7361   for (unsigned i = 0; i < InputElements; i += 2)
7362     Indices.push_back(i + Odd);
7363   return Builder.CreateShuffleVector(V, llvm::UndefValue::get(V->getType()),
7364                                      Indices);
7365 }
7366
7367 static llvm::Value *VectorZip(CGBuilderTy &Builder, llvm::Value *V0,
7368                               llvm::Value *V1) {
7369   // Make a shufflevector that interleaves two vectors element by element.
7370   assert(V0->getType() == V1->getType() && "Can't zip different vector types");
7371   SmallVector<int, 16> Indices;
7372   unsigned InputElements =
7373       cast<llvm::VectorType>(V0->getType())->getNumElements();
7374   for (unsigned i = 0; i < InputElements; i++) {
7375     Indices.push_back(i);
7376     Indices.push_back(i + InputElements);
7377   }
7378   return Builder.CreateShuffleVector(V0, V1, Indices);
7379 }
7380
7381 template<unsigned HighBit, unsigned OtherBits>
7382 static llvm::Value *ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT) {
7383   // MVE-specific helper function to make a vector splat of a constant such as
7384   // UINT_MAX or INT_MIN, in which all bits below the highest one are equal.
7385   llvm::Type *T = cast<llvm::VectorType>(VT)->getElementType();
7386   unsigned LaneBits = T->getPrimitiveSizeInBits();
7387   uint32_t Value = HighBit << (LaneBits - 1);
7388   if (OtherBits)
7389     Value |= (1UL << (LaneBits - 1)) - 1;
7390   llvm::Value *Lane = llvm::ConstantInt::get(T, Value);
7391   return ARMMVEVectorSplat(Builder, Lane);
7392 }
7393
7394 static llvm::Value *ARMMVEVectorElementReverse(CGBuilderTy &Builder,
7395                                                llvm::Value *V,
7396                                                unsigned ReverseWidth) {
7397   // MVE-specific helper function which reverses the elements of a
7398   // vector within every (ReverseWidth)-bit collection of lanes.
7399   SmallVector<int, 16> Indices;
7400   unsigned LaneSize = V->getType()->getScalarSizeInBits();
7401   unsigned Elements = 128 / LaneSize;
7402   unsigned Mask = ReverseWidth / LaneSize - 1;
7403   for (unsigned i = 0; i < Elements; i++)
7404     Indices.push_back(i ^ Mask);
7405   return Builder.CreateShuffleVector(V, llvm::UndefValue::get(V->getType()),
7406                                      Indices);
7407 }
7408
7409 Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID,
7410                                               const CallExpr *E,
7411                                               ReturnValueSlot ReturnValue,
7412                                               llvm::Triple::ArchType Arch) {
7413   enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
7414   Intrinsic::ID IRIntr;
7415   unsigned NumVectors;
7416
7417   // Code autogenerated by Tablegen will handle all the simple builtins.
7418   switch (BuiltinID) {
7419     #include "clang/Basic/arm_mve_builtin_cg.inc"
7420
7421     // If we didn't match an MVE builtin id at all, go back to the
7422     // main EmitARMBuiltinExpr.
7423   default:
7424     return nullptr;
7425   }
7426
7427   // Anything that breaks from that switch is an MVE builtin that
7428   // needs handwritten code to generate.
7429
7430   switch (CustomCodeGenType) {
7431
7432   case CustomCodeGen::VLD24: {
7433     llvm::SmallVector<Value *, 4> Ops;
7434     llvm::SmallVector<llvm::Type *, 4> Tys;
7435
7436     auto MvecCType = E->getType();
7437     auto MvecLType = ConvertType(MvecCType);
7438     assert(MvecLType->isStructTy() &&
7439            "Return type for vld[24]q should be a struct");
7440     assert(MvecLType->getStructNumElements() == 1 &&
7441            "Return-type struct for vld[24]q should have one element");
7442     auto MvecLTypeInner = MvecLType->getStructElementType(0);
7443     assert(MvecLTypeInner->isArrayTy() &&
7444            "Return-type struct for vld[24]q should contain an array");
7445     assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
7446            "Array member of return-type struct vld[24]q has wrong length");
7447     auto VecLType = MvecLTypeInner->getArrayElementType();
7448
7449     Tys.push_back(VecLType);
7450
7451     auto Addr = E->getArg(0);
7452     Ops.push_back(EmitScalarExpr(Addr));
7453     Tys.push_back(ConvertType(Addr->getType()));
7454
7455     Function *F = CGM.getIntrinsic(IRIntr, makeArrayRef(Tys));
7456     Value *LoadResult = Builder.CreateCall(F, Ops);
7457     Value *MvecOut = UndefValue::get(MvecLType);
7458     for (unsigned i = 0; i < NumVectors; ++i) {
7459       Value *Vec = Builder.CreateExtractValue(LoadResult, i);
7460       MvecOut = Builder.CreateInsertValue(MvecOut, Vec, {0, i});
7461     }
7462
7463     if (ReturnValue.isNull())
7464       return MvecOut;
7465     else
7466       return Builder.CreateStore(MvecOut, ReturnValue.getValue());
7467   }
7468
7469   case CustomCodeGen::VST24: {
7470     llvm::SmallVector<Value *, 4> Ops;
7471     llvm::SmallVector<llvm::Type *, 4> Tys;
7472
7473     auto Addr = E->getArg(0);
7474     Ops.push_back(EmitScalarExpr(Addr));
7475     Tys.push_back(ConvertType(Addr->getType()));
7476
7477     auto MvecCType = E->getArg(1)->getType();
7478     auto MvecLType = ConvertType(MvecCType);
7479     assert(MvecLType->isStructTy() && "Data type for vst2q should be a struct");
7480     assert(MvecLType->getStructNumElements() == 1 &&
7481            "Data-type struct for vst2q should have one element");
7482     auto MvecLTypeInner = MvecLType->getStructElementType(0);
7483     assert(MvecLTypeInner->isArrayTy() &&
7484            "Data-type struct for vst2q should contain an array");
7485     assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
7486            "Array member of return-type struct vld[24]q has wrong length");
7487     auto VecLType = MvecLTypeInner->getArrayElementType();
7488
7489     Tys.push_back(VecLType);
7490
7491     AggValueSlot MvecSlot = CreateAggTemp(MvecCType);
7492     EmitAggExpr(E->getArg(1), MvecSlot);
7493     auto Mvec = Builder.CreateLoad(MvecSlot.getAddress());
7494     for (unsigned i = 0; i < NumVectors; i++)
7495       Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i}));
7496
7497     Function *F = CGM.getIntrinsic(IRIntr, makeArrayRef(Tys));
7498     Value *ToReturn = nullptr;
7499     for (unsigned i = 0; i < NumVectors; i++) {
7500       Ops.push_back(llvm::ConstantInt::get(Int32Ty, i));
7501       ToReturn = Builder.CreateCall(F, Ops);
7502       Ops.pop_back();
7503     }
7504     return ToReturn;
7505   }
7506   }
7507   llvm_unreachable("unknown custom codegen type.");
7508 }
7509
7510 Value *CodeGenFunction::EmitARMCDEBuiltinExpr(unsigned BuiltinID,
7511                                               const CallExpr *E,
7512                                               ReturnValueSlot ReturnValue,
7513                                               llvm::Triple::ArchType Arch) {
7514   switch (BuiltinID) {
7515   default:
7516     return nullptr;
7517 #include "clang/Basic/arm_cde_builtin_cg.inc"
7518   }
7519 }
7520
7521 static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
7522                                       const CallExpr *E,
7523                                       SmallVectorImpl<Value *> &Ops,
7524                                       llvm::Triple::ArchType Arch) {
7525   unsigned int Int = 0;
7526   const char *s = nullptr;
7527
7528   switch (BuiltinID) {
7529   default:
7530     return nullptr;
7531   case NEON::BI__builtin_neon_vtbl1_v:
7532   case NEON::BI__builtin_neon_vqtbl1_v:
7533   case NEON::BI__builtin_neon_vqtbl1q_v:
7534   case NEON::BI__builtin_neon_vtbl2_v:
7535   case NEON::BI__builtin_neon_vqtbl2_v:
7536   case NEON::BI__builtin_neon_vqtbl2q_v:
7537   case NEON::BI__builtin_neon_vtbl3_v:
7538   case NEON::BI__builtin_neon_vqtbl3_v:
7539   case NEON::BI__builtin_neon_vqtbl3q_v:
7540   case NEON::BI__builtin_neon_vtbl4_v:
7541   case NEON::BI__builtin_neon_vqtbl4_v:
7542   case NEON::BI__builtin_neon_vqtbl4q_v:
7543     break;
7544   case NEON::BI__builtin_neon_vtbx1_v:
7545   case NEON::BI__builtin_neon_vqtbx1_v:
7546   case NEON::BI__builtin_neon_vqtbx1q_v:
7547   case NEON::BI__builtin_neon_vtbx2_v:
7548   case NEON::BI__builtin_neon_vqtbx2_v:
7549   case NEON::BI__builtin_neon_vqtbx2q_v:
7550   case NEON::BI__builtin_neon_vtbx3_v:
7551   case NEON::BI__builtin_neon_vqtbx3_v:
7552   case NEON::BI__builtin_neon_vqtbx3q_v:
7553   case NEON::BI__builtin_neon_vtbx4_v:
7554   case NEON::BI__builtin_neon_vqtbx4_v:
7555   case NEON::BI__builtin_neon_vqtbx4q_v:
7556     break;
7557   }
7558
7559   assert(E->getNumArgs() >= 3);
7560
7561   // Get the last argument, which specifies the vector type.
7562   llvm::APSInt Result;
7563   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
7564   if (!Arg->isIntegerConstantExpr(Result, CGF.getContext()))
7565     return nullptr;
7566
7567   // Determine the type of this overloaded NEON intrinsic.
7568   NeonTypeFlags Type(Result.getZExtValue());
7569   llvm::VectorType *Ty = GetNeonType(&CGF, Type);
7570   if (!Ty)
7571     return nullptr;
7572
7573   CodeGen::CGBuilderTy &Builder = CGF.Builder;
7574
7575   // AArch64 scalar builtins are not overloaded, they do not have an extra
7576   // argument that specifies the vector type, need to handle each case.
7577   switch (BuiltinID) {
7578   case NEON::BI__builtin_neon_vtbl1_v: {
7579     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr,
7580                               Ops[1], Ty, Intrinsic::aarch64_neon_tbl1,
7581                               "vtbl1");
7582   }
7583   case NEON::BI__builtin_neon_vtbl2_v: {
7584     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr,
7585                               Ops[2], Ty, Intrinsic::aarch64_neon_tbl1,
7586                               "vtbl1");
7587   }
7588   case NEON::BI__builtin_neon_vtbl3_v: {
7589     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr,
7590                               Ops[3], Ty, Intrinsic::aarch64_neon_tbl2,
7591                               "vtbl2");
7592   }
7593   case NEON::BI__builtin_neon_vtbl4_v: {
7594     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr,
7595                               Ops[4], Ty, Intrinsic::aarch64_neon_tbl2,
7596                               "vtbl2");
7597   }
7598   case NEON::BI__builtin_neon_vtbx1_v: {
7599     Value *TblRes =
7600         packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2],
7601                            Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
7602
7603     llvm::Constant *EightV = ConstantInt::get(Ty, 8);
7604     Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
7605     CmpRes = Builder.CreateSExt(CmpRes, Ty);
7606
7607     Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
7608     Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
7609     return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
7610   }
7611   case NEON::BI__builtin_neon_vtbx2_v: {
7612     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0],
7613                               Ops[3], Ty, Intrinsic::aarch64_neon_tbx1,
7614                               "vtbx1");
7615   }
7616   case NEON::BI__builtin_neon_vtbx3_v: {
7617     Value *TblRes =
7618         packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4],
7619                            Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
7620
7621     llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
7622     Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
7623                                            TwentyFourV);
7624     CmpRes = Builder.CreateSExt(CmpRes, Ty);
7625
7626     Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
7627     Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
7628     return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
7629   }
7630   case NEON::BI__builtin_neon_vtbx4_v: {
7631     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0],
7632                               Ops[5], Ty, Intrinsic::aarch64_neon_tbx2,
7633                               "vtbx2");
7634   }
7635   case NEON::BI__builtin_neon_vqtbl1_v:
7636   case NEON::BI__builtin_neon_vqtbl1q_v:
7637     Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
7638   case NEON::BI__builtin_neon_vqtbl2_v:
7639   case NEON::BI__builtin_neon_vqtbl2q_v: {
7640     Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
7641   case NEON::BI__builtin_neon_vqtbl3_v:
7642   case NEON::BI__builtin_neon_vqtbl3q_v:
7643     Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
7644   case NEON::BI__builtin_neon_vqtbl4_v:
7645   case NEON::BI__builtin_neon_vqtbl4q_v:
7646     Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
7647   case NEON::BI__builtin_neon_vqtbx1_v:
7648   case NEON::BI__builtin_neon_vqtbx1q_v:
7649     Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
7650   case NEON::BI__builtin_neon_vqtbx2_v:
7651   case NEON::BI__builtin_neon_vqtbx2q_v:
7652     Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
7653   case NEON::BI__builtin_neon_vqtbx3_v:
7654   case NEON::BI__builtin_neon_vqtbx3q_v:
7655     Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
7656   case NEON::BI__builtin_neon_vqtbx4_v:
7657   case NEON::BI__builtin_neon_vqtbx4q_v:
7658     Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
7659   }
7660   }
7661
7662   if (!Int)
7663     return nullptr;
7664
7665   Function *F = CGF.CGM.getIntrinsic(Int, Ty);
7666   return CGF.EmitNeonCall(F, Ops, s);
7667 }
7668
7669 Value *CodeGenFunction::vectorWrapScalar16(Value *Op) {
7670   auto *VTy = llvm::FixedVectorType::get(Int16Ty, 4);
7671   Op = Builder.CreateBitCast(Op, Int16Ty);
7672   Value *V = UndefValue::get(VTy);
7673   llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
7674   Op = Builder.CreateInsertElement(V, Op, CI);
7675   return Op;
7676 }
7677
7678 /// SVEBuiltinMemEltTy - Returns the memory element type for this memory
7679 /// access builtin.  Only required if it can't be inferred from the base pointer
7680 /// operand.
7681 llvm::Type *CodeGenFunction::SVEBuiltinMemEltTy(SVETypeFlags TypeFlags) {
7682   switch (TypeFlags.getMemEltType()) {
7683   case SVETypeFlags::MemEltTyDefault:
7684     return getEltType(TypeFlags);
7685   case SVETypeFlags::MemEltTyInt8:
7686     return Builder.getInt8Ty();
7687   case SVETypeFlags::MemEltTyInt16:
7688     return Builder.getInt16Ty();
7689   case SVETypeFlags::MemEltTyInt32:
7690     return Builder.getInt32Ty();
7691   case SVETypeFlags::MemEltTyInt64:
7692     return Builder.getInt64Ty();
7693   }
7694   llvm_unreachable("Unknown MemEltType");
7695 }
7696
7697 llvm::Type *CodeGenFunction::getEltType(SVETypeFlags TypeFlags) {
7698   switch (TypeFlags.getEltType()) {
7699   default:
7700     llvm_unreachable("Invalid SVETypeFlag!");
7701
7702   case SVETypeFlags::EltTyInt8:
7703     return Builder.getInt8Ty();
7704   case SVETypeFlags::EltTyInt16:
7705     return Builder.getInt16Ty();
7706   case SVETypeFlags::EltTyInt32:
7707     return Builder.getInt32Ty();
7708   case SVETypeFlags::EltTyInt64:
7709     return Builder.getInt64Ty();
7710
7711   case SVETypeFlags::EltTyFloat16:
7712     return Builder.getHalfTy();
7713   case SVETypeFlags::EltTyFloat32:
7714     return Builder.getFloatTy();
7715   case SVETypeFlags::EltTyFloat64:
7716     return Builder.getDoubleTy();
7717
7718   case SVETypeFlags::EltTyBFloat16:
7719     return Builder.getBFloatTy();
7720
7721   case SVETypeFlags::EltTyBool8:
7722   case SVETypeFlags::EltTyBool16:
7723   case SVETypeFlags::EltTyBool32:
7724   case SVETypeFlags::EltTyBool64:
7725     return Builder.getInt1Ty();
7726   }
7727 }
7728
7729 // Return the llvm predicate vector type corresponding to the specified element
7730 // TypeFlags.
7731 llvm::ScalableVectorType *
7732 CodeGenFunction::getSVEPredType(SVETypeFlags TypeFlags) {
7733   switch (TypeFlags.getEltType()) {
7734   default: llvm_unreachable("Unhandled SVETypeFlag!");
7735
7736   case SVETypeFlags::EltTyInt8:
7737     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
7738   case SVETypeFlags::EltTyInt16:
7739     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
7740   case SVETypeFlags::EltTyInt32:
7741     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
7742   case SVETypeFlags::EltTyInt64:
7743     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
7744
7745   case SVETypeFlags::EltTyBFloat16:
7746     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
7747   case SVETypeFlags::EltTyFloat16:
7748     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
7749   case SVETypeFlags::EltTyFloat32:
7750     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
7751   case SVETypeFlags::EltTyFloat64:
7752     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
7753
7754   case SVETypeFlags::EltTyBool8:
7755     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
7756   case SVETypeFlags::EltTyBool16:
7757     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
7758   case SVETypeFlags::EltTyBool32:
7759     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
7760   case SVETypeFlags::EltTyBool64:
7761     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
7762   }
7763 }
7764
7765 // Return the llvm vector type corresponding to the specified element TypeFlags.
7766 llvm::ScalableVectorType *
7767 CodeGenFunction::getSVEType(const SVETypeFlags &TypeFlags) {
7768   switch (TypeFlags.getEltType()) {
7769   default:
7770     llvm_unreachable("Invalid SVETypeFlag!");
7771
7772   case SVETypeFlags::EltTyInt8:
7773     return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
7774   case SVETypeFlags::EltTyInt16:
7775     return llvm::ScalableVectorType::get(Builder.getInt16Ty(), 8);
7776   case SVETypeFlags::EltTyInt32:
7777     return llvm::ScalableVectorType::get(Builder.getInt32Ty(), 4);
7778   case SVETypeFlags::EltTyInt64:
7779     return llvm::ScalableVectorType::get(Builder.getInt64Ty(), 2);
7780
7781   case SVETypeFlags::EltTyFloat16:
7782     return llvm::ScalableVectorType::get(Builder.getHalfTy(), 8);
7783   case SVETypeFlags::EltTyBFloat16:
7784     return llvm::ScalableVectorType::get(Builder.getBFloatTy(), 8);
7785   case SVETypeFlags::EltTyFloat32:
7786     return llvm::ScalableVectorType::get(Builder.getFloatTy(), 4);
7787   case SVETypeFlags::EltTyFloat64:
7788     return llvm::ScalableVectorType::get(Builder.getDoubleTy(), 2);
7789
7790   case SVETypeFlags::EltTyBool8:
7791     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
7792   case SVETypeFlags::EltTyBool16:
7793     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
7794   case SVETypeFlags::EltTyBool32:
7795     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
7796   case SVETypeFlags::EltTyBool64:
7797     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
7798   }
7799 }
7800
7801 llvm::Value *CodeGenFunction::EmitSVEAllTruePred(SVETypeFlags TypeFlags) {
7802   Function *Ptrue =
7803       CGM.getIntrinsic(Intrinsic::aarch64_sve_ptrue, getSVEPredType(TypeFlags));
7804   return Builder.CreateCall(Ptrue, {Builder.getInt32(/*SV_ALL*/ 31)});
7805 }
7806
7807 constexpr unsigned SVEBitsPerBlock = 128;
7808
7809 static llvm::ScalableVectorType *getSVEVectorForElementType(llvm::Type *EltTy) {
7810   unsigned NumElts = SVEBitsPerBlock / EltTy->getScalarSizeInBits();
7811   return llvm::ScalableVectorType::get(EltTy, NumElts);
7812 }
7813
7814 // Reinterpret the input predicate so that it can be used to correctly isolate
7815 // the elements of the specified datatype.
7816 Value *CodeGenFunction::EmitSVEPredicateCast(Value *Pred,
7817                                              llvm::ScalableVectorType *VTy) {
7818   auto *RTy = llvm::VectorType::get(IntegerType::get(getLLVMContext(), 1), VTy);
7819   if (Pred->getType() == RTy)
7820     return Pred;
7821
7822   unsigned IntID;
7823   llvm::Type *IntrinsicTy;
7824   switch (VTy->getMinNumElements()) {
7825   default:
7826     llvm_unreachable("unsupported element count!");
7827   case 2:
7828   case 4:
7829   case 8:
7830     IntID = Intrinsic::aarch64_sve_convert_from_svbool;
7831     IntrinsicTy = RTy;
7832     break;
7833   case 16:
7834     IntID = Intrinsic::aarch64_sve_convert_to_svbool;
7835     IntrinsicTy = Pred->getType();
7836     break;
7837   }
7838
7839   Function *F = CGM.getIntrinsic(IntID, IntrinsicTy);
7840   Value *C = Builder.CreateCall(F, Pred);
7841   assert(C->getType() == RTy && "Unexpected return type!");
7842   return C;
7843 }
7844
7845 Value *CodeGenFunction::EmitSVEGatherLoad(SVETypeFlags TypeFlags,
7846                                           SmallVectorImpl<Value *> &Ops,
7847                                           unsigned IntID) {
7848   auto *ResultTy = getSVEType(TypeFlags);
7849   auto *OverloadedTy =
7850       llvm::ScalableVectorType::get(SVEBuiltinMemEltTy(TypeFlags), ResultTy);
7851
7852   // At the ACLE level there's only one predicate type, svbool_t, which is
7853   // mapped to <n x 16 x i1>. However, this might be incompatible with the
7854   // actual type being loaded. For example, when loading doubles (i64) the
7855   // predicated should be <n x 2 x i1> instead. At the IR level the type of
7856   // the predicate and the data being loaded must match. Cast accordingly.
7857   Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy);
7858
7859   Function *F = nullptr;
7860   if (Ops[1]->getType()->isVectorTy())
7861     // This is the "vector base, scalar offset" case. In order to uniquely
7862     // map this built-in to an LLVM IR intrinsic, we need both the return type
7863     // and the type of the vector base.
7864     F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[1]->getType()});
7865   else
7866     // This is the "scalar base, vector offset case". The type of the offset
7867     // is encoded in the name of the intrinsic. We only need to specify the
7868     // return type in order to uniquely map this built-in to an LLVM IR
7869     // intrinsic.
7870     F = CGM.getIntrinsic(IntID, OverloadedTy);
7871
7872   // Pass 0 when the offset is missing. This can only be applied when using
7873   // the "vector base" addressing mode for which ACLE allows no offset. The
7874   // corresponding LLVM IR always requires an offset.
7875   if (Ops.size() == 2) {
7876     assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
7877     Ops.push_back(ConstantInt::get(Int64Ty, 0));
7878   }
7879
7880   // For "vector base, scalar index" scale the index so that it becomes a
7881   // scalar offset.
7882   if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
7883     unsigned BytesPerElt =
7884         OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
7885     Value *Scale = ConstantInt::get(Int64Ty, BytesPerElt);
7886     Ops[2] = Builder.CreateMul(Ops[2], Scale);
7887   }
7888
7889   Value *Call = Builder.CreateCall(F, Ops);
7890
7891   // The following sext/zext is only needed when ResultTy != OverloadedTy. In
7892   // other cases it's folded into a nop.
7893   return TypeFlags.isZExtReturn() ? Builder.CreateZExt(Call, ResultTy)
7894                                   : Builder.CreateSExt(Call, ResultTy);
7895 }
7896
7897 Value *CodeGenFunction::EmitSVEScatterStore(SVETypeFlags TypeFlags,
7898                                             SmallVectorImpl<Value *> &Ops,
7899                                             unsigned IntID) {
7900   auto *SrcDataTy = getSVEType(TypeFlags);
7901   auto *OverloadedTy =
7902       llvm::ScalableVectorType::get(SVEBuiltinMemEltTy(TypeFlags), SrcDataTy);
7903
7904   // In ACLE the source data is passed in the last argument, whereas in LLVM IR
7905   // it's the first argument. Move it accordingly.
7906   Ops.insert(Ops.begin(), Ops.pop_back_val());
7907
7908   Function *F = nullptr;
7909   if (Ops[2]->getType()->isVectorTy())
7910     // This is the "vector base, scalar offset" case. In order to uniquely
7911     // map this built-in to an LLVM IR intrinsic, we need both the return type
7912     // and the type of the vector base.
7913     F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[2]->getType()});
7914   else
7915     // This is the "scalar base, vector offset case". The type of the offset
7916     // is encoded in the name of the intrinsic. We only need to specify the
7917     // return type in order to uniquely map this built-in to an LLVM IR
7918     // intrinsic.
7919     F = CGM.getIntrinsic(IntID, OverloadedTy);
7920
7921   // Pass 0 when the offset is missing. This can only be applied when using
7922   // the "vector base" addressing mode for which ACLE allows no offset. The
7923   // corresponding LLVM IR always requires an offset.
7924   if (Ops.size() == 3) {
7925     assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
7926     Ops.push_back(ConstantInt::get(Int64Ty, 0));
7927   }
7928
7929   // Truncation is needed when SrcDataTy != OverloadedTy. In other cases it's
7930   // folded into a nop.
7931   Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy);
7932
7933   // At the ACLE level there's only one predicate type, svbool_t, which is
7934   // mapped to <n x 16 x i1>. However, this might be incompatible with the
7935   // actual type being stored. For example, when storing doubles (i64) the
7936   // predicated should be <n x 2 x i1> instead. At the IR level the type of
7937   // the predicate and the data being stored must match. Cast accordingly.
7938   Ops[1] = EmitSVEPredicateCast(Ops[1], OverloadedTy);
7939
7940   // For "vector base, scalar index" scale the index so that it becomes a
7941   // scalar offset.
7942   if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
7943     unsigned BytesPerElt =
7944         OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
7945     Value *Scale = ConstantInt::get(Int64Ty, BytesPerElt);
7946     Ops[3] = Builder.CreateMul(Ops[3], Scale);
7947   }
7948
7949   return Builder.CreateCall(F, Ops);
7950 }
7951
7952 Value *CodeGenFunction::EmitSVEGatherPrefetch(SVETypeFlags TypeFlags,
7953                                               SmallVectorImpl<Value *> &Ops,
7954                                               unsigned IntID) {
7955   // The gather prefetches are overloaded on the vector input - this can either
7956   // be the vector of base addresses or vector of offsets.
7957   auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
7958   if (!OverloadedTy)
7959     OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
7960
7961   // Cast the predicate from svbool_t to the right number of elements.
7962   Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy);
7963
7964   // vector + imm addressing modes
7965   if (Ops[1]->getType()->isVectorTy()) {
7966     if (Ops.size() == 3) {
7967       // Pass 0 for 'vector+imm' when the index is omitted.
7968       Ops.push_back(ConstantInt::get(Int64Ty, 0));
7969
7970       // The sv_prfop is the last operand in the builtin and IR intrinsic.
7971       std::swap(Ops[2], Ops[3]);
7972     } else {
7973       // Index needs to be passed as scaled offset.
7974       llvm::Type *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
7975       unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
7976       Value *Scale = ConstantInt::get(Int64Ty, BytesPerElt);
7977       Ops[2] = Builder.CreateMul(Ops[2], Scale);
7978     }
7979   }
7980
7981   Function *F = CGM.getIntrinsic(IntID, OverloadedTy);
7982   return Builder.CreateCall(F, Ops);
7983 }
7984
7985 Value *CodeGenFunction::EmitSVEStructLoad(SVETypeFlags TypeFlags,
7986                                           SmallVectorImpl<Value*> &Ops,
7987                                           unsigned IntID) {
7988   llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
7989   auto VecPtrTy = llvm::PointerType::getUnqual(VTy);
7990   auto EltPtrTy = llvm::PointerType::getUnqual(VTy->getElementType());
7991
7992   unsigned N;
7993   switch (IntID) {
7994   case Intrinsic::aarch64_sve_ld2:
7995     N = 2;
7996     break;
7997   case Intrinsic::aarch64_sve_ld3:
7998     N = 3;
7999     break;
8000   case Intrinsic::aarch64_sve_ld4:
8001     N = 4;
8002     break;
8003   default:
8004     llvm_unreachable("unknown intrinsic!");
8005   }
8006   auto RetTy = llvm::VectorType::get(VTy->getElementType(),
8007                                      VTy->getElementCount() * N);
8008
8009         Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy);
8010   Value *BasePtr= Builder.CreateBitCast(Ops[1], VecPtrTy);
8011   Value *Offset = Ops.size() > 2 ? Ops[2] : Builder.getInt32(0);
8012   BasePtr = Builder.CreateGEP(VTy, BasePtr, Offset);
8013   BasePtr = Builder.CreateBitCast(BasePtr, EltPtrTy);
8014
8015   Function *F = CGM.getIntrinsic(IntID, {RetTy, Predicate->getType()});
8016   return Builder.CreateCall(F, { Predicate, BasePtr });
8017 }
8018
8019 Value *CodeGenFunction::EmitSVEStructStore(SVETypeFlags TypeFlags,
8020                                            SmallVectorImpl<Value*> &Ops,
8021                                            unsigned IntID) {
8022   llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
8023   auto VecPtrTy = llvm::PointerType::getUnqual(VTy);
8024   auto EltPtrTy = llvm::PointerType::getUnqual(VTy->getElementType());
8025
8026   unsigned N;
8027   switch (IntID) {
8028   case Intrinsic::aarch64_sve_st2:
8029     N = 2;
8030     break;
8031   case Intrinsic::aarch64_sve_st3:
8032     N = 3;
8033     break;
8034   case Intrinsic::aarch64_sve_st4:
8035     N = 4;
8036     break;
8037   default:
8038     llvm_unreachable("unknown intrinsic!");
8039   }
8040   auto TupleTy =
8041       llvm::VectorType::get(VTy->getElementType(), VTy->getElementCount() * N);
8042
8043   Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy);
8044   Value *BasePtr = Builder.CreateBitCast(Ops[1], VecPtrTy);
8045   Value *Offset = Ops.size() > 3 ? Ops[2] : Builder.getInt32(0);
8046   Value *Val = Ops.back();
8047   BasePtr = Builder.CreateGEP(VTy, BasePtr, Offset);
8048   BasePtr = Builder.CreateBitCast(BasePtr, EltPtrTy);
8049
8050   // The llvm.aarch64.sve.st2/3/4 intrinsics take legal part vectors, so we
8051   // need to break up the tuple vector.
8052   SmallVector<llvm::Value*, 5> Operands;
8053   Function *FExtr =
8054       CGM.getIntrinsic(Intrinsic::aarch64_sve_tuple_get, {VTy, TupleTy});
8055   for (unsigned I = 0; I < N; ++I)
8056     Operands.push_back(Builder.CreateCall(FExtr, {Val, Builder.getInt32(I)}));
8057   Operands.append({Predicate, BasePtr});
8058
8059   Function *F = CGM.getIntrinsic(IntID, { VTy });
8060   return Builder.CreateCall(F, Operands);
8061 }
8062
8063 // SVE2's svpmullb and svpmullt builtins are similar to the svpmullb_pair and
8064 // svpmullt_pair intrinsics, with the exception that their results are bitcast
8065 // to a wider type.
8066 Value *CodeGenFunction::EmitSVEPMull(SVETypeFlags TypeFlags,
8067                                      SmallVectorImpl<Value *> &Ops,
8068                                      unsigned BuiltinID) {
8069   // Splat scalar operand to vector (intrinsics with _n infix)
8070   if (TypeFlags.hasSplatOperand()) {
8071     unsigned OpNo = TypeFlags.getSplatOperand();
8072     Ops[OpNo] = EmitSVEDupX(Ops[OpNo]);
8073   }
8074
8075   // The pair-wise function has a narrower overloaded type.
8076   Function *F = CGM.getIntrinsic(BuiltinID, Ops[0]->getType());
8077   Value *Call = Builder.CreateCall(F, {Ops[0], Ops[1]});
8078
8079   // Now bitcast to the wider result type.
8080   llvm::ScalableVectorType *Ty = getSVEType(TypeFlags);
8081   return EmitSVEReinterpret(Call, Ty);
8082 }
8083
8084 Value *CodeGenFunction::EmitSVEMovl(SVETypeFlags TypeFlags,
8085                                     ArrayRef<Value *> Ops, unsigned BuiltinID) {
8086   llvm::Type *OverloadedTy = getSVEType(TypeFlags);
8087   Function *F = CGM.getIntrinsic(BuiltinID, OverloadedTy);
8088   return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)});
8089 }
8090
8091 Value *CodeGenFunction::EmitSVEPrefetchLoad(SVETypeFlags TypeFlags,
8092                                             SmallVectorImpl<Value *> &Ops,
8093                                             unsigned BuiltinID) {
8094   auto *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
8095   auto *VectorTy = getSVEVectorForElementType(MemEltTy);
8096   auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
8097
8098   Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy);
8099   Value *BasePtr = Ops[1];
8100
8101   // Implement the index operand if not omitted.
8102   if (Ops.size() > 3) {
8103     BasePtr = Builder.CreateBitCast(BasePtr, MemoryTy->getPointerTo());
8104     BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
8105   }
8106
8107   // Prefetch intriniscs always expect an i8*
8108   BasePtr = Builder.CreateBitCast(BasePtr, llvm::PointerType::getUnqual(Int8Ty));
8109   Value *PrfOp = Ops.back();
8110
8111   Function *F = CGM.getIntrinsic(BuiltinID, Predicate->getType());
8112   return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
8113 }
8114
8115 Value *CodeGenFunction::EmitSVEMaskedLoad(const CallExpr *E,
8116                                           llvm::Type *ReturnTy,
8117                                           SmallVectorImpl<Value *> &Ops,
8118                                           unsigned BuiltinID,
8119                                           bool IsZExtReturn) {
8120   QualType LangPTy = E->getArg(1)->getType();
8121   llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
8122       LangPTy->getAs<PointerType>()->getPointeeType());
8123
8124   // The vector type that is returned may be different from the
8125   // eventual type loaded from memory.
8126   auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
8127   auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
8128
8129   Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy);
8130   Value *BasePtr = Builder.CreateBitCast(Ops[1], MemoryTy->getPointerTo());
8131   Value *Offset = Ops.size() > 2 ? Ops[2] : Builder.getInt32(0);
8132   BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Offset);
8133
8134   BasePtr = Builder.CreateBitCast(BasePtr, MemEltTy->getPointerTo());
8135   Function *F = CGM.getIntrinsic(BuiltinID, MemoryTy);
8136   Value *Load = Builder.CreateCall(F, {Predicate, BasePtr});
8137
8138   return IsZExtReturn ? Builder.CreateZExt(Load, VectorTy)
8139                      : Builder.CreateSExt(Load, VectorTy);
8140 }
8141
8142 Value *CodeGenFunction::EmitSVEMaskedStore(const CallExpr *E,
8143                                            SmallVectorImpl<Value *> &Ops,
8144                                            unsigned BuiltinID) {
8145   QualType LangPTy = E->getArg(1)->getType();
8146   llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
8147       LangPTy->getAs<PointerType>()->getPointeeType());
8148
8149   // The vector type that is stored may be different from the
8150   // eventual type stored to memory.
8151   auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
8152   auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
8153
8154   Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy);
8155   Value *BasePtr = Builder.CreateBitCast(Ops[1], MemoryTy->getPointerTo());
8156   Value *Offset = Ops.size() == 4 ? Ops[2] : Builder.getInt32(0);
8157   BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Offset);
8158
8159   // Last value is always the data
8160   llvm::Value *Val = Builder.CreateTrunc(Ops.back(), MemoryTy);
8161
8162   BasePtr = Builder.CreateBitCast(BasePtr, MemEltTy->getPointerTo());
8163   Function *F = CGM.getIntrinsic(BuiltinID, MemoryTy);
8164   return Builder.CreateCall(F, {Val, Predicate, BasePtr});
8165 }
8166
8167 // Limit the usage of scalable llvm IR generated by the ACLE by using the
8168 // sve dup.x intrinsic instead of IRBuilder::CreateVectorSplat.
8169 Value *CodeGenFunction::EmitSVEDupX(Value *Scalar, llvm::Type *Ty) {
8170   auto F = CGM.getIntrinsic(Intrinsic::aarch64_sve_dup_x, Ty);
8171   return Builder.CreateCall(F, Scalar);
8172 }
8173
8174 Value *CodeGenFunction::EmitSVEDupX(Value* Scalar) {
8175   return EmitSVEDupX(Scalar, getSVEVectorForElementType(Scalar->getType()));
8176 }
8177
8178 Value *CodeGenFunction::EmitSVEReinterpret(Value *Val, llvm::Type *Ty) {
8179   // FIXME: For big endian this needs an additional REV, or needs a separate
8180   // intrinsic that is code-generated as a no-op, because the LLVM bitcast
8181   // instruction is defined as 'bitwise' equivalent from memory point of
8182   // view (when storing/reloading), whereas the svreinterpret builtin
8183   // implements bitwise equivalent cast from register point of view.
8184   // LLVM CodeGen for a bitcast must add an explicit REV for big-endian.
8185   return Builder.CreateBitCast(Val, Ty);
8186 }
8187
8188 static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty,
8189                                       SmallVectorImpl<Value *> &Ops) {
8190   auto *SplatZero = Constant::getNullValue(Ty);
8191   Ops.insert(Ops.begin(), SplatZero);
8192 }
8193
8194 static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty,
8195                                        SmallVectorImpl<Value *> &Ops) {
8196   auto *SplatUndef = UndefValue::get(Ty);
8197   Ops.insert(Ops.begin(), SplatUndef);
8198 }
8199
8200 SmallVector<llvm::Type *, 2> CodeGenFunction::getSVEOverloadTypes(
8201     SVETypeFlags TypeFlags, llvm::Type *ResultType, ArrayRef<Value *> Ops) {
8202   if (TypeFlags.isOverloadNone())
8203     return {};
8204
8205   llvm::Type *DefaultType = getSVEType(TypeFlags);
8206
8207   if (TypeFlags.isOverloadWhile())
8208     return {DefaultType, Ops[1]->getType()};
8209
8210   if (TypeFlags.isOverloadWhileRW())
8211     return {getSVEPredType(TypeFlags), Ops[0]->getType()};
8212
8213   if (TypeFlags.isOverloadCvt() || TypeFlags.isTupleSet())
8214     return {Ops[0]->getType(), Ops.back()->getType()};
8215
8216   if (TypeFlags.isTupleCreate() || TypeFlags.isTupleGet())
8217     return {ResultType, Ops[0]->getType()};
8218
8219   assert(TypeFlags.isOverloadDefault() && "Unexpected value for overloads");
8220   return {DefaultType};
8221 }
8222
8223 Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
8224                                                   const CallExpr *E) {
8225   // Find out if any arguments are required to be integer constant expressions.
8226   unsigned ICEArguments = 0;
8227   ASTContext::GetBuiltinTypeError Error;
8228   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
8229   assert(Error == ASTContext::GE_None && "Should not codegen an error");
8230
8231   llvm::Type *Ty = ConvertType(E->getType());
8232   if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
8233       BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64) {
8234     Value *Val = EmitScalarExpr(E->getArg(0));
8235     return EmitSVEReinterpret(Val, Ty);
8236   }
8237
8238   llvm::SmallVector<Value *, 4> Ops;
8239   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
8240     if ((ICEArguments & (1 << i)) == 0)
8241       Ops.push_back(EmitScalarExpr(E->getArg(i)));
8242     else {
8243       // If this is required to be a constant, constant fold it so that we know
8244       // that the generated intrinsic gets a ConstantInt.
8245       llvm::APSInt Result;
8246       if (!E->getArg(i)->isIntegerConstantExpr(Result, getContext()))
8247         llvm_unreachable("Expected argument to be a constant");
8248
8249       // Immediates for SVE llvm intrinsics are always 32bit.  We can safely
8250       // truncate because the immediate has been range checked and no valid
8251       // immediate requires more than a handful of bits.
8252       Result = Result.extOrTrunc(32);
8253       Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
8254     }
8255   }
8256
8257   auto *Builtin = findARMVectorIntrinsicInMap(AArch64SVEIntrinsicMap, BuiltinID,
8258                                               AArch64SVEIntrinsicsProvenSorted);
8259   SVETypeFlags TypeFlags(Builtin->TypeModifier);
8260   if (TypeFlags.isLoad())
8261     return EmitSVEMaskedLoad(E, Ty, Ops, Builtin->LLVMIntrinsic,
8262                              TypeFlags.isZExtReturn());
8263   else if (TypeFlags.isStore())
8264     return EmitSVEMaskedStore(E, Ops, Builtin->LLVMIntrinsic);
8265   else if (TypeFlags.isGatherLoad())
8266     return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
8267   else if (TypeFlags.isScatterStore())
8268     return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
8269   else if (TypeFlags.isPrefetch())
8270     return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
8271   else if (TypeFlags.isGatherPrefetch())
8272     return EmitSVEGatherPrefetch(TypeFlags, Ops, Builtin->LLVMIntrinsic);
8273         else if (TypeFlags.isStructLoad())
8274                 return EmitSVEStructLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
8275         else if (TypeFlags.isStructStore())
8276                 return EmitSVEStructStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
8277   else if (TypeFlags.isUndef())
8278     return UndefValue::get(Ty);
8279   else if (Builtin->LLVMIntrinsic != 0) {
8280     if (TypeFlags.getMergeType() == SVETypeFlags::MergeZeroExp)
8281       InsertExplicitZeroOperand(Builder, Ty, Ops);
8282
8283     if (TypeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
8284       InsertExplicitUndefOperand(Builder, Ty, Ops);
8285
8286     // Some ACLE builtins leave out the argument to specify the predicate
8287     // pattern, which is expected to be expanded to an SV_ALL pattern.
8288     if (TypeFlags.isAppendSVALL())
8289       Ops.push_back(Builder.getInt32(/*SV_ALL*/ 31));
8290     if (TypeFlags.isInsertOp1SVALL())
8291       Ops.insert(&Ops[1], Builder.getInt32(/*SV_ALL*/ 31));
8292
8293     // Predicates must match the main datatype.
8294     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
8295       if (auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
8296         if (PredTy->getElementType()->isIntegerTy(1))
8297           Ops[i] = EmitSVEPredicateCast(Ops[i], getSVEType(TypeFlags));
8298
8299     // Splat scalar operand to vector (intrinsics with _n infix)
8300     if (TypeFlags.hasSplatOperand()) {
8301       unsigned OpNo = TypeFlags.getSplatOperand();
8302       Ops[OpNo] = EmitSVEDupX(Ops[OpNo]);
8303     }
8304
8305     if (TypeFlags.isReverseCompare())
8306       std::swap(Ops[1], Ops[2]);
8307
8308     if (TypeFlags.isReverseUSDOT())
8309       std::swap(Ops[1], Ops[2]);
8310
8311     // Predicated intrinsics with _z suffix need a select w/ zeroinitializer.
8312     if (TypeFlags.getMergeType() == SVETypeFlags::MergeZero) {
8313       llvm::Type *OpndTy = Ops[1]->getType();
8314       auto *SplatZero = Constant::getNullValue(OpndTy);
8315       Function *Sel = CGM.getIntrinsic(Intrinsic::aarch64_sve_sel, OpndTy);
8316       Ops[1] = Builder.CreateCall(Sel, {Ops[0], Ops[1], SplatZero});
8317     }
8318
8319     Function *F = CGM.getIntrinsic(Builtin->LLVMIntrinsic,
8320                                    getSVEOverloadTypes(TypeFlags, Ty, Ops));
8321     Value *Call = Builder.CreateCall(F, Ops);
8322
8323     // Predicate results must be converted to svbool_t.
8324     if (auto PredTy = dyn_cast<llvm::VectorType>(Call->getType()))
8325       if (PredTy->getScalarType()->isIntegerTy(1))
8326         Call = EmitSVEPredicateCast(Call, cast<llvm::ScalableVectorType>(Ty));
8327
8328     return Call;
8329   }
8330
8331   switch (BuiltinID) {
8332   default:
8333     return nullptr;
8334
8335   case SVE::BI__builtin_sve_svmov_b_z: {
8336     // svmov_b_z(pg, op) <=> svand_b_z(pg, op, op)
8337     SVETypeFlags TypeFlags(Builtin->TypeModifier);
8338     llvm::Type* OverloadedTy = getSVEType(TypeFlags);
8339     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_and_z, OverloadedTy);
8340     return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
8341   }
8342
8343   case SVE::BI__builtin_sve_svnot_b_z: {
8344     // svnot_b_z(pg, op) <=> sveor_b_z(pg, op, pg)
8345     SVETypeFlags TypeFlags(Builtin->TypeModifier);
8346     llvm::Type* OverloadedTy = getSVEType(TypeFlags);
8347     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_eor_z, OverloadedTy);
8348     return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
8349   }
8350
8351   case SVE::BI__builtin_sve_svmovlb_u16:
8352   case SVE::BI__builtin_sve_svmovlb_u32:
8353   case SVE::BI__builtin_sve_svmovlb_u64:
8354     return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
8355
8356   case SVE::BI__builtin_sve_svmovlb_s16:
8357   case SVE::BI__builtin_sve_svmovlb_s32:
8358   case SVE::BI__builtin_sve_svmovlb_s64:
8359     return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
8360
8361   case SVE::BI__builtin_sve_svmovlt_u16:
8362   case SVE::BI__builtin_sve_svmovlt_u32:
8363   case SVE::BI__builtin_sve_svmovlt_u64:
8364     return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
8365
8366   case SVE::BI__builtin_sve_svmovlt_s16:
8367   case SVE::BI__builtin_sve_svmovlt_s32:
8368   case SVE::BI__builtin_sve_svmovlt_s64:
8369     return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
8370
8371   case SVE::BI__builtin_sve_svpmullt_u16:
8372   case SVE::BI__builtin_sve_svpmullt_u64:
8373   case SVE::BI__builtin_sve_svpmullt_n_u16:
8374   case SVE::BI__builtin_sve_svpmullt_n_u64:
8375     return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
8376
8377   case SVE::BI__builtin_sve_svpmullb_u16:
8378   case SVE::BI__builtin_sve_svpmullb_u64:
8379   case SVE::BI__builtin_sve_svpmullb_n_u16:
8380   case SVE::BI__builtin_sve_svpmullb_n_u64:
8381     return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
8382
8383   case SVE::BI__builtin_sve_svdup_n_b8:
8384   case SVE::BI__builtin_sve_svdup_n_b16:
8385   case SVE::BI__builtin_sve_svdup_n_b32:
8386   case SVE::BI__builtin_sve_svdup_n_b64: {
8387     Value *CmpNE =
8388         Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
8389     llvm::ScalableVectorType *OverloadedTy = getSVEType(TypeFlags);
8390     Value *Dup = EmitSVEDupX(CmpNE, OverloadedTy);
8391     return EmitSVEPredicateCast(Dup, cast<llvm::ScalableVectorType>(Ty));
8392   }
8393
8394   case SVE::BI__builtin_sve_svdupq_n_b8:
8395   case SVE::BI__builtin_sve_svdupq_n_b16:
8396   case SVE::BI__builtin_sve_svdupq_n_b32:
8397   case SVE::BI__builtin_sve_svdupq_n_b64:
8398   case SVE::BI__builtin_sve_svdupq_n_u8:
8399   case SVE::BI__builtin_sve_svdupq_n_s8:
8400   case SVE::BI__builtin_sve_svdupq_n_u64:
8401   case SVE::BI__builtin_sve_svdupq_n_f64:
8402   case SVE::BI__builtin_sve_svdupq_n_s64:
8403   case SVE::BI__builtin_sve_svdupq_n_u16:
8404   case SVE::BI__builtin_sve_svdupq_n_f16:
8405   case SVE::BI__builtin_sve_svdupq_n_bf16:
8406   case SVE::BI__builtin_sve_svdupq_n_s16:
8407   case SVE::BI__builtin_sve_svdupq_n_u32:
8408   case SVE::BI__builtin_sve_svdupq_n_f32:
8409   case SVE::BI__builtin_sve_svdupq_n_s32: {
8410     // These builtins are implemented by storing each element to an array and using
8411     // ld1rq to materialize a vector.
8412     unsigned NumOpnds = Ops.size();
8413
8414     bool IsBoolTy =
8415         cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
8416
8417     // For svdupq_n_b* the element type of is an integer of type 128/numelts,
8418     // so that the compare can use the width that is natural for the expected
8419     // number of predicate lanes.
8420     llvm::Type *EltTy = Ops[0]->getType();
8421     if (IsBoolTy)
8422       EltTy = IntegerType::get(getLLVMContext(), SVEBitsPerBlock / NumOpnds);
8423
8424     Address Alloca = CreateTempAlloca(llvm::ArrayType::get(EltTy, NumOpnds),
8425                                      CharUnits::fromQuantity(16));
8426     for (unsigned I = 0; I < NumOpnds; ++I)
8427       Builder.CreateDefaultAlignedStore(
8428           IsBoolTy ? Builder.CreateZExt(Ops[I], EltTy) : Ops[I],
8429           Builder.CreateGEP(Alloca.getPointer(),
8430                             {Builder.getInt64(0), Builder.getInt64(I)}));
8431
8432     SVETypeFlags TypeFlags(Builtin->TypeModifier);
8433     Value *Pred = EmitSVEAllTruePred(TypeFlags);
8434
8435     llvm::Type *OverloadedTy = getSVEVectorForElementType(EltTy);
8436     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_ld1rq, OverloadedTy);
8437     Value *Alloca0 = Builder.CreateGEP(
8438         Alloca.getPointer(), {Builder.getInt64(0), Builder.getInt64(0)});
8439     Value *LD1RQ = Builder.CreateCall(F, {Pred, Alloca0});
8440
8441     if (!IsBoolTy)
8442       return LD1RQ;
8443
8444     // For svdupq_n_b* we need to add an additional 'cmpne' with '0'.
8445     F = CGM.getIntrinsic(NumOpnds == 2 ? Intrinsic::aarch64_sve_cmpne
8446                                        : Intrinsic::aarch64_sve_cmpne_wide,
8447                          OverloadedTy);
8448     Value *Call =
8449         Builder.CreateCall(F, {Pred, LD1RQ, EmitSVEDupX(Builder.getInt64(0))});
8450     return EmitSVEPredicateCast(Call, cast<llvm::ScalableVectorType>(Ty));
8451   }
8452
8453   case SVE::BI__builtin_sve_svpfalse_b:
8454     return ConstantInt::getFalse(Ty);
8455
8456   case SVE::BI__builtin_sve_svlen_bf16:
8457   case SVE::BI__builtin_sve_svlen_f16:
8458   case SVE::BI__builtin_sve_svlen_f32:
8459   case SVE::BI__builtin_sve_svlen_f64:
8460   case SVE::BI__builtin_sve_svlen_s8:
8461   case SVE::BI__builtin_sve_svlen_s16:
8462   case SVE::BI__builtin_sve_svlen_s32:
8463   case SVE::BI__builtin_sve_svlen_s64:
8464   case SVE::BI__builtin_sve_svlen_u8:
8465   case SVE::BI__builtin_sve_svlen_u16:
8466   case SVE::BI__builtin_sve_svlen_u32:
8467   case SVE::BI__builtin_sve_svlen_u64: {
8468     SVETypeFlags TF(Builtin->TypeModifier);
8469     auto VTy = cast<llvm::VectorType>(getSVEType(TF));
8470     auto NumEls = llvm::ConstantInt::get(Ty, VTy->getElementCount().Min);
8471
8472     Function *F = CGM.getIntrinsic(Intrinsic::vscale, Ty);
8473     return Builder.CreateMul(NumEls, Builder.CreateCall(F));
8474   }
8475
8476   case SVE::BI__builtin_sve_svtbl2_u8:
8477   case SVE::BI__builtin_sve_svtbl2_s8:
8478   case SVE::BI__builtin_sve_svtbl2_u16:
8479   case SVE::BI__builtin_sve_svtbl2_s16:
8480   case SVE::BI__builtin_sve_svtbl2_u32:
8481   case SVE::BI__builtin_sve_svtbl2_s32:
8482   case SVE::BI__builtin_sve_svtbl2_u64:
8483   case SVE::BI__builtin_sve_svtbl2_s64:
8484   case SVE::BI__builtin_sve_svtbl2_f16:
8485   case SVE::BI__builtin_sve_svtbl2_bf16:
8486   case SVE::BI__builtin_sve_svtbl2_f32:
8487   case SVE::BI__builtin_sve_svtbl2_f64: {
8488     SVETypeFlags TF(Builtin->TypeModifier);
8489     auto VTy = cast<llvm::VectorType>(getSVEType(TF));
8490     auto TupleTy = llvm::VectorType::get(VTy->getElementType(),
8491                                          VTy->getElementCount() * 2);
8492     Function *FExtr =
8493         CGM.getIntrinsic(Intrinsic::aarch64_sve_tuple_get, {VTy, TupleTy});
8494     Value *V0 = Builder.CreateCall(FExtr, {Ops[0], Builder.getInt32(0)});
8495     Value *V1 = Builder.CreateCall(FExtr, {Ops[0], Builder.getInt32(1)});
8496     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_tbl2, VTy);
8497     return Builder.CreateCall(F, {V0, V1, Ops[1]});
8498   }
8499   }
8500
8501   /// Should not happen
8502   return nullptr;
8503 }
8504
8505 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
8506                                                const CallExpr *E,
8507                                                llvm::Triple::ArchType Arch) {
8508   if (BuiltinID >= AArch64::FirstSVEBuiltin &&
8509       BuiltinID <= AArch64::LastSVEBuiltin)
8510     return EmitAArch64SVEBuiltinExpr(BuiltinID, E);
8511
8512   unsigned HintID = static_cast<unsigned>(-1);
8513   switch (BuiltinID) {
8514   default: break;
8515   case AArch64::BI__builtin_arm_nop:
8516     HintID = 0;
8517     break;
8518   case AArch64::BI__builtin_arm_yield:
8519   case AArch64::BI__yield:
8520     HintID = 1;
8521     break;
8522   case AArch64::BI__builtin_arm_wfe:
8523   case AArch64::BI__wfe:
8524     HintID = 2;
8525     break;
8526   case AArch64::BI__builtin_arm_wfi:
8527   case AArch64::BI__wfi:
8528     HintID = 3;
8529     break;
8530   case AArch64::BI__builtin_arm_sev:
8531   case AArch64::BI__sev:
8532     HintID = 4;
8533     break;
8534   case AArch64::BI__builtin_arm_sevl:
8535   case AArch64::BI__sevl:
8536     HintID = 5;
8537     break;
8538   }
8539
8540   if (HintID != static_cast<unsigned>(-1)) {
8541     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
8542     return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
8543   }
8544
8545   if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
8546     Value *Address         = EmitScalarExpr(E->getArg(0));
8547     Value *RW              = EmitScalarExpr(E->getArg(1));
8548     Value *CacheLevel      = EmitScalarExpr(E->getArg(2));
8549     Value *RetentionPolicy = EmitScalarExpr(E->getArg(3));
8550     Value *IsData          = EmitScalarExpr(E->getArg(4));
8551
8552     Value *Locality = nullptr;
8553     if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) {
8554       // Temporal fetch, needs to convert cache level to locality.
8555       Locality = llvm::ConstantInt::get(Int32Ty,
8556         -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3);
8557     } else {
8558       // Streaming fetch.
8559       Locality = llvm::ConstantInt::get(Int32Ty, 0);
8560     }
8561
8562     // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify
8563     // PLDL3STRM or PLDL2STRM.
8564     Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
8565     return Builder.CreateCall(F, {Address, RW, Locality, IsData});
8566   }
8567
8568   if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
8569     assert((getContext().getTypeSize(E->getType()) == 32) &&
8570            "rbit of unusual size!");
8571     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
8572     return Builder.CreateCall(
8573         CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
8574   }
8575   if (BuiltinID == AArch64::BI__builtin_arm_rbit64) {
8576     assert((getContext().getTypeSize(E->getType()) == 64) &&
8577            "rbit of unusual size!");
8578     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
8579     return Builder.CreateCall(
8580         CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
8581   }
8582
8583   if (BuiltinID == AArch64::BI__builtin_arm_cls) {
8584     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
8585     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls), Arg,
8586                               "cls");
8587   }
8588   if (BuiltinID == AArch64::BI__builtin_arm_cls64) {
8589     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
8590     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls64), Arg,
8591                               "cls");
8592   }
8593
8594   if (BuiltinID == AArch64::BI__builtin_arm_jcvt) {
8595     assert((getContext().getTypeSize(E->getType()) == 32) &&
8596            "__jcvt of unusual size!");
8597     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
8598     return Builder.CreateCall(
8599         CGM.getIntrinsic(Intrinsic::aarch64_fjcvtzs), Arg);
8600   }
8601
8602   if (BuiltinID == AArch64::BI__clear_cache) {
8603     assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
8604     const FunctionDecl *FD = E->getDirectCallee();
8605     Value *Ops[2];
8606     for (unsigned i = 0; i < 2; i++)
8607       Ops[i] = EmitScalarExpr(E->getArg(i));
8608     llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
8609     llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
8610     StringRef Name = FD->getName();
8611     return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
8612   }
8613
8614   if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
8615       BuiltinID == AArch64::BI__builtin_arm_ldaex) &&
8616       getContext().getTypeSize(E->getType()) == 128) {
8617     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
8618                                        ? Intrinsic::aarch64_ldaxp
8619                                        : Intrinsic::aarch64_ldxp);
8620
8621     Value *LdPtr = EmitScalarExpr(E->getArg(0));
8622     Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
8623                                     "ldxp");
8624
8625     Value *Val0 = Builder.CreateExtractValue(Val, 1);
8626     Value *Val1 = Builder.CreateExtractValue(Val, 0);
8627     llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
8628     Val0 = Builder.CreateZExt(Val0, Int128Ty);
8629     Val1 = Builder.CreateZExt(Val1, Int128Ty);
8630
8631     Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
8632     Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
8633     Val = Builder.CreateOr(Val, Val1);
8634     return Builder.CreateBitCast(Val, ConvertType(E->getType()));
8635   } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
8636              BuiltinID == AArch64::BI__builtin_arm_ldaex) {
8637     Value *LoadAddr = EmitScalarExpr(E->getArg(0));
8638
8639     QualType Ty = E->getType();
8640     llvm::Type *RealResTy = ConvertType(Ty);
8641     llvm::Type *PtrTy = llvm::IntegerType::get(
8642         getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
8643     LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
8644
8645     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
8646                                        ? Intrinsic::aarch64_ldaxr
8647                                        : Intrinsic::aarch64_ldxr,
8648                                    PtrTy);
8649     Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
8650
8651     if (RealResTy->isPointerTy())
8652       return Builder.CreateIntToPtr(Val, RealResTy);
8653
8654     llvm::Type *IntResTy = llvm::IntegerType::get(
8655         getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
8656     Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
8657     return Builder.CreateBitCast(Val, RealResTy);
8658   }
8659
8660   if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
8661        BuiltinID == AArch64::BI__builtin_arm_stlex) &&
8662       getContext().getTypeSize(E->getArg(0)->getType()) == 128) {
8663     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
8664                                        ? Intrinsic::aarch64_stlxp
8665                                        : Intrinsic::aarch64_stxp);
8666     llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty);
8667
8668     Address Tmp = CreateMemTemp(E->getArg(0)->getType());
8669     EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true);
8670
8671     Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy));
8672     llvm::Value *Val = Builder.CreateLoad(Tmp);
8673
8674     Value *Arg0 = Builder.CreateExtractValue(Val, 0);
8675     Value *Arg1 = Builder.CreateExtractValue(Val, 1);
8676     Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)),
8677                                          Int8PtrTy);
8678     return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp");
8679   }
8680
8681   if (BuiltinID == AArch64::BI__builtin_arm_strex ||
8682       BuiltinID == AArch64::BI__builtin_arm_stlex) {
8683     Value *StoreVal = EmitScalarExpr(E->getArg(0));
8684     Value *StoreAddr = EmitScalarExpr(E->getArg(1));
8685
8686     QualType Ty = E->getArg(0)->getType();
8687     llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
8688                                                  getContext().getTypeSize(Ty));
8689     StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
8690
8691     if (StoreVal->getType()->isPointerTy())
8692       StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
8693     else {
8694       llvm::Type *IntTy = llvm::IntegerType::get(
8695           getLLVMContext(),
8696           CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
8697       StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
8698       StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
8699     }
8700
8701     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
8702                                        ? Intrinsic::aarch64_stlxr
8703                                        : Intrinsic::aarch64_stxr,
8704                                    StoreAddr->getType());
8705     return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
8706   }
8707
8708   if (BuiltinID == AArch64::BI__getReg) {
8709     Expr::EvalResult Result;
8710     if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
8711       llvm_unreachable("Sema will ensure that the parameter is constant");
8712
8713     llvm::APSInt Value = Result.Val.getInt();
8714     LLVMContext &Context = CGM.getLLVMContext();
8715     std::string Reg = Value == 31 ? "sp" : "x" + Value.toString(10);
8716
8717     llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
8718     llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
8719     llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
8720
8721     llvm::Function *F =
8722         CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
8723     return Builder.CreateCall(F, Metadata);
8724   }
8725
8726   if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
8727     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
8728     return Builder.CreateCall(F);
8729   }
8730
8731   if (BuiltinID == AArch64::BI_ReadWriteBarrier)
8732     return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
8733                                llvm::SyncScope::SingleThread);
8734
8735   // CRC32
8736   Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
8737   switch (BuiltinID) {
8738   case AArch64::BI__builtin_arm_crc32b:
8739     CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
8740   case AArch64::BI__builtin_arm_crc32cb:
8741     CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
8742   case AArch64::BI__builtin_arm_crc32h:
8743     CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
8744   case AArch64::BI__builtin_arm_crc32ch:
8745     CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
8746   case AArch64::BI__builtin_arm_crc32w:
8747     CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
8748   case AArch64::BI__builtin_arm_crc32cw:
8749     CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
8750   case AArch64::BI__builtin_arm_crc32d:
8751     CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
8752   case AArch64::BI__builtin_arm_crc32cd:
8753     CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
8754   }
8755
8756   if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
8757     Value *Arg0 = EmitScalarExpr(E->getArg(0));
8758     Value *Arg1 = EmitScalarExpr(E->getArg(1));
8759     Function *F = CGM.getIntrinsic(CRCIntrinsicID);
8760
8761     llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
8762     Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
8763
8764     return Builder.CreateCall(F, {Arg0, Arg1});
8765   }
8766
8767   // Memory Tagging Extensions (MTE) Intrinsics
8768   Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
8769   switch (BuiltinID) {
8770   case AArch64::BI__builtin_arm_irg:
8771     MTEIntrinsicID = Intrinsic::aarch64_irg; break;
8772   case  AArch64::BI__builtin_arm_addg:
8773     MTEIntrinsicID = Intrinsic::aarch64_addg; break;
8774   case  AArch64::BI__builtin_arm_gmi:
8775     MTEIntrinsicID = Intrinsic::aarch64_gmi; break;
8776   case  AArch64::BI__builtin_arm_ldg:
8777     MTEIntrinsicID = Intrinsic::aarch64_ldg; break;
8778   case AArch64::BI__builtin_arm_stg:
8779     MTEIntrinsicID = Intrinsic::aarch64_stg; break;
8780   case AArch64::BI__builtin_arm_subp:
8781     MTEIntrinsicID = Intrinsic::aarch64_subp; break;
8782   }
8783
8784   if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
8785     llvm::Type *T = ConvertType(E->getType());
8786
8787     if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
8788       Value *Pointer = EmitScalarExpr(E->getArg(0));
8789       Value *Mask = EmitScalarExpr(E->getArg(1));
8790
8791       Pointer = Builder.CreatePointerCast(Pointer, Int8PtrTy);
8792       Mask = Builder.CreateZExt(Mask, Int64Ty);
8793       Value *RV = Builder.CreateCall(
8794                        CGM.getIntrinsic(MTEIntrinsicID), {Pointer, Mask});
8795        return Builder.CreatePointerCast(RV, T);
8796     }
8797     if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
8798       Value *Pointer = EmitScalarExpr(E->getArg(0));
8799       Value *TagOffset = EmitScalarExpr(E->getArg(1));
8800
8801       Pointer = Builder.CreatePointerCast(Pointer, Int8PtrTy);
8802       TagOffset = Builder.CreateZExt(TagOffset, Int64Ty);
8803       Value *RV = Builder.CreateCall(
8804                        CGM.getIntrinsic(MTEIntrinsicID), {Pointer, TagOffset});
8805       return Builder.CreatePointerCast(RV, T);
8806     }
8807     if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
8808       Value *Pointer = EmitScalarExpr(E->getArg(0));
8809       Value *ExcludedMask = EmitScalarExpr(E->getArg(1));
8810
8811       ExcludedMask = Builder.CreateZExt(ExcludedMask, Int64Ty);
8812       Pointer = Builder.CreatePointerCast(Pointer, Int8PtrTy);
8813       return Builder.CreateCall(
8814                        CGM.getIntrinsic(MTEIntrinsicID), {Pointer, ExcludedMask});
8815     }
8816     // Although it is possible to supply a different return
8817     // address (first arg) to this intrinsic, for now we set
8818     // return address same as input address.
8819     if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
8820       Value *TagAddress = EmitScalarExpr(E->getArg(0));
8821       TagAddress = Builder.CreatePointerCast(TagAddress, Int8PtrTy);
8822       Value *RV = Builder.CreateCall(
8823                     CGM.getIntrinsic(MTEIntrinsicID), {TagAddress, TagAddress});
8824       return Builder.CreatePointerCast(RV, T);
8825     }
8826     // Although it is possible to supply a different tag (to set)
8827     // to this intrinsic (as first arg), for now we supply
8828     // the tag that is in input address arg (common use case).
8829     if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
8830         Value *TagAddress = EmitScalarExpr(E->getArg(0));
8831         TagAddress = Builder.CreatePointerCast(TagAddress, Int8PtrTy);
8832         return Builder.CreateCall(
8833                  CGM.getIntrinsic(MTEIntrinsicID), {TagAddress, TagAddress});
8834     }
8835     if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
8836       Value *PointerA = EmitScalarExpr(E->getArg(0));
8837       Value *PointerB = EmitScalarExpr(E->getArg(1));
8838       PointerA = Builder.CreatePointerCast(PointerA, Int8PtrTy);
8839       PointerB = Builder.CreatePointerCast(PointerB, Int8PtrTy);
8840       return Builder.CreateCall(
8841                        CGM.getIntrinsic(MTEIntrinsicID), {PointerA, PointerB});
8842     }
8843   }
8844
8845   if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
8846       BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
8847       BuiltinID == AArch64::BI__builtin_arm_rsrp ||
8848       BuiltinID == AArch64::BI__builtin_arm_wsr ||
8849       BuiltinID == AArch64::BI__builtin_arm_wsr64 ||
8850       BuiltinID == AArch64::BI__builtin_arm_wsrp) {
8851
8852     SpecialRegisterAccessKind AccessKind = Write;
8853     if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
8854         BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
8855         BuiltinID == AArch64::BI__builtin_arm_rsrp)
8856       AccessKind = VolatileRead;
8857
8858     bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp ||
8859                             BuiltinID == AArch64::BI__builtin_arm_wsrp;
8860
8861     bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr &&
8862                    BuiltinID != AArch64::BI__builtin_arm_wsr;
8863
8864     llvm::Type *ValueType;
8865     llvm::Type *RegisterType = Int64Ty;
8866     if (IsPointerBuiltin) {
8867       ValueType = VoidPtrTy;
8868     } else if (Is64Bit) {
8869       ValueType = Int64Ty;
8870     } else {
8871       ValueType = Int32Ty;
8872     }
8873
8874     return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType,
8875                                       AccessKind);
8876   }
8877
8878   if (BuiltinID == AArch64::BI_ReadStatusReg ||
8879       BuiltinID == AArch64::BI_WriteStatusReg) {
8880     LLVMContext &Context = CGM.getLLVMContext();
8881
8882     unsigned SysReg =
8883       E->getArg(0)->EvaluateKnownConstInt(getContext()).getZExtValue();
8884
8885     std::string SysRegStr;
8886     llvm::raw_string_ostream(SysRegStr) <<
8887                        ((1 << 1) | ((SysReg >> 14) & 1))  << ":" <<
8888                        ((SysReg >> 11) & 7)               << ":" <<
8889                        ((SysReg >> 7)  & 15)              << ":" <<
8890                        ((SysReg >> 3)  & 15)              << ":" <<
8891                        ( SysReg        & 7);
8892
8893     llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
8894     llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
8895     llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
8896
8897     llvm::Type *RegisterType = Int64Ty;
8898     llvm::Type *Types[] = { RegisterType };
8899
8900     if (BuiltinID == AArch64::BI_ReadStatusReg) {
8901       llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
8902
8903       return Builder.CreateCall(F, Metadata);
8904     }
8905
8906     llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
8907     llvm::Value *ArgValue = EmitScalarExpr(E->getArg(1));
8908
8909     return Builder.CreateCall(F, { Metadata, ArgValue });
8910   }
8911
8912   if (BuiltinID == AArch64::BI_AddressOfReturnAddress) {
8913     llvm::Function *F =
8914         CGM.getIntrinsic(Intrinsic::addressofreturnaddress, AllocaInt8PtrTy);
8915     return Builder.CreateCall(F);
8916   }
8917
8918   if (BuiltinID == AArch64::BI__builtin_sponentry) {
8919     llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
8920     return Builder.CreateCall(F);
8921   }
8922
8923   // Find out if any arguments are required to be integer constant
8924   // expressions.
8925   unsigned ICEArguments = 0;
8926   ASTContext::GetBuiltinTypeError Error;
8927   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
8928   assert(Error == ASTContext::GE_None && "Should not codegen an error");
8929
8930   llvm::SmallVector<Value*, 4> Ops;
8931   Address PtrOp0 = Address::invalid();
8932   for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
8933     if (i == 0) {
8934       switch (BuiltinID) {
8935       case NEON::BI__builtin_neon_vld1_v:
8936       case NEON::BI__builtin_neon_vld1q_v:
8937       case NEON::BI__builtin_neon_vld1_dup_v:
8938       case NEON::BI__builtin_neon_vld1q_dup_v:
8939       case NEON::BI__builtin_neon_vld1_lane_v:
8940       case NEON::BI__builtin_neon_vld1q_lane_v:
8941       case NEON::BI__builtin_neon_vst1_v:
8942       case NEON::BI__builtin_neon_vst1q_v:
8943       case NEON::BI__builtin_neon_vst1_lane_v:
8944       case NEON::BI__builtin_neon_vst1q_lane_v:
8945         // Get the alignment for the argument in addition to the value;
8946         // we'll use it later.
8947         PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
8948         Ops.push_back(PtrOp0.getPointer());
8949         continue;
8950       }
8951     }
8952     if ((ICEArguments & (1 << i)) == 0) {
8953       Ops.push_back(EmitScalarExpr(E->getArg(i)));
8954     } else {
8955       // If this is required to be a constant, constant fold it so that we know
8956       // that the generated intrinsic gets a ConstantInt.
8957       llvm::APSInt Result;
8958       bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
8959       assert(IsConst && "Constant arg isn't actually constant?");
8960       (void)IsConst;
8961       Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
8962     }
8963   }
8964
8965   auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap);
8966   const ARMVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap(
8967       SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
8968
8969   if (Builtin) {
8970     Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1)));
8971     Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E);
8972     assert(Result && "SISD intrinsic should have been handled");
8973     return Result;
8974   }
8975
8976   llvm::APSInt Result;
8977   const Expr *Arg = E->getArg(E->getNumArgs()-1);
8978   NeonTypeFlags Type(0);
8979   if (Arg->isIntegerConstantExpr(Result, getContext()))
8980     // Determine the type of this overloaded NEON intrinsic.
8981     Type = NeonTypeFlags(Result.getZExtValue());
8982
8983   bool usgn = Type.isUnsigned();
8984   bool quad = Type.isQuad();
8985
8986   // Handle non-overloaded intrinsics first.
8987   switch (BuiltinID) {
8988   default: break;
8989   case NEON::BI__builtin_neon_vabsh_f16:
8990     Ops.push_back(EmitScalarExpr(E->getArg(0)));
8991     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs");
8992   case NEON::BI__builtin_neon_vldrq_p128: {
8993     llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
8994     llvm::Type *Int128PTy = llvm::PointerType::get(Int128Ty, 0);
8995     Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy);
8996     return Builder.CreateAlignedLoad(Int128Ty, Ptr,
8997                                      CharUnits::fromQuantity(16));
8998   }
8999   case NEON::BI__builtin_neon_vstrq_p128: {
9000     llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128);
9001     Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
9002     return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr);
9003   }
9004   case NEON::BI__builtin_neon_vcvts_u32_f32:
9005   case NEON::BI__builtin_neon_vcvtd_u64_f64:
9006     usgn = true;
9007     LLVM_FALLTHROUGH;
9008   case NEON::BI__builtin_neon_vcvts_s32_f32:
9009   case NEON::BI__builtin_neon_vcvtd_s64_f64: {
9010     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9011     bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
9012     llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
9013     llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
9014     Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
9015     if (usgn)
9016       return Builder.CreateFPToUI(Ops[0], InTy);
9017     return Builder.CreateFPToSI(Ops[0], InTy);
9018   }
9019   case NEON::BI__builtin_neon_vcvts_f32_u32:
9020   case NEON::BI__builtin_neon_vcvtd_f64_u64:
9021     usgn = true;
9022     LLVM_FALLTHROUGH;
9023   case NEON::BI__builtin_neon_vcvts_f32_s32:
9024   case NEON::BI__builtin_neon_vcvtd_f64_s64: {
9025     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9026     bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
9027     llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
9028     llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
9029     Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
9030     if (usgn)
9031       return Builder.CreateUIToFP(Ops[0], FTy);
9032     return Builder.CreateSIToFP(Ops[0], FTy);
9033   }
9034   case NEON::BI__builtin_neon_vcvth_f16_u16:
9035   case NEON::BI__builtin_neon_vcvth_f16_u32:
9036   case NEON::BI__builtin_neon_vcvth_f16_u64:
9037     usgn = true;
9038     LLVM_FALLTHROUGH;
9039   case NEON::BI__builtin_neon_vcvth_f16_s16:
9040   case NEON::BI__builtin_neon_vcvth_f16_s32:
9041   case NEON::BI__builtin_neon_vcvth_f16_s64: {
9042     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9043     llvm::Type *FTy = HalfTy;
9044     llvm::Type *InTy;
9045     if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
9046       InTy = Int64Ty;
9047     else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
9048       InTy = Int32Ty;
9049     else
9050       InTy = Int16Ty;
9051     Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
9052     if (usgn)
9053       return Builder.CreateUIToFP(Ops[0], FTy);
9054     return Builder.CreateSIToFP(Ops[0], FTy);
9055   }
9056   case NEON::BI__builtin_neon_vcvth_u16_f16:
9057     usgn = true;
9058     LLVM_FALLTHROUGH;
9059   case NEON::BI__builtin_neon_vcvth_s16_f16: {
9060     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9061     Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
9062     if (usgn)
9063       return Builder.CreateFPToUI(Ops[0], Int16Ty);
9064     return Builder.CreateFPToSI(Ops[0], Int16Ty);
9065   }
9066   case NEON::BI__builtin_neon_vcvth_u32_f16:
9067     usgn = true;
9068     LLVM_FALLTHROUGH;
9069   case NEON::BI__builtin_neon_vcvth_s32_f16: {
9070     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9071     Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
9072     if (usgn)
9073       return Builder.CreateFPToUI(Ops[0], Int32Ty);
9074     return Builder.CreateFPToSI(Ops[0], Int32Ty);
9075   }
9076   case NEON::BI__builtin_neon_vcvth_u64_f16:
9077     usgn = true;
9078     LLVM_FALLTHROUGH;
9079   case NEON::BI__builtin_neon_vcvth_s64_f16: {
9080     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9081     Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
9082     if (usgn)
9083       return Builder.CreateFPToUI(Ops[0], Int64Ty);
9084     return Builder.CreateFPToSI(Ops[0], Int64Ty);
9085   }
9086   case NEON::BI__builtin_neon_vcvtah_u16_f16:
9087   case NEON::BI__builtin_neon_vcvtmh_u16_f16:
9088   case NEON::BI__builtin_neon_vcvtnh_u16_f16:
9089   case NEON::BI__builtin_neon_vcvtph_u16_f16:
9090   case NEON::BI__builtin_neon_vcvtah_s16_f16:
9091   case NEON::BI__builtin_neon_vcvtmh_s16_f16:
9092   case NEON::BI__builtin_neon_vcvtnh_s16_f16:
9093   case NEON::BI__builtin_neon_vcvtph_s16_f16: {
9094     unsigned Int;
9095     llvm::Type* InTy = Int32Ty;
9096     llvm::Type* FTy  = HalfTy;
9097     llvm::Type *Tys[2] = {InTy, FTy};
9098     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9099     switch (BuiltinID) {
9100     default: llvm_unreachable("missing builtin ID in switch!");
9101     case NEON::BI__builtin_neon_vcvtah_u16_f16:
9102       Int = Intrinsic::aarch64_neon_fcvtau; break;
9103     case NEON::BI__builtin_neon_vcvtmh_u16_f16:
9104       Int = Intrinsic::aarch64_neon_fcvtmu; break;
9105     case NEON::BI__builtin_neon_vcvtnh_u16_f16:
9106       Int = Intrinsic::aarch64_neon_fcvtnu; break;
9107     case NEON::BI__builtin_neon_vcvtph_u16_f16:
9108       Int = Intrinsic::aarch64_neon_fcvtpu; break;
9109     case NEON::BI__builtin_neon_vcvtah_s16_f16:
9110       Int = Intrinsic::aarch64_neon_fcvtas; break;
9111     case NEON::BI__builtin_neon_vcvtmh_s16_f16:
9112       Int = Intrinsic::aarch64_neon_fcvtms; break;
9113     case NEON::BI__builtin_neon_vcvtnh_s16_f16:
9114       Int = Intrinsic::aarch64_neon_fcvtns; break;
9115     case NEON::BI__builtin_neon_vcvtph_s16_f16:
9116       Int = Intrinsic::aarch64_neon_fcvtps; break;
9117     }
9118     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt");
9119     return Builder.CreateTrunc(Ops[0], Int16Ty);
9120   }
9121   case NEON::BI__builtin_neon_vcaleh_f16:
9122   case NEON::BI__builtin_neon_vcalth_f16:
9123   case NEON::BI__builtin_neon_vcageh_f16:
9124   case NEON::BI__builtin_neon_vcagth_f16: {
9125     unsigned Int;
9126     llvm::Type* InTy = Int32Ty;
9127     llvm::Type* FTy  = HalfTy;
9128     llvm::Type *Tys[2] = {InTy, FTy};
9129     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9130     switch (BuiltinID) {
9131     default: llvm_unreachable("missing builtin ID in switch!");
9132     case NEON::BI__builtin_neon_vcageh_f16:
9133       Int = Intrinsic::aarch64_neon_facge; break;
9134     case NEON::BI__builtin_neon_vcagth_f16:
9135       Int = Intrinsic::aarch64_neon_facgt; break;
9136     case NEON::BI__builtin_neon_vcaleh_f16:
9137       Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break;
9138     case NEON::BI__builtin_neon_vcalth_f16:
9139       Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break;
9140     }
9141     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg");
9142     return Builder.CreateTrunc(Ops[0], Int16Ty);
9143   }
9144   case NEON::BI__builtin_neon_vcvth_n_s16_f16:
9145   case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
9146     unsigned Int;
9147     llvm::Type* InTy = Int32Ty;
9148     llvm::Type* FTy  = HalfTy;
9149     llvm::Type *Tys[2] = {InTy, FTy};
9150     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9151     switch (BuiltinID) {
9152     default: llvm_unreachable("missing builtin ID in switch!");
9153     case NEON::BI__builtin_neon_vcvth_n_s16_f16:
9154       Int = Intrinsic::aarch64_neon_vcvtfp2fxs; break;
9155     case NEON::BI__builtin_neon_vcvth_n_u16_f16:
9156       Int = Intrinsic::aarch64_neon_vcvtfp2fxu; break;
9157     }
9158     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
9159     return Builder.CreateTrunc(Ops[0], Int16Ty);
9160   }
9161   case NEON::BI__builtin_neon_vcvth_n_f16_s16:
9162   case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
9163     unsigned Int;
9164     llvm::Type* FTy  = HalfTy;
9165     llvm::Type* InTy = Int32Ty;
9166     llvm::Type *Tys[2] = {FTy, InTy};
9167     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9168     switch (BuiltinID) {
9169     default: llvm_unreachable("missing builtin ID in switch!");
9170     case NEON::BI__builtin_neon_vcvth_n_f16_s16:
9171       Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
9172       Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext");
9173       break;
9174     case NEON::BI__builtin_neon_vcvth_n_f16_u16:
9175       Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
9176       Ops[0] = Builder.CreateZExt(Ops[0], InTy);
9177       break;
9178     }
9179     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
9180   }
9181   case NEON::BI__builtin_neon_vpaddd_s64: {
9182     auto *Ty = llvm::FixedVectorType::get(Int64Ty, 2);
9183     Value *Vec = EmitScalarExpr(E->getArg(0));
9184     // The vector is v2f64, so make sure it's bitcast to that.
9185     Vec = Builder.CreateBitCast(Vec, Ty, "v2i64");
9186     llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
9187     llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
9188     Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
9189     Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
9190     // Pairwise addition of a v2f64 into a scalar f64.
9191     return Builder.CreateAdd(Op0, Op1, "vpaddd");
9192   }
9193   case NEON::BI__builtin_neon_vpaddd_f64: {
9194     auto *Ty = llvm::FixedVectorType::get(DoubleTy, 2);
9195     Value *Vec = EmitScalarExpr(E->getArg(0));
9196     // The vector is v2f64, so make sure it's bitcast to that.
9197     Vec = Builder.CreateBitCast(Vec, Ty, "v2f64");
9198     llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
9199     llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
9200     Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
9201     Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
9202     // Pairwise addition of a v2f64 into a scalar f64.
9203     return Builder.CreateFAdd(Op0, Op1, "vpaddd");
9204   }
9205   case NEON::BI__builtin_neon_vpadds_f32: {
9206     auto *Ty = llvm::FixedVectorType::get(FloatTy, 2);
9207     Value *Vec = EmitScalarExpr(E->getArg(0));
9208     // The vector is v2f32, so make sure it's bitcast to that.
9209     Vec = Builder.CreateBitCast(Vec, Ty, "v2f32");
9210     llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
9211     llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
9212     Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
9213     Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
9214     // Pairwise addition of a v2f32 into a scalar f32.
9215     return Builder.CreateFAdd(Op0, Op1, "vpaddd");
9216   }
9217   case NEON::BI__builtin_neon_vceqzd_s64:
9218   case NEON::BI__builtin_neon_vceqzd_f64:
9219   case NEON::BI__builtin_neon_vceqzs_f32:
9220   case NEON::BI__builtin_neon_vceqzh_f16:
9221     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9222     return EmitAArch64CompareBuiltinExpr(
9223         Ops[0], ConvertType(E->getCallReturnType(getContext())),
9224         ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz");
9225   case NEON::BI__builtin_neon_vcgezd_s64:
9226   case NEON::BI__builtin_neon_vcgezd_f64:
9227   case NEON::BI__builtin_neon_vcgezs_f32:
9228   case NEON::BI__builtin_neon_vcgezh_f16:
9229     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9230     return EmitAArch64CompareBuiltinExpr(
9231         Ops[0], ConvertType(E->getCallReturnType(getContext())),
9232         ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez");
9233   case NEON::BI__builtin_neon_vclezd_s64:
9234   case NEON::BI__builtin_neon_vclezd_f64:
9235   case NEON::BI__builtin_neon_vclezs_f32:
9236   case NEON::BI__builtin_neon_vclezh_f16:
9237     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9238     return EmitAArch64CompareBuiltinExpr(
9239         Ops[0], ConvertType(E->getCallReturnType(getContext())),
9240         ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez");
9241   case NEON::BI__builtin_neon_vcgtzd_s64:
9242   case NEON::BI__builtin_neon_vcgtzd_f64:
9243   case NEON::BI__builtin_neon_vcgtzs_f32:
9244   case NEON::BI__builtin_neon_vcgtzh_f16:
9245     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9246     return EmitAArch64CompareBuiltinExpr(
9247         Ops[0], ConvertType(E->getCallReturnType(getContext())),
9248         ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz");
9249   case NEON::BI__builtin_neon_vcltzd_s64:
9250   case NEON::BI__builtin_neon_vcltzd_f64:
9251   case NEON::BI__builtin_neon_vcltzs_f32:
9252   case NEON::BI__builtin_neon_vcltzh_f16:
9253     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9254     return EmitAArch64CompareBuiltinExpr(
9255         Ops[0], ConvertType(E->getCallReturnType(getContext())),
9256         ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz");
9257
9258   case NEON::BI__builtin_neon_vceqzd_u64: {
9259     Ops.push_back(EmitScalarExpr(E->getArg(0)));
9260     Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
9261     Ops[0] =
9262         Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
9263     return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd");
9264   }
9265   case NEON::BI__builtin_neon_vceqd_f64:
9266   case NEON::BI__builtin_neon_vcled_f64:
9267   case NEON::BI__builtin_neon_vcltd_f64:
9268   case NEON::BI__builtin_neon_vcged_f64:
9269   case NEON::BI__builtin_neon_vcgtd_f64: {
9270     llvm::CmpInst::Predicate P;
9271     switch (BuiltinID) {
9272     default: llvm_unreachable("missing builtin ID in switch!");
9273     case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
9274     case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
9275     case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
9276     case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
9277     case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
9278     }
9279     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9280     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
9281     Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
9282     Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
9283     return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
9284   }
9285   case NEON::BI__builtin_neon_vceqs_f32:
9286   case NEON::BI__builtin_neon_vcles_f32:
9287   case NEON::BI__builtin_neon_vclts_f32:
9288   case NEON::BI__builtin_neon_vcges_f32:
9289   case NEON::BI__builtin_neon_vcgts_f32: {
9290     llvm::CmpInst::Predicate P;
9291     switch (BuiltinID) {
9292     default: llvm_unreachable("missing builtin ID in switch!");
9293     case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
9294     case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
9295     case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
9296     case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
9297     case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
9298     }
9299     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9300     Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
9301     Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
9302     Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
9303     return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
9304   }
9305   case NEON::BI__builtin_neon_vceqh_f16:
9306   case NEON::BI__builtin_neon_vcleh_f16:
9307   case NEON::BI__builtin_neon_vclth_f16:
9308   case NEON::BI__builtin_neon_vcgeh_f16:
9309   case NEON::BI__builtin_neon_vcgth_f16: {
9310     llvm::CmpInst::Predicate P;
9311     switch (BuiltinID) {
9312     default: llvm_unreachable("missing builtin ID in switch!");
9313     case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ; break;
9314     case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE; break;
9315     case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT; break;
9316     case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE; break;
9317     case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT; break;
9318     }
9319     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9320     Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
9321     Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
9322     Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
9323     return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd");
9324   }
9325   case NEON::BI__builtin_neon_vceqd_s64:
9326   case NEON::BI__builtin_neon_vceqd_u64:
9327   case NEON::BI__builtin_neon_vcgtd_s64:
9328   case NEON::BI__builtin_neon_vcgtd_u64:
9329   case NEON::BI__builtin_neon_vcltd_s64:
9330   case NEON::BI__builtin_neon_vcltd_u64:
9331   case NEON::BI__builtin_neon_vcged_u64:
9332   case NEON::BI__builtin_neon_vcged_s64:
9333   case NEON::BI__builtin_neon_vcled_u64:
9334   case NEON::BI__builtin_neon_vcled_s64: {
9335     llvm::CmpInst::Predicate P;
9336     switch (BuiltinID) {
9337     default: llvm_unreachable("missing builtin ID in switch!");
9338     case NEON::BI__builtin_neon_vceqd_s64:
9339     case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
9340     case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
9341     case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
9342     case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
9343     case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
9344     case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
9345     case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
9346     case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
9347     case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
9348     }
9349     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9350     Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
9351     Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
9352     Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
9353     return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd");
9354   }
9355   case NEON::BI__builtin_neon_vtstd_s64:
9356   case NEON::BI__builtin_neon_vtstd_u64: {
9357     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9358     Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
9359     Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
9360     Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
9361     Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
9362                                 llvm::Constant::getNullValue(Int64Ty));
9363     return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd");
9364   }
9365   case NEON::BI__builtin_neon_vset_lane_i8:
9366   case NEON::BI__builtin_neon_vset_lane_i16:
9367   case NEON::BI__builtin_neon_vset_lane_i32:
9368   case NEON::BI__builtin_neon_vset_lane_i64:
9369   case NEON::BI__builtin_neon_vset_lane_bf16:
9370   case NEON::BI__builtin_neon_vset_lane_f32:
9371   case NEON::BI__builtin_neon_vsetq_lane_i8:
9372   case NEON::BI__builtin_neon_vsetq_lane_i16:
9373   case NEON::BI__builtin_neon_vsetq_lane_i32:
9374   case NEON::BI__builtin_neon_vsetq_lane_i64:
9375   case NEON::BI__builtin_neon_vsetq_lane_bf16:
9376   case NEON::BI__builtin_neon_vsetq_lane_f32:
9377     Ops.push_back(EmitScalarExpr(E->getArg(2)));
9378     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
9379   case NEON::BI__builtin_neon_vset_lane_f64:
9380     // The vector type needs a cast for the v1f64 variant.
9381     Ops[1] =
9382         Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 1));
9383     Ops.push_back(EmitScalarExpr(E->getArg(2)));
9384     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
9385   case NEON::BI__builtin_neon_vsetq_lane_f64:
9386     // The vector type needs a cast for the v2f64 variant.
9387     Ops[1] =
9388         Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 2));
9389     Ops.push_back(EmitScalarExpr(E->getArg(2)));
9390     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
9391
9392   case NEON::BI__builtin_neon_vget_lane_i8:
9393   case NEON::BI__builtin_neon_vdupb_lane_i8:
9394     Ops[0] =
9395         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 8));
9396     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9397                                         "vget_lane");
9398   case NEON::BI__builtin_neon_vgetq_lane_i8:
9399   case NEON::BI__builtin_neon_vdupb_laneq_i8:
9400     Ops[0] =
9401         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 16));
9402     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9403                                         "vgetq_lane");
9404   case NEON::BI__builtin_neon_vget_lane_i16:
9405   case NEON::BI__builtin_neon_vduph_lane_i16:
9406     Ops[0] =
9407         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 4));
9408     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9409                                         "vget_lane");
9410   case NEON::BI__builtin_neon_vgetq_lane_i16:
9411   case NEON::BI__builtin_neon_vduph_laneq_i16:
9412     Ops[0] =
9413         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 8));
9414     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9415                                         "vgetq_lane");
9416   case NEON::BI__builtin_neon_vget_lane_i32:
9417   case NEON::BI__builtin_neon_vdups_lane_i32:
9418     Ops[0] =
9419         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 2));
9420     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9421                                         "vget_lane");
9422   case NEON::BI__builtin_neon_vdups_lane_f32:
9423     Ops[0] =
9424         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2));
9425     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9426                                         "vdups_lane");
9427   case NEON::BI__builtin_neon_vgetq_lane_i32:
9428   case NEON::BI__builtin_neon_vdups_laneq_i32:
9429     Ops[0] =
9430         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4));
9431     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9432                                         "vgetq_lane");
9433   case NEON::BI__builtin_neon_vget_lane_i64:
9434   case NEON::BI__builtin_neon_vdupd_lane_i64:
9435     Ops[0] =
9436         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 1));
9437     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9438                                         "vget_lane");
9439   case NEON::BI__builtin_neon_vdupd_lane_f64:
9440     Ops[0] =
9441         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1));
9442     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9443                                         "vdupd_lane");
9444   case NEON::BI__builtin_neon_vgetq_lane_i64:
9445   case NEON::BI__builtin_neon_vdupd_laneq_i64:
9446     Ops[0] =
9447         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2));
9448     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9449                                         "vgetq_lane");
9450   case NEON::BI__builtin_neon_vget_lane_f32:
9451     Ops[0] =
9452         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2));
9453     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9454                                         "vget_lane");
9455   case NEON::BI__builtin_neon_vget_lane_f64:
9456     Ops[0] =
9457         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1));
9458     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9459                                         "vget_lane");
9460   case NEON::BI__builtin_neon_vgetq_lane_f32:
9461   case NEON::BI__builtin_neon_vdups_laneq_f32:
9462     Ops[0] =
9463         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 4));
9464     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9465                                         "vgetq_lane");
9466   case NEON::BI__builtin_neon_vgetq_lane_f64:
9467   case NEON::BI__builtin_neon_vdupd_laneq_f64:
9468     Ops[0] =
9469         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 2));
9470     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9471                                         "vgetq_lane");
9472   case NEON::BI__builtin_neon_vaddh_f16:
9473     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9474     return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh");
9475   case NEON::BI__builtin_neon_vsubh_f16:
9476     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9477     return Builder.CreateFSub(Ops[0], Ops[1], "vsubh");
9478   case NEON::BI__builtin_neon_vmulh_f16:
9479     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9480     return Builder.CreateFMul(Ops[0], Ops[1], "vmulh");
9481   case NEON::BI__builtin_neon_vdivh_f16:
9482     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9483     return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh");
9484   case NEON::BI__builtin_neon_vfmah_f16:
9485     // NEON intrinsic puts accumulator first, unlike the LLVM fma.
9486     return emitCallMaybeConstrainedFPBuiltin(
9487         *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
9488         {EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), Ops[0]});
9489   case NEON::BI__builtin_neon_vfmsh_f16: {
9490     // FIXME: This should be an fneg instruction:
9491     Value *Zero = llvm::ConstantFP::getZeroValueForNegation(HalfTy);
9492     Value* Sub = Builder.CreateFSub(Zero, EmitScalarExpr(E->getArg(1)), "vsubh");
9493
9494     // NEON intrinsic puts accumulator first, unlike the LLVM fma.
9495     return emitCallMaybeConstrainedFPBuiltin(
9496         *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
9497         {Sub, EmitScalarExpr(E->getArg(2)), Ops[0]});
9498   }
9499   case NEON::BI__builtin_neon_vaddd_s64:
9500   case NEON::BI__builtin_neon_vaddd_u64:
9501     return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd");
9502   case NEON::BI__builtin_neon_vsubd_s64:
9503   case NEON::BI__builtin_neon_vsubd_u64:
9504     return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd");
9505   case NEON::BI__builtin_neon_vqdmlalh_s16:
9506   case NEON::BI__builtin_neon_vqdmlslh_s16: {
9507     SmallVector<Value *, 2> ProductOps;
9508     ProductOps.push_back(vectorWrapScalar16(Ops[1]));
9509     ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2))));
9510     auto *VTy = llvm::FixedVectorType::get(Int32Ty, 4);
9511     Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
9512                           ProductOps, "vqdmlXl");
9513     Constant *CI = ConstantInt::get(SizeTy, 0);
9514     Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
9515
9516     unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
9517                                         ? Intrinsic::aarch64_neon_sqadd
9518                                         : Intrinsic::aarch64_neon_sqsub;
9519     return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl");
9520   }
9521   case NEON::BI__builtin_neon_vqshlud_n_s64: {
9522     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9523     Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
9524     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
9525                         Ops, "vqshlu_n");
9526   }
9527   case NEON::BI__builtin_neon_vqshld_n_u64:
9528   case NEON::BI__builtin_neon_vqshld_n_s64: {
9529     unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
9530                                    ? Intrinsic::aarch64_neon_uqshl
9531                                    : Intrinsic::aarch64_neon_sqshl;
9532     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9533     Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
9534     return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n");
9535   }
9536   case NEON::BI__builtin_neon_vrshrd_n_u64:
9537   case NEON::BI__builtin_neon_vrshrd_n_s64: {
9538     unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
9539                                    ? Intrinsic::aarch64_neon_urshl
9540                                    : Intrinsic::aarch64_neon_srshl;
9541     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9542     int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
9543     Ops[1] = ConstantInt::get(Int64Ty, -SV);
9544     return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n");
9545   }
9546   case NEON::BI__builtin_neon_vrsrad_n_u64:
9547   case NEON::BI__builtin_neon_vrsrad_n_s64: {
9548     unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
9549                                    ? Intrinsic::aarch64_neon_urshl
9550                                    : Intrinsic::aarch64_neon_srshl;
9551     Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
9552     Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2))));
9553     Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
9554                                 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
9555     return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
9556   }
9557   case NEON::BI__builtin_neon_vshld_n_s64:
9558   case NEON::BI__builtin_neon_vshld_n_u64: {
9559     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
9560     return Builder.CreateShl(
9561         Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
9562   }
9563   case NEON::BI__builtin_neon_vshrd_n_s64: {
9564     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
9565     return Builder.CreateAShr(
9566         Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
9567                                                    Amt->getZExtValue())),
9568         "shrd_n");
9569   }
9570   case NEON::BI__builtin_neon_vshrd_n_u64: {
9571     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
9572     uint64_t ShiftAmt = Amt->getZExtValue();
9573     // Right-shifting an unsigned value by its size yields 0.
9574     if (ShiftAmt == 64)
9575       return ConstantInt::get(Int64Ty, 0);
9576     return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
9577                               "shrd_n");
9578   }
9579   case NEON::BI__builtin_neon_vsrad_n_s64: {
9580     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
9581     Ops[1] = Builder.CreateAShr(
9582         Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
9583                                                    Amt->getZExtValue())),
9584         "shrd_n");
9585     return Builder.CreateAdd(Ops[0], Ops[1]);
9586   }
9587   case NEON::BI__builtin_neon_vsrad_n_u64: {
9588     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
9589     uint64_t ShiftAmt = Amt->getZExtValue();
9590     // Right-shifting an unsigned value by its size yields 0.
9591     // As Op + 0 = Op, return Ops[0] directly.
9592     if (ShiftAmt == 64)
9593       return Ops[0];
9594     Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
9595                                 "shrd_n");
9596     return Builder.CreateAdd(Ops[0], Ops[1]);
9597   }
9598   case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
9599   case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
9600   case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
9601   case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
9602     Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
9603                                           "lane");
9604     SmallVector<Value *, 2> ProductOps;
9605     ProductOps.push_back(vectorWrapScalar16(Ops[1]));
9606     ProductOps.push_back(vectorWrapScalar16(Ops[2]));
9607     auto *VTy = llvm::FixedVectorType::get(Int32Ty, 4);
9608     Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
9609                           ProductOps, "vqdmlXl");
9610     Constant *CI = ConstantInt::get(SizeTy, 0);
9611     Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
9612     Ops.pop_back();
9613
9614     unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
9615                        BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
9616                           ? Intrinsic::aarch64_neon_sqadd
9617                           : Intrinsic::aarch64_neon_sqsub;
9618     return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl");
9619   }
9620   case NEON::BI__builtin_neon_vqdmlals_s32:
9621   case NEON::BI__builtin_neon_vqdmlsls_s32: {
9622     SmallVector<Value *, 2> ProductOps;
9623     ProductOps.push_back(Ops[1]);
9624     ProductOps.push_back(EmitScalarExpr(E->getArg(2)));
9625     Ops[1] =
9626         EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
9627                      ProductOps, "vqdmlXl");
9628
9629     unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
9630                                         ? Intrinsic::aarch64_neon_sqadd
9631                                         : Intrinsic::aarch64_neon_sqsub;
9632     return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl");
9633   }
9634   case NEON::BI__builtin_neon_vqdmlals_lane_s32:
9635   case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
9636   case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
9637   case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
9638     Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
9639                                           "lane");
9640     SmallVector<Value *, 2> ProductOps;
9641     ProductOps.push_back(Ops[1]);
9642     ProductOps.push_back(Ops[2]);
9643     Ops[1] =
9644         EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
9645                      ProductOps, "vqdmlXl");
9646     Ops.pop_back();
9647
9648     unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
9649                        BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
9650                           ? Intrinsic::aarch64_neon_sqadd
9651                           : Intrinsic::aarch64_neon_sqsub;
9652     return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
9653   }
9654   case NEON::BI__builtin_neon_vget_lane_bf16:
9655   case NEON::BI__builtin_neon_vduph_lane_bf16:
9656   case NEON::BI__builtin_neon_vduph_lane_f16: {
9657     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9658                                         "vget_lane");
9659   }
9660   case NEON::BI__builtin_neon_vgetq_lane_bf16:
9661   case NEON::BI__builtin_neon_vduph_laneq_bf16:
9662   case NEON::BI__builtin_neon_vduph_laneq_f16: {
9663     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
9664                                         "vgetq_lane");
9665   }
9666   case AArch64::BI_BitScanForward:
9667   case AArch64::BI_BitScanForward64:
9668     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
9669   case AArch64::BI_BitScanReverse:
9670   case AArch64::BI_BitScanReverse64:
9671     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
9672   case AArch64::BI_InterlockedAnd64:
9673     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
9674   case AArch64::BI_InterlockedExchange64:
9675     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
9676   case AArch64::BI_InterlockedExchangeAdd64:
9677     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
9678   case AArch64::BI_InterlockedExchangeSub64:
9679     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
9680   case AArch64::BI_InterlockedOr64:
9681     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
9682   case AArch64::BI_InterlockedXor64:
9683     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
9684   case AArch64::BI_InterlockedDecrement64:
9685     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
9686   case AArch64::BI_InterlockedIncrement64:
9687     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
9688   case AArch64::BI_InterlockedExchangeAdd8_acq:
9689   case AArch64::BI_InterlockedExchangeAdd16_acq:
9690   case AArch64::BI_InterlockedExchangeAdd_acq:
9691   case AArch64::BI_InterlockedExchangeAdd64_acq:
9692     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_acq, E);
9693   case AArch64::BI_InterlockedExchangeAdd8_rel:
9694   case AArch64::BI_InterlockedExchangeAdd16_rel:
9695   case AArch64::BI_InterlockedExchangeAdd_rel:
9696   case AArch64::BI_InterlockedExchangeAdd64_rel:
9697     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_rel, E);
9698   case AArch64::BI_InterlockedExchangeAdd8_nf:
9699   case AArch64::BI_InterlockedExchangeAdd16_nf:
9700   case AArch64::BI_InterlockedExchangeAdd_nf:
9701   case AArch64::BI_InterlockedExchangeAdd64_nf:
9702     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_nf, E);
9703   case AArch64::BI_InterlockedExchange8_acq:
9704   case AArch64::BI_InterlockedExchange16_acq:
9705   case AArch64::BI_InterlockedExchange_acq:
9706   case AArch64::BI_InterlockedExchange64_acq:
9707     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_acq, E);
9708   case AArch64::BI_InterlockedExchange8_rel:
9709   case AArch64::BI_InterlockedExchange16_rel:
9710   case AArch64::BI_InterlockedExchange_rel:
9711   case AArch64::BI_InterlockedExchange64_rel:
9712     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_rel, E);
9713   case AArch64::BI_InterlockedExchange8_nf:
9714   case AArch64::BI_InterlockedExchange16_nf:
9715   case AArch64::BI_InterlockedExchange_nf:
9716   case AArch64::BI_InterlockedExchange64_nf:
9717     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_nf, E);
9718   case AArch64::BI_InterlockedCompareExchange8_acq:
9719   case AArch64::BI_InterlockedCompareExchange16_acq:
9720   case AArch64::BI_InterlockedCompareExchange_acq:
9721   case AArch64::BI_InterlockedCompareExchange64_acq:
9722     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_acq, E);
9723   case AArch64::BI_InterlockedCompareExchange8_rel:
9724   case AArch64::BI_InterlockedCompareExchange16_rel:
9725   case AArch64::BI_InterlockedCompareExchange_rel:
9726   case AArch64::BI_InterlockedCompareExchange64_rel:
9727     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_rel, E);
9728   case AArch64::BI_InterlockedCompareExchange8_nf:
9729   case AArch64::BI_InterlockedCompareExchange16_nf:
9730   case AArch64::BI_InterlockedCompareExchange_nf:
9731   case AArch64::BI_InterlockedCompareExchange64_nf:
9732     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E);
9733   case AArch64::BI_InterlockedOr8_acq:
9734   case AArch64::BI_InterlockedOr16_acq:
9735   case AArch64::BI_InterlockedOr_acq:
9736   case AArch64::BI_InterlockedOr64_acq:
9737     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_acq, E);
9738   case AArch64::BI_InterlockedOr8_rel:
9739   case AArch64::BI_InterlockedOr16_rel:
9740   case AArch64::BI_InterlockedOr_rel:
9741   case AArch64::BI_InterlockedOr64_rel:
9742     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_rel, E);
9743   case AArch64::BI_InterlockedOr8_nf:
9744   case AArch64::BI_InterlockedOr16_nf:
9745   case AArch64::BI_InterlockedOr_nf:
9746   case AArch64::BI_InterlockedOr64_nf:
9747     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_nf, E);
9748   case AArch64::BI_InterlockedXor8_acq:
9749   case AArch64::BI_InterlockedXor16_acq:
9750   case AArch64::BI_InterlockedXor_acq:
9751   case AArch64::BI_InterlockedXor64_acq:
9752     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_acq, E);
9753   case AArch64::BI_InterlockedXor8_rel:
9754   case AArch64::BI_InterlockedXor16_rel:
9755   case AArch64::BI_InterlockedXor_rel:
9756   case AArch64::BI_InterlockedXor64_rel:
9757     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_rel, E);
9758   case AArch64::BI_InterlockedXor8_nf:
9759   case AArch64::BI_InterlockedXor16_nf:
9760   case AArch64::BI_InterlockedXor_nf:
9761   case AArch64::BI_InterlockedXor64_nf:
9762     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_nf, E);
9763   case AArch64::BI_InterlockedAnd8_acq:
9764   case AArch64::BI_InterlockedAnd16_acq:
9765   case AArch64::BI_InterlockedAnd_acq:
9766   case AArch64::BI_InterlockedAnd64_acq:
9767     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_acq, E);
9768   case AArch64::BI_InterlockedAnd8_rel:
9769   case AArch64::BI_InterlockedAnd16_rel:
9770   case AArch64::BI_InterlockedAnd_rel:
9771   case AArch64::BI_InterlockedAnd64_rel:
9772     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_rel, E);
9773   case AArch64::BI_InterlockedAnd8_nf:
9774   case AArch64::BI_InterlockedAnd16_nf:
9775   case AArch64::BI_InterlockedAnd_nf:
9776   case AArch64::BI_InterlockedAnd64_nf:
9777     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_nf, E);
9778   case AArch64::BI_InterlockedIncrement16_acq:
9779   case AArch64::BI_InterlockedIncrement_acq:
9780   case AArch64::BI_InterlockedIncrement64_acq:
9781     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_acq, E);
9782   case AArch64::BI_InterlockedIncrement16_rel:
9783   case AArch64::BI_InterlockedIncrement_rel:
9784   case AArch64::BI_InterlockedIncrement64_rel:
9785     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_rel, E);
9786   case AArch64::BI_InterlockedIncrement16_nf:
9787   case AArch64::BI_InterlockedIncrement_nf:
9788   case AArch64::BI_InterlockedIncrement64_nf:
9789     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_nf, E);
9790   case AArch64::BI_InterlockedDecrement16_acq:
9791   case AArch64::BI_InterlockedDecrement_acq:
9792   case AArch64::BI_InterlockedDecrement64_acq:
9793     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_acq, E);
9794   case AArch64::BI_InterlockedDecrement16_rel:
9795   case AArch64::BI_InterlockedDecrement_rel:
9796   case AArch64::BI_InterlockedDecrement64_rel:
9797     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_rel, E);
9798   case AArch64::BI_InterlockedDecrement16_nf:
9799   case AArch64::BI_InterlockedDecrement_nf:
9800   case AArch64::BI_InterlockedDecrement64_nf:
9801     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_nf, E);
9802
9803   case AArch64::BI_InterlockedAdd: {
9804     Value *Arg0 = EmitScalarExpr(E->getArg(0));
9805     Value *Arg1 = EmitScalarExpr(E->getArg(1));
9806     AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
9807       AtomicRMWInst::Add, Arg0, Arg1,
9808       llvm::AtomicOrdering::SequentiallyConsistent);
9809     return Builder.CreateAdd(RMWI, Arg1);
9810   }
9811   }
9812
9813   llvm::VectorType *VTy = GetNeonType(this, Type);
9814   llvm::Type *Ty = VTy;
9815   if (!Ty)
9816     return nullptr;
9817
9818   // Not all intrinsics handled by the common case work for AArch64 yet, so only
9819   // defer to common code if it's been added to our special map.
9820   Builtin = findARMVectorIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID,
9821                                         AArch64SIMDIntrinsicsProvenSorted);
9822
9823   if (Builtin)
9824     return EmitCommonNeonBuiltinExpr(
9825         Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
9826         Builtin->NameHint, Builtin->TypeModifier, E, Ops,
9827         /*never use addresses*/ Address::invalid(), Address::invalid(), Arch);
9828
9829   if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch))
9830     return V;
9831
9832   unsigned Int;
9833   switch (BuiltinID) {
9834   default: return nullptr;
9835   case NEON::BI__builtin_neon_vbsl_v:
9836   case NEON::BI__builtin_neon_vbslq_v: {
9837     llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
9838     Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
9839     Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
9840     Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
9841
9842     Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
9843     Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
9844     Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
9845     return Builder.CreateBitCast(Ops[0], Ty);
9846   }
9847   case NEON::BI__builtin_neon_vfma_lane_v:
9848   case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
9849     // The ARM builtins (and instructions) have the addend as the first
9850     // operand, but the 'fma' intrinsics have it last. Swap it around here.
9851     Value *Addend = Ops[0];
9852     Value *Multiplicand = Ops[1];
9853     Value *LaneSource = Ops[2];
9854     Ops[0] = Multiplicand;
9855     Ops[1] = LaneSource;
9856     Ops[2] = Addend;
9857
9858     // Now adjust things to handle the lane access.
9859     auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
9860                          ? llvm::FixedVectorType::get(VTy->getElementType(),
9861                                                       VTy->getNumElements() / 2)
9862                          : VTy;
9863     llvm::Constant *cst = cast<Constant>(Ops[3]);
9864     Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
9865     Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
9866     Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
9867
9868     Ops.pop_back();
9869     Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
9870                                        : Intrinsic::fma;
9871     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
9872   }
9873   case NEON::BI__builtin_neon_vfma_laneq_v: {
9874     llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
9875     // v1f64 fma should be mapped to Neon scalar f64 fma
9876     if (VTy && VTy->getElementType() == DoubleTy) {
9877       Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
9878       Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
9879       llvm::Type *VTy = GetNeonType(this,
9880         NeonTypeFlags(NeonTypeFlags::Float64, false, true));
9881       Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
9882       Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
9883       Value *Result;
9884       Result = emitCallMaybeConstrainedFPBuiltin(
9885           *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
9886           DoubleTy, {Ops[1], Ops[2], Ops[0]});
9887       return Builder.CreateBitCast(Result, Ty);
9888     }
9889     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9890     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9891
9892     auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
9893                                            VTy->getNumElements() * 2);
9894     Ops[2] = Builder.CreateBitCast(Ops[2], STy);
9895     Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
9896                                                cast<ConstantInt>(Ops[3]));
9897     Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
9898
9899     return emitCallMaybeConstrainedFPBuiltin(
9900         *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
9901         {Ops[2], Ops[1], Ops[0]});
9902   }
9903   case NEON::BI__builtin_neon_vfmaq_laneq_v: {
9904     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9905     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9906
9907     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9908     Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
9909     return emitCallMaybeConstrainedFPBuiltin(
9910         *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
9911         {Ops[2], Ops[1], Ops[0]});
9912   }
9913   case NEON::BI__builtin_neon_vfmah_lane_f16:
9914   case NEON::BI__builtin_neon_vfmas_lane_f32:
9915   case NEON::BI__builtin_neon_vfmah_laneq_f16:
9916   case NEON::BI__builtin_neon_vfmas_laneq_f32:
9917   case NEON::BI__builtin_neon_vfmad_lane_f64:
9918   case NEON::BI__builtin_neon_vfmad_laneq_f64: {
9919     Ops.push_back(EmitScalarExpr(E->getArg(3)));
9920     llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
9921     Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
9922     return emitCallMaybeConstrainedFPBuiltin(
9923         *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
9924         {Ops[1], Ops[2], Ops[0]});
9925   }
9926   case NEON::BI__builtin_neon_vmull_v:
9927     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
9928     Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
9929     if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
9930     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
9931   case NEON::BI__builtin_neon_vmax_v:
9932   case NEON::BI__builtin_neon_vmaxq_v:
9933     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
9934     Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
9935     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
9936     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
9937   case NEON::BI__builtin_neon_vmaxh_f16: {
9938     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9939     Int = Intrinsic::aarch64_neon_fmax;
9940     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax");
9941   }
9942   case NEON::BI__builtin_neon_vmin_v:
9943   case NEON::BI__builtin_neon_vminq_v:
9944     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
9945     Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
9946     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
9947     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
9948   case NEON::BI__builtin_neon_vminh_f16: {
9949     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9950     Int = Intrinsic::aarch64_neon_fmin;
9951     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin");
9952   }
9953   case NEON::BI__builtin_neon_vabd_v:
9954   case NEON::BI__builtin_neon_vabdq_v:
9955     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
9956     Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
9957     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
9958     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
9959   case NEON::BI__builtin_neon_vpadal_v:
9960   case NEON::BI__builtin_neon_vpadalq_v: {
9961     unsigned ArgElts = VTy->getNumElements();
9962     llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
9963     unsigned BitWidth = EltTy->getBitWidth();
9964     auto *ArgTy = llvm::FixedVectorType::get(
9965         llvm::IntegerType::get(getLLVMContext(), BitWidth / 2), 2 * ArgElts);
9966     llvm::Type* Tys[2] = { VTy, ArgTy };
9967     Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
9968     SmallVector<llvm::Value*, 1> TmpOps;
9969     TmpOps.push_back(Ops[1]);
9970     Function *F = CGM.getIntrinsic(Int, Tys);
9971     llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal");
9972     llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
9973     return Builder.CreateAdd(tmp, addend);
9974   }
9975   case NEON::BI__builtin_neon_vpmin_v:
9976   case NEON::BI__builtin_neon_vpminq_v:
9977     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
9978     Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
9979     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
9980     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
9981   case NEON::BI__builtin_neon_vpmax_v:
9982   case NEON::BI__builtin_neon_vpmaxq_v:
9983     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
9984     Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
9985     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
9986     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
9987   case NEON::BI__builtin_neon_vminnm_v:
9988   case NEON::BI__builtin_neon_vminnmq_v:
9989     Int = Intrinsic::aarch64_neon_fminnm;
9990     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
9991   case NEON::BI__builtin_neon_vminnmh_f16:
9992     Ops.push_back(EmitScalarExpr(E->getArg(1)));
9993     Int = Intrinsic::aarch64_neon_fminnm;
9994     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vminnm");
9995   case NEON::BI__builtin_neon_vmaxnm_v:
9996   case NEON::BI__builtin_neon_vmaxnmq_v:
9997     Int = Intrinsic::aarch64_neon_fmaxnm;
9998     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
9999   case NEON::BI__builtin_neon_vmaxnmh_f16:
10000     Ops.push_back(EmitScalarExpr(E->getArg(1)));
10001     Int = Intrinsic::aarch64_neon_fmaxnm;
10002     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmaxnm");
10003   case NEON::BI__builtin_neon_vrecpss_f32: {
10004     Ops.push_back(EmitScalarExpr(E->getArg(1)));
10005     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
10006                         Ops, "vrecps");
10007   }
10008   case NEON::BI__builtin_neon_vrecpsd_f64:
10009     Ops.push_back(EmitScalarExpr(E->getArg(1)));
10010     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
10011                         Ops, "vrecps");
10012   case NEON::BI__builtin_neon_vrecpsh_f16:
10013     Ops.push_back(EmitScalarExpr(E->getArg(1)));
10014     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, HalfTy),
10015                         Ops, "vrecps");
10016   case NEON::BI__builtin_neon_vqshrun_n_v:
10017     Int = Intrinsic::aarch64_neon_sqshrun;
10018     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
10019   case NEON::BI__builtin_neon_vqrshrun_n_v:
10020     Int = Intrinsic::aarch64_neon_sqrshrun;
10021     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
10022   case NEON::BI__builtin_neon_vqshrn_n_v:
10023     Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
10024     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
10025   case NEON::BI__builtin_neon_vrshrn_n_v:
10026     Int = Intrinsic::aarch64_neon_rshrn;
10027     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
10028   case NEON::BI__builtin_neon_vqrshrn_n_v:
10029     Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
10030     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
10031   case NEON::BI__builtin_neon_vrndah_f16: {
10032     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10033     Int = Builder.getIsFPConstrained()
10034               ? Intrinsic::experimental_constrained_round
10035               : Intrinsic::round;
10036     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrnda");
10037   }
10038   case NEON::BI__builtin_neon_vrnda_v:
10039   case NEON::BI__builtin_neon_vrndaq_v: {
10040     Int = Builder.getIsFPConstrained()
10041               ? Intrinsic::experimental_constrained_round
10042               : Intrinsic::round;
10043     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
10044   }
10045   case NEON::BI__builtin_neon_vrndih_f16: {
10046     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10047     Int = Builder.getIsFPConstrained()
10048               ? Intrinsic::experimental_constrained_nearbyint
10049               : Intrinsic::nearbyint;
10050     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndi");
10051   }
10052   case NEON::BI__builtin_neon_vrndmh_f16: {
10053     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10054     Int = Builder.getIsFPConstrained()
10055               ? Intrinsic::experimental_constrained_floor
10056               : Intrinsic::floor;
10057     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndm");
10058   }
10059   case NEON::BI__builtin_neon_vrndm_v:
10060   case NEON::BI__builtin_neon_vrndmq_v: {
10061     Int = Builder.getIsFPConstrained()
10062               ? Intrinsic::experimental_constrained_floor
10063               : Intrinsic::floor;
10064     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
10065   }
10066   case NEON::BI__builtin_neon_vrndnh_f16: {
10067     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10068     Int = Intrinsic::aarch64_neon_frintn;
10069     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn");
10070   }
10071   case NEON::BI__builtin_neon_vrndn_v:
10072   case NEON::BI__builtin_neon_vrndnq_v: {
10073     Int = Intrinsic::aarch64_neon_frintn;
10074     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
10075   }
10076   case NEON::BI__builtin_neon_vrndns_f32: {
10077     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10078     Int = Intrinsic::aarch64_neon_frintn;
10079     return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops, "vrndn");
10080   }
10081   case NEON::BI__builtin_neon_vrndph_f16: {
10082     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10083     Int = Builder.getIsFPConstrained()
10084               ? Intrinsic::experimental_constrained_ceil
10085               : Intrinsic::ceil;
10086     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndp");
10087   }
10088   case NEON::BI__builtin_neon_vrndp_v:
10089   case NEON::BI__builtin_neon_vrndpq_v: {
10090     Int = Builder.getIsFPConstrained()
10091               ? Intrinsic::experimental_constrained_ceil
10092               : Intrinsic::ceil;
10093     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
10094   }
10095   case NEON::BI__builtin_neon_vrndxh_f16: {
10096     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10097     Int = Builder.getIsFPConstrained()
10098               ? Intrinsic::experimental_constrained_rint
10099               : Intrinsic::rint;
10100     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndx");
10101   }
10102   case NEON::BI__builtin_neon_vrndx_v:
10103   case NEON::BI__builtin_neon_vrndxq_v: {
10104     Int = Builder.getIsFPConstrained()
10105               ? Intrinsic::experimental_constrained_rint
10106               : Intrinsic::rint;
10107     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
10108   }
10109   case NEON::BI__builtin_neon_vrndh_f16: {
10110     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10111     Int = Builder.getIsFPConstrained()
10112               ? Intrinsic::experimental_constrained_trunc
10113               : Intrinsic::trunc;
10114     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndz");
10115   }
10116   case NEON::BI__builtin_neon_vrnd_v:
10117   case NEON::BI__builtin_neon_vrndq_v: {
10118     Int = Builder.getIsFPConstrained()
10119               ? Intrinsic::experimental_constrained_trunc
10120               : Intrinsic::trunc;
10121     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz");
10122   }
10123   case NEON::BI__builtin_neon_vcvt_f64_v:
10124   case NEON::BI__builtin_neon_vcvtq_f64_v:
10125     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10126     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
10127     return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
10128                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
10129   case NEON::BI__builtin_neon_vcvt_f64_f32: {
10130     assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&
10131            "unexpected vcvt_f64_f32 builtin");
10132     NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
10133     Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
10134
10135     return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
10136   }
10137   case NEON::BI__builtin_neon_vcvt_f32_f64: {
10138     assert(Type.getEltType() == NeonTypeFlags::Float32 &&
10139            "unexpected vcvt_f32_f64 builtin");
10140     NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
10141     Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
10142
10143     return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
10144   }
10145   case NEON::BI__builtin_neon_vcvt_s32_v:
10146   case NEON::BI__builtin_neon_vcvt_u32_v:
10147   case NEON::BI__builtin_neon_vcvt_s64_v:
10148   case NEON::BI__builtin_neon_vcvt_u64_v:
10149   case NEON::BI__builtin_neon_vcvt_s16_v:
10150   case NEON::BI__builtin_neon_vcvt_u16_v:
10151   case NEON::BI__builtin_neon_vcvtq_s32_v:
10152   case NEON::BI__builtin_neon_vcvtq_u32_v:
10153   case NEON::BI__builtin_neon_vcvtq_s64_v:
10154   case NEON::BI__builtin_neon_vcvtq_u64_v:
10155   case NEON::BI__builtin_neon_vcvtq_s16_v:
10156   case NEON::BI__builtin_neon_vcvtq_u16_v: {
10157     Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
10158     if (usgn)
10159       return Builder.CreateFPToUI(Ops[0], Ty);
10160     return Builder.CreateFPToSI(Ops[0], Ty);
10161   }
10162   case NEON::BI__builtin_neon_vcvta_s16_v:
10163   case NEON::BI__builtin_neon_vcvta_u16_v:
10164   case NEON::BI__builtin_neon_vcvta_s32_v:
10165   case NEON::BI__builtin_neon_vcvtaq_s16_v:
10166   case NEON::BI__builtin_neon_vcvtaq_s32_v:
10167   case NEON::BI__builtin_neon_vcvta_u32_v:
10168   case NEON::BI__builtin_neon_vcvtaq_u16_v:
10169   case NEON::BI__builtin_neon_vcvtaq_u32_v:
10170   case NEON::BI__builtin_neon_vcvta_s64_v:
10171   case NEON::BI__builtin_neon_vcvtaq_s64_v:
10172   case NEON::BI__builtin_neon_vcvta_u64_v:
10173   case NEON::BI__builtin_neon_vcvtaq_u64_v: {
10174     Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
10175     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
10176     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
10177   }
10178   case NEON::BI__builtin_neon_vcvtm_s16_v:
10179   case NEON::BI__builtin_neon_vcvtm_s32_v:
10180   case NEON::BI__builtin_neon_vcvtmq_s16_v:
10181   case NEON::BI__builtin_neon_vcvtmq_s32_v:
10182   case NEON::BI__builtin_neon_vcvtm_u16_v:
10183   case NEON::BI__builtin_neon_vcvtm_u32_v:
10184   case NEON::BI__builtin_neon_vcvtmq_u16_v:
10185   case NEON::BI__builtin_neon_vcvtmq_u32_v:
10186   case NEON::BI__builtin_neon_vcvtm_s64_v:
10187   case NEON::BI__builtin_neon_vcvtmq_s64_v:
10188   case NEON::BI__builtin_neon_vcvtm_u64_v:
10189   case NEON::BI__builtin_neon_vcvtmq_u64_v: {
10190     Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
10191     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
10192     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
10193   }
10194   case NEON::BI__builtin_neon_vcvtn_s16_v:
10195   case NEON::BI__builtin_neon_vcvtn_s32_v:
10196   case NEON::BI__builtin_neon_vcvtnq_s16_v:
10197   case NEON::BI__builtin_neon_vcvtnq_s32_v:
10198   case NEON::BI__builtin_neon_vcvtn_u16_v:
10199   case NEON::BI__builtin_neon_vcvtn_u32_v:
10200   case NEON::BI__builtin_neon_vcvtnq_u16_v:
10201   case NEON::BI__builtin_neon_vcvtnq_u32_v:
10202   case NEON::BI__builtin_neon_vcvtn_s64_v:
10203   case NEON::BI__builtin_neon_vcvtnq_s64_v:
10204   case NEON::BI__builtin_neon_vcvtn_u64_v:
10205   case NEON::BI__builtin_neon_vcvtnq_u64_v: {
10206     Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
10207     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
10208     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
10209   }
10210   case NEON::BI__builtin_neon_vcvtp_s16_v:
10211   case NEON::BI__builtin_neon_vcvtp_s32_v:
10212   case NEON::BI__builtin_neon_vcvtpq_s16_v:
10213   case NEON::BI__builtin_neon_vcvtpq_s32_v:
10214   case NEON::BI__builtin_neon_vcvtp_u16_v:
10215   case NEON::BI__builtin_neon_vcvtp_u32_v:
10216   case NEON::BI__builtin_neon_vcvtpq_u16_v:
10217   case NEON::BI__builtin_neon_vcvtpq_u32_v:
10218   case NEON::BI__builtin_neon_vcvtp_s64_v:
10219   case NEON::BI__builtin_neon_vcvtpq_s64_v:
10220   case NEON::BI__builtin_neon_vcvtp_u64_v:
10221   case NEON::BI__builtin_neon_vcvtpq_u64_v: {
10222     Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
10223     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
10224     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
10225   }
10226   case NEON::BI__builtin_neon_vmulx_v:
10227   case NEON::BI__builtin_neon_vmulxq_v: {
10228     Int = Intrinsic::aarch64_neon_fmulx;
10229     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
10230   }
10231   case NEON::BI__builtin_neon_vmulxh_lane_f16:
10232   case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
10233     // vmulx_lane should be mapped to Neon scalar mulx after
10234     // extracting the scalar element
10235     Ops.push_back(EmitScalarExpr(E->getArg(2)));
10236     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
10237     Ops.pop_back();
10238     Int = Intrinsic::aarch64_neon_fmulx;
10239     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmulx");
10240   }
10241   case NEON::BI__builtin_neon_vmul_lane_v:
10242   case NEON::BI__builtin_neon_vmul_laneq_v: {
10243     // v1f64 vmul_lane should be mapped to Neon scalar mul lane
10244     bool Quad = false;
10245     if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
10246       Quad = true;
10247     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
10248     llvm::Type *VTy = GetNeonType(this,
10249       NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
10250     Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
10251     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
10252     Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
10253     return Builder.CreateBitCast(Result, Ty);
10254   }
10255   case NEON::BI__builtin_neon_vnegd_s64:
10256     return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd");
10257   case NEON::BI__builtin_neon_vnegh_f16:
10258     return Builder.CreateFNeg(EmitScalarExpr(E->getArg(0)), "vnegh");
10259   case NEON::BI__builtin_neon_vpmaxnm_v:
10260   case NEON::BI__builtin_neon_vpmaxnmq_v: {
10261     Int = Intrinsic::aarch64_neon_fmaxnmp;
10262     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
10263   }
10264   case NEON::BI__builtin_neon_vpminnm_v:
10265   case NEON::BI__builtin_neon_vpminnmq_v: {
10266     Int = Intrinsic::aarch64_neon_fminnmp;
10267     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
10268   }
10269   case NEON::BI__builtin_neon_vsqrth_f16: {
10270     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10271     Int = Builder.getIsFPConstrained()
10272               ? Intrinsic::experimental_constrained_sqrt
10273               : Intrinsic::sqrt;
10274     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vsqrt");
10275   }
10276   case NEON::BI__builtin_neon_vsqrt_v:
10277   case NEON::BI__builtin_neon_vsqrtq_v: {
10278     Int = Builder.getIsFPConstrained()
10279               ? Intrinsic::experimental_constrained_sqrt
10280               : Intrinsic::sqrt;
10281     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10282     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
10283   }
10284   case NEON::BI__builtin_neon_vrbit_v:
10285   case NEON::BI__builtin_neon_vrbitq_v: {
10286     Int = Intrinsic::aarch64_neon_rbit;
10287     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
10288   }
10289   case NEON::BI__builtin_neon_vaddv_u8:
10290     // FIXME: These are handled by the AArch64 scalar code.
10291     usgn = true;
10292     LLVM_FALLTHROUGH;
10293   case NEON::BI__builtin_neon_vaddv_s8: {
10294     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
10295     Ty = Int32Ty;
10296     VTy = llvm::FixedVectorType::get(Int8Ty, 8);
10297     llvm::Type *Tys[2] = { Ty, VTy };
10298     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10299     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
10300     return Builder.CreateTrunc(Ops[0], Int8Ty);
10301   }
10302   case NEON::BI__builtin_neon_vaddv_u16:
10303     usgn = true;
10304     LLVM_FALLTHROUGH;
10305   case NEON::BI__builtin_neon_vaddv_s16: {
10306     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
10307     Ty = Int32Ty;
10308     VTy = llvm::FixedVectorType::get(Int16Ty, 4);
10309     llvm::Type *Tys[2] = { Ty, VTy };
10310     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10311     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
10312     return Builder.CreateTrunc(Ops[0], Int16Ty);
10313   }
10314   case NEON::BI__builtin_neon_vaddvq_u8:
10315     usgn = true;
10316     LLVM_FALLTHROUGH;
10317   case NEON::BI__builtin_neon_vaddvq_s8: {
10318     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
10319     Ty = Int32Ty;
10320     VTy = llvm::FixedVectorType::get(Int8Ty, 16);
10321     llvm::Type *Tys[2] = { Ty, VTy };
10322     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10323     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
10324     return Builder.CreateTrunc(Ops[0], Int8Ty);
10325   }
10326   case NEON::BI__builtin_neon_vaddvq_u16:
10327     usgn = true;
10328     LLVM_FALLTHROUGH;
10329   case NEON::BI__builtin_neon_vaddvq_s16: {
10330     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
10331     Ty = Int32Ty;
10332     VTy = llvm::FixedVectorType::get(Int16Ty, 8);
10333     llvm::Type *Tys[2] = { Ty, VTy };
10334     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10335     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
10336     return Builder.CreateTrunc(Ops[0], Int16Ty);
10337   }
10338   case NEON::BI__builtin_neon_vmaxv_u8: {
10339     Int = Intrinsic::aarch64_neon_umaxv;
10340     Ty = Int32Ty;
10341     VTy = llvm::FixedVectorType::get(Int8Ty, 8);
10342     llvm::Type *Tys[2] = { Ty, VTy };
10343     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10344     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
10345     return Builder.CreateTrunc(Ops[0], Int8Ty);
10346   }
10347   case NEON::BI__builtin_neon_vmaxv_u16: {
10348     Int = Intrinsic::aarch64_neon_umaxv;
10349     Ty = Int32Ty;
10350     VTy = llvm::FixedVectorType::get(Int16Ty, 4);
10351     llvm::Type *Tys[2] = { Ty, VTy };
10352     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10353     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
10354     return Builder.CreateTrunc(Ops[0], Int16Ty);
10355   }
10356   case NEON::BI__builtin_neon_vmaxvq_u8: {
10357     Int = Intrinsic::aarch64_neon_umaxv;
10358     Ty = Int32Ty;
10359     VTy = llvm::FixedVectorType::get(Int8Ty, 16);
10360     llvm::Type *Tys[2] = { Ty, VTy };
10361     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10362     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
10363     return Builder.CreateTrunc(Ops[0], Int8Ty);
10364   }
10365   case NEON::BI__builtin_neon_vmaxvq_u16: {
10366     Int = Intrinsic::aarch64_neon_umaxv;
10367     Ty = Int32Ty;
10368     VTy = llvm::FixedVectorType::get(Int16Ty, 8);
10369     llvm::Type *Tys[2] = { Ty, VTy };
10370     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10371     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
10372     return Builder.CreateTrunc(Ops[0], Int16Ty);
10373   }
10374   case NEON::BI__builtin_neon_vmaxv_s8: {
10375     Int = Intrinsic::aarch64_neon_smaxv;
10376     Ty = Int32Ty;
10377     VTy = llvm::FixedVectorType::get(Int8Ty, 8);
10378     llvm::Type *Tys[2] = { Ty, VTy };
10379     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10380     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
10381     return Builder.CreateTrunc(Ops[0], Int8Ty);
10382   }
10383   case NEON::BI__builtin_neon_vmaxv_s16: {
10384     Int = Intrinsic::aarch64_neon_smaxv;
10385     Ty = Int32Ty;
10386     VTy = llvm::FixedVectorType::get(Int16Ty, 4);
10387     llvm::Type *Tys[2] = { Ty, VTy };
10388     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10389     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
10390     return Builder.CreateTrunc(Ops[0], Int16Ty);
10391   }
10392   case NEON::BI__builtin_neon_vmaxvq_s8: {
10393     Int = Intrinsic::aarch64_neon_smaxv;
10394     Ty = Int32Ty;
10395     VTy = llvm::FixedVectorType::get(Int8Ty, 16);
10396     llvm::Type *Tys[2] = { Ty, VTy };
10397     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10398     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
10399     return Builder.CreateTrunc(Ops[0], Int8Ty);
10400   }
10401   case NEON::BI__builtin_neon_vmaxvq_s16: {
10402     Int = Intrinsic::aarch64_neon_smaxv;
10403     Ty = Int32Ty;
10404     VTy = llvm::FixedVectorType::get(Int16Ty, 8);
10405     llvm::Type *Tys[2] = { Ty, VTy };
10406     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10407     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
10408     return Builder.CreateTrunc(Ops[0], Int16Ty);
10409   }
10410   case NEON::BI__builtin_neon_vmaxv_f16: {
10411     Int = Intrinsic::aarch64_neon_fmaxv;
10412     Ty = HalfTy;
10413     VTy = llvm::FixedVectorType::get(HalfTy, 4);
10414     llvm::Type *Tys[2] = { Ty, VTy };
10415     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10416     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
10417     return Builder.CreateTrunc(Ops[0], HalfTy);
10418   }
10419   case NEON::BI__builtin_neon_vmaxvq_f16: {
10420     Int = Intrinsic::aarch64_neon_fmaxv;
10421     Ty = HalfTy;
10422     VTy = llvm::FixedVectorType::get(HalfTy, 8);
10423     llvm::Type *Tys[2] = { Ty, VTy };
10424     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10425     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
10426     return Builder.CreateTrunc(Ops[0], HalfTy);
10427   }
10428   case NEON::BI__builtin_neon_vminv_u8: {
10429     Int = Intrinsic::aarch64_neon_uminv;
10430     Ty = Int32Ty;
10431     VTy = llvm::FixedVectorType::get(Int8Ty, 8);
10432     llvm::Type *Tys[2] = { Ty, VTy };
10433     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10434     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
10435     return Builder.CreateTrunc(Ops[0], Int8Ty);
10436   }
10437   case NEON::BI__builtin_neon_vminv_u16: {
10438     Int = Intrinsic::aarch64_neon_uminv;
10439     Ty = Int32Ty;
10440     VTy = llvm::FixedVectorType::get(Int16Ty, 4);
10441     llvm::Type *Tys[2] = { Ty, VTy };
10442     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10443     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
10444     return Builder.CreateTrunc(Ops[0], Int16Ty);
10445   }
10446   case NEON::BI__builtin_neon_vminvq_u8: {
10447     Int = Intrinsic::aarch64_neon_uminv;
10448     Ty = Int32Ty;
10449     VTy = llvm::FixedVectorType::get(Int8Ty, 16);
10450     llvm::Type *Tys[2] = { Ty, VTy };
10451     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10452     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
10453     return Builder.CreateTrunc(Ops[0], Int8Ty);
10454   }
10455   case NEON::BI__builtin_neon_vminvq_u16: {
10456     Int = Intrinsic::aarch64_neon_uminv;
10457     Ty = Int32Ty;
10458     VTy = llvm::FixedVectorType::get(Int16Ty, 8);
10459     llvm::Type *Tys[2] = { Ty, VTy };
10460     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10461     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
10462     return Builder.CreateTrunc(Ops[0], Int16Ty);
10463   }
10464   case NEON::BI__builtin_neon_vminv_s8: {
10465     Int = Intrinsic::aarch64_neon_sminv;
10466     Ty = Int32Ty;
10467     VTy = llvm::FixedVectorType::get(Int8Ty, 8);
10468     llvm::Type *Tys[2] = { Ty, VTy };
10469     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10470     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
10471     return Builder.CreateTrunc(Ops[0], Int8Ty);
10472   }
10473   case NEON::BI__builtin_neon_vminv_s16: {
10474     Int = Intrinsic::aarch64_neon_sminv;
10475     Ty = Int32Ty;
10476     VTy = llvm::FixedVectorType::get(Int16Ty, 4);
10477     llvm::Type *Tys[2] = { Ty, VTy };
10478     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10479     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
10480     return Builder.CreateTrunc(Ops[0], Int16Ty);
10481   }
10482   case NEON::BI__builtin_neon_vminvq_s8: {
10483     Int = Intrinsic::aarch64_neon_sminv;
10484     Ty = Int32Ty;
10485     VTy = llvm::FixedVectorType::get(Int8Ty, 16);
10486     llvm::Type *Tys[2] = { Ty, VTy };
10487     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10488     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
10489     return Builder.CreateTrunc(Ops[0], Int8Ty);
10490   }
10491   case NEON::BI__builtin_neon_vminvq_s16: {
10492     Int = Intrinsic::aarch64_neon_sminv;
10493     Ty = Int32Ty;
10494     VTy = llvm::FixedVectorType::get(Int16Ty, 8);
10495     llvm::Type *Tys[2] = { Ty, VTy };
10496     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10497     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
10498     return Builder.CreateTrunc(Ops[0], Int16Ty);
10499   }
10500   case NEON::BI__builtin_neon_vminv_f16: {
10501     Int = Intrinsic::aarch64_neon_fminv;
10502     Ty = HalfTy;
10503     VTy = llvm::FixedVectorType::get(HalfTy, 4);
10504     llvm::Type *Tys[2] = { Ty, VTy };
10505     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10506     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
10507     return Builder.CreateTrunc(Ops[0], HalfTy);
10508   }
10509   case NEON::BI__builtin_neon_vminvq_f16: {
10510     Int = Intrinsic::aarch64_neon_fminv;
10511     Ty = HalfTy;
10512     VTy = llvm::FixedVectorType::get(HalfTy, 8);
10513     llvm::Type *Tys[2] = { Ty, VTy };
10514     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10515     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
10516     return Builder.CreateTrunc(Ops[0], HalfTy);
10517   }
10518   case NEON::BI__builtin_neon_vmaxnmv_f16: {
10519     Int = Intrinsic::aarch64_neon_fmaxnmv;
10520     Ty = HalfTy;
10521     VTy = llvm::FixedVectorType::get(HalfTy, 4);
10522     llvm::Type *Tys[2] = { Ty, VTy };
10523     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10524     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
10525     return Builder.CreateTrunc(Ops[0], HalfTy);
10526   }
10527   case NEON::BI__builtin_neon_vmaxnmvq_f16: {
10528     Int = Intrinsic::aarch64_neon_fmaxnmv;
10529     Ty = HalfTy;
10530     VTy = llvm::FixedVectorType::get(HalfTy, 8);
10531     llvm::Type *Tys[2] = { Ty, VTy };
10532     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10533     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
10534     return Builder.CreateTrunc(Ops[0], HalfTy);
10535   }
10536   case NEON::BI__builtin_neon_vminnmv_f16: {
10537     Int = Intrinsic::aarch64_neon_fminnmv;
10538     Ty = HalfTy;
10539     VTy = llvm::FixedVectorType::get(HalfTy, 4);
10540     llvm::Type *Tys[2] = { Ty, VTy };
10541     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10542     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
10543     return Builder.CreateTrunc(Ops[0], HalfTy);
10544   }
10545   case NEON::BI__builtin_neon_vminnmvq_f16: {
10546     Int = Intrinsic::aarch64_neon_fminnmv;
10547     Ty = HalfTy;
10548     VTy = llvm::FixedVectorType::get(HalfTy, 8);
10549     llvm::Type *Tys[2] = { Ty, VTy };
10550     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10551     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
10552     return Builder.CreateTrunc(Ops[0], HalfTy);
10553   }
10554   case NEON::BI__builtin_neon_vmul_n_f64: {
10555     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
10556     Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy);
10557     return Builder.CreateFMul(Ops[0], RHS);
10558   }
10559   case NEON::BI__builtin_neon_vaddlv_u8: {
10560     Int = Intrinsic::aarch64_neon_uaddlv;
10561     Ty = Int32Ty;
10562     VTy = llvm::FixedVectorType::get(Int8Ty, 8);
10563     llvm::Type *Tys[2] = { Ty, VTy };
10564     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10565     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
10566     return Builder.CreateTrunc(Ops[0], Int16Ty);
10567   }
10568   case NEON::BI__builtin_neon_vaddlv_u16: {
10569     Int = Intrinsic::aarch64_neon_uaddlv;
10570     Ty = Int32Ty;
10571     VTy = llvm::FixedVectorType::get(Int16Ty, 4);
10572     llvm::Type *Tys[2] = { Ty, VTy };
10573     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10574     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
10575   }
10576   case NEON::BI__builtin_neon_vaddlvq_u8: {
10577     Int = Intrinsic::aarch64_neon_uaddlv;
10578     Ty = Int32Ty;
10579     VTy = llvm::FixedVectorType::get(Int8Ty, 16);
10580     llvm::Type *Tys[2] = { Ty, VTy };
10581     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10582     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
10583     return Builder.CreateTrunc(Ops[0], Int16Ty);
10584   }
10585   case NEON::BI__builtin_neon_vaddlvq_u16: {
10586     Int = Intrinsic::aarch64_neon_uaddlv;
10587     Ty = Int32Ty;
10588     VTy = llvm::FixedVectorType::get(Int16Ty, 8);
10589     llvm::Type *Tys[2] = { Ty, VTy };
10590     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10591     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
10592   }
10593   case NEON::BI__builtin_neon_vaddlv_s8: {
10594     Int = Intrinsic::aarch64_neon_saddlv;
10595     Ty = Int32Ty;
10596     VTy = llvm::FixedVectorType::get(Int8Ty, 8);
10597     llvm::Type *Tys[2] = { Ty, VTy };
10598     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10599     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
10600     return Builder.CreateTrunc(Ops[0], Int16Ty);
10601   }
10602   case NEON::BI__builtin_neon_vaddlv_s16: {
10603     Int = Intrinsic::aarch64_neon_saddlv;
10604     Ty = Int32Ty;
10605     VTy = llvm::FixedVectorType::get(Int16Ty, 4);
10606     llvm::Type *Tys[2] = { Ty, VTy };
10607     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10608     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
10609   }
10610   case NEON::BI__builtin_neon_vaddlvq_s8: {
10611     Int = Intrinsic::aarch64_neon_saddlv;
10612     Ty = Int32Ty;
10613     VTy = llvm::FixedVectorType::get(Int8Ty, 16);
10614     llvm::Type *Tys[2] = { Ty, VTy };
10615     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10616     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
10617     return Builder.CreateTrunc(Ops[0], Int16Ty);
10618   }
10619   case NEON::BI__builtin_neon_vaddlvq_s16: {
10620     Int = Intrinsic::aarch64_neon_saddlv;
10621     Ty = Int32Ty;
10622     VTy = llvm::FixedVectorType::get(Int16Ty, 8);
10623     llvm::Type *Tys[2] = { Ty, VTy };
10624     Ops.push_back(EmitScalarExpr(E->getArg(0)));
10625     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
10626   }
10627   case NEON::BI__builtin_neon_vsri_n_v:
10628   case NEON::BI__builtin_neon_vsriq_n_v: {
10629     Int = Intrinsic::aarch64_neon_vsri;
10630     llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
10631     return EmitNeonCall(Intrin, Ops, "vsri_n");
10632   }
10633   case NEON::BI__builtin_neon_vsli_n_v:
10634   case NEON::BI__builtin_neon_vsliq_n_v: {
10635     Int = Intrinsic::aarch64_neon_vsli;
10636     llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
10637     return EmitNeonCall(Intrin, Ops, "vsli_n");
10638   }
10639   case NEON::BI__builtin_neon_vsra_n_v:
10640   case NEON::BI__builtin_neon_vsraq_n_v:
10641     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10642     Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
10643     return Builder.CreateAdd(Ops[0], Ops[1]);
10644   case NEON::BI__builtin_neon_vrsra_n_v:
10645   case NEON::BI__builtin_neon_vrsraq_n_v: {
10646     Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
10647     SmallVector<llvm::Value*,2> TmpOps;
10648     TmpOps.push_back(Ops[1]);
10649     TmpOps.push_back(Ops[2]);
10650     Function* F = CGM.getIntrinsic(Int, Ty);
10651     llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true);
10652     Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
10653     return Builder.CreateAdd(Ops[0], tmp);
10654   }
10655   case NEON::BI__builtin_neon_vld1_v:
10656   case NEON::BI__builtin_neon_vld1q_v: {
10657     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
10658     return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.getAlignment());
10659   }
10660   case NEON::BI__builtin_neon_vst1_v:
10661   case NEON::BI__builtin_neon_vst1q_v:
10662     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
10663     Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
10664     return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
10665   case NEON::BI__builtin_neon_vld1_lane_v:
10666   case NEON::BI__builtin_neon_vld1q_lane_v: {
10667     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10668     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
10669     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10670     Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
10671                                        PtrOp0.getAlignment());
10672     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
10673   }
10674   case NEON::BI__builtin_neon_vld1_dup_v:
10675   case NEON::BI__builtin_neon_vld1q_dup_v: {
10676     Value *V = UndefValue::get(Ty);
10677     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
10678     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10679     Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
10680                                        PtrOp0.getAlignment());
10681     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
10682     Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
10683     return EmitNeonSplat(Ops[0], CI);
10684   }
10685   case NEON::BI__builtin_neon_vst1_lane_v:
10686   case NEON::BI__builtin_neon_vst1q_lane_v:
10687     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10688     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
10689     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
10690     return Builder.CreateAlignedStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty),
10691                                       PtrOp0.getAlignment());
10692   case NEON::BI__builtin_neon_vld2_v:
10693   case NEON::BI__builtin_neon_vld2q_v: {
10694     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
10695     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
10696     llvm::Type *Tys[2] = { VTy, PTy };
10697     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
10698     Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
10699     Ops[0] = Builder.CreateBitCast(Ops[0],
10700                 llvm::PointerType::getUnqual(Ops[1]->getType()));
10701     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
10702   }
10703   case NEON::BI__builtin_neon_vld3_v:
10704   case NEON::BI__builtin_neon_vld3q_v: {
10705     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
10706     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
10707     llvm::Type *Tys[2] = { VTy, PTy };
10708     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
10709     Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
10710     Ops[0] = Builder.CreateBitCast(Ops[0],
10711                 llvm::PointerType::getUnqual(Ops[1]->getType()));
10712     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
10713   }
10714   case NEON::BI__builtin_neon_vld4_v:
10715   case NEON::BI__builtin_neon_vld4q_v: {
10716     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
10717     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
10718     llvm::Type *Tys[2] = { VTy, PTy };
10719     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
10720     Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
10721     Ops[0] = Builder.CreateBitCast(Ops[0],
10722                 llvm::PointerType::getUnqual(Ops[1]->getType()));
10723     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
10724   }
10725   case NEON::BI__builtin_neon_vld2_dup_v:
10726   case NEON::BI__builtin_neon_vld2q_dup_v: {
10727     llvm::Type *PTy =
10728       llvm::PointerType::getUnqual(VTy->getElementType());
10729     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
10730     llvm::Type *Tys[2] = { VTy, PTy };
10731     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
10732     Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
10733     Ops[0] = Builder.CreateBitCast(Ops[0],
10734                 llvm::PointerType::getUnqual(Ops[1]->getType()));
10735     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
10736   }
10737   case NEON::BI__builtin_neon_vld3_dup_v:
10738   case NEON::BI__builtin_neon_vld3q_dup_v: {
10739     llvm::Type *PTy =
10740       llvm::PointerType::getUnqual(VTy->getElementType());
10741     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
10742     llvm::Type *Tys[2] = { VTy, PTy };
10743     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
10744     Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
10745     Ops[0] = Builder.CreateBitCast(Ops[0],
10746                 llvm::PointerType::getUnqual(Ops[1]->getType()));
10747     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
10748   }
10749   case NEON::BI__builtin_neon_vld4_dup_v:
10750   case NEON::BI__builtin_neon_vld4q_dup_v: {
10751     llvm::Type *PTy =
10752       llvm::PointerType::getUnqual(VTy->getElementType());
10753     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
10754     llvm::Type *Tys[2] = { VTy, PTy };
10755     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
10756     Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
10757     Ops[0] = Builder.CreateBitCast(Ops[0],
10758                 llvm::PointerType::getUnqual(Ops[1]->getType()));
10759     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
10760   }
10761   case NEON::BI__builtin_neon_vld2_lane_v:
10762   case NEON::BI__builtin_neon_vld2q_lane_v: {
10763     llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
10764     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
10765     Ops.push_back(Ops[1]);
10766     Ops.erase(Ops.begin()+1);
10767     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10768     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
10769     Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
10770     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
10771     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
10772     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10773     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
10774   }
10775   case NEON::BI__builtin_neon_vld3_lane_v:
10776   case NEON::BI__builtin_neon_vld3q_lane_v: {
10777     llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
10778     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
10779     Ops.push_back(Ops[1]);
10780     Ops.erase(Ops.begin()+1);
10781     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10782     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
10783     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
10784     Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
10785     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
10786     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
10787     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10788     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
10789   }
10790   case NEON::BI__builtin_neon_vld4_lane_v:
10791   case NEON::BI__builtin_neon_vld4q_lane_v: {
10792     llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
10793     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
10794     Ops.push_back(Ops[1]);
10795     Ops.erase(Ops.begin()+1);
10796     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10797     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
10798     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
10799     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
10800     Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
10801     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane");
10802     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
10803     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10804     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
10805   }
10806   case NEON::BI__builtin_neon_vst2_v:
10807   case NEON::BI__builtin_neon_vst2q_v: {
10808     Ops.push_back(Ops[0]);
10809     Ops.erase(Ops.begin());
10810     llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
10811     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
10812                         Ops, "");
10813   }
10814   case NEON::BI__builtin_neon_vst2_lane_v:
10815   case NEON::BI__builtin_neon_vst2q_lane_v: {
10816     Ops.push_back(Ops[0]);
10817     Ops.erase(Ops.begin());
10818     Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
10819     llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
10820     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
10821                         Ops, "");
10822   }
10823   case NEON::BI__builtin_neon_vst3_v:
10824   case NEON::BI__builtin_neon_vst3q_v: {
10825     Ops.push_back(Ops[0]);
10826     Ops.erase(Ops.begin());
10827     llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
10828     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
10829                         Ops, "");
10830   }
10831   case NEON::BI__builtin_neon_vst3_lane_v:
10832   case NEON::BI__builtin_neon_vst3q_lane_v: {
10833     Ops.push_back(Ops[0]);
10834     Ops.erase(Ops.begin());
10835     Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
10836     llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
10837     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
10838                         Ops, "");
10839   }
10840   case NEON::BI__builtin_neon_vst4_v:
10841   case NEON::BI__builtin_neon_vst4q_v: {
10842     Ops.push_back(Ops[0]);
10843     Ops.erase(Ops.begin());
10844     llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
10845     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
10846                         Ops, "");
10847   }
10848   case NEON::BI__builtin_neon_vst4_lane_v:
10849   case NEON::BI__builtin_neon_vst4q_lane_v: {
10850     Ops.push_back(Ops[0]);
10851     Ops.erase(Ops.begin());
10852     Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
10853     llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
10854     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
10855                         Ops, "");
10856   }
10857   case NEON::BI__builtin_neon_vtrn_v:
10858   case NEON::BI__builtin_neon_vtrnq_v: {
10859     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
10860     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10861     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
10862     Value *SV = nullptr;
10863
10864     for (unsigned vi = 0; vi != 2; ++vi) {
10865       SmallVector<int, 16> Indices;
10866       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
10867         Indices.push_back(i+vi);
10868         Indices.push_back(i+e+vi);
10869       }
10870       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
10871       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
10872       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
10873     }
10874     return SV;
10875   }
10876   case NEON::BI__builtin_neon_vuzp_v:
10877   case NEON::BI__builtin_neon_vuzpq_v: {
10878     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
10879     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10880     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
10881     Value *SV = nullptr;
10882
10883     for (unsigned vi = 0; vi != 2; ++vi) {
10884       SmallVector<int, 16> Indices;
10885       for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
10886         Indices.push_back(2*i+vi);
10887
10888       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
10889       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
10890       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
10891     }
10892     return SV;
10893   }
10894   case NEON::BI__builtin_neon_vzip_v:
10895   case NEON::BI__builtin_neon_vzipq_v: {
10896     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
10897     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10898     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
10899     Value *SV = nullptr;
10900
10901     for (unsigned vi = 0; vi != 2; ++vi) {
10902       SmallVector<int, 16> Indices;
10903       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
10904         Indices.push_back((i + vi*e) >> 1);
10905         Indices.push_back(((i + vi*e) >> 1)+e);
10906       }
10907       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
10908       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
10909       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
10910     }
10911     return SV;
10912   }
10913   case NEON::BI__builtin_neon_vqtbl1q_v: {
10914     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
10915                         Ops, "vtbl1");
10916   }
10917   case NEON::BI__builtin_neon_vqtbl2q_v: {
10918     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
10919                         Ops, "vtbl2");
10920   }
10921   case NEON::BI__builtin_neon_vqtbl3q_v: {
10922     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
10923                         Ops, "vtbl3");
10924   }
10925   case NEON::BI__builtin_neon_vqtbl4q_v: {
10926     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
10927                         Ops, "vtbl4");
10928   }
10929   case NEON::BI__builtin_neon_vqtbx1q_v: {
10930     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
10931                         Ops, "vtbx1");
10932   }
10933   case NEON::BI__builtin_neon_vqtbx2q_v: {
10934     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
10935                         Ops, "vtbx2");
10936   }
10937   case NEON::BI__builtin_neon_vqtbx3q_v: {
10938     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
10939                         Ops, "vtbx3");
10940   }
10941   case NEON::BI__builtin_neon_vqtbx4q_v: {
10942     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
10943                         Ops, "vtbx4");
10944   }
10945   case NEON::BI__builtin_neon_vsqadd_v:
10946   case NEON::BI__builtin_neon_vsqaddq_v: {
10947     Int = Intrinsic::aarch64_neon_usqadd;
10948     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
10949   }
10950   case NEON::BI__builtin_neon_vuqadd_v:
10951   case NEON::BI__builtin_neon_vuqaddq_v: {
10952     Int = Intrinsic::aarch64_neon_suqadd;
10953     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");
10954   }
10955   }
10956 }
10957
10958 Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
10959                                            const CallExpr *E) {
10960   assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
10961           BuiltinID == BPF::BI__builtin_btf_type_id) &&
10962          "unexpected BPF builtin");
10963
10964   switch (BuiltinID) {
10965   default:
10966     llvm_unreachable("Unexpected BPF builtin");
10967   case BPF::BI__builtin_preserve_field_info: {
10968     const Expr *Arg = E->getArg(0);
10969     bool IsBitField = Arg->IgnoreParens()->getObjectKind() == OK_BitField;
10970
10971     if (!getDebugInfo()) {
10972       CGM.Error(E->getExprLoc(),
10973                 "using __builtin_preserve_field_info() without -g");
10974       return IsBitField ? EmitLValue(Arg).getBitFieldPointer()
10975                         : EmitLValue(Arg).getPointer(*this);
10976     }
10977
10978     // Enable underlying preserve_*_access_index() generation.
10979     bool OldIsInPreservedAIRegion = IsInPreservedAIRegion;
10980     IsInPreservedAIRegion = true;
10981     Value *FieldAddr = IsBitField ? EmitLValue(Arg).getBitFieldPointer()
10982                                   : EmitLValue(Arg).getPointer(*this);
10983     IsInPreservedAIRegion = OldIsInPreservedAIRegion;
10984
10985     ConstantInt *C = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
10986     Value *InfoKind = ConstantInt::get(Int64Ty, C->getSExtValue());
10987
10988     // Built the IR for the preserve_field_info intrinsic.
10989     llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
10990         &CGM.getModule(), llvm::Intrinsic::bpf_preserve_field_info,
10991         {FieldAddr->getType()});
10992     return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
10993   }
10994   case BPF::BI__builtin_btf_type_id: {
10995     Value *FieldVal = nullptr;
10996
10997     // The LValue cannot be converted Value in order to be used as the function
10998     // parameter. If it is a structure, it is the "alloca" result of the LValue
10999     // (a pointer) is used in the parameter. If it is a simple type,
11000     // the value will be loaded from its corresponding "alloca" and used as
11001     // the parameter. In our case, let us just get a pointer of the LValue
11002     // since we do not really use the parameter. The purpose of parameter
11003     // is to prevent the generated IR llvm.bpf.btf.type.id intrinsic call,
11004     // which carries metadata, from being changed.
11005     bool IsLValue = E->getArg(0)->isLValue();
11006     if (IsLValue)
11007       FieldVal = EmitLValue(E->getArg(0)).getPointer(*this);
11008     else
11009       FieldVal = EmitScalarExpr(E->getArg(0));
11010
11011     if (!getDebugInfo()) {
11012       CGM.Error(E->getExprLoc(), "using __builtin_btf_type_id() without -g");
11013       return nullptr;
11014     }
11015
11016     // Generate debuginfo type for the first argument.
11017     llvm::DIType *DbgInfo =
11018         getDebugInfo()->getOrCreateStandaloneType(E->getArg(0)->getType(),
11019                                                   E->getArg(0)->getExprLoc());
11020
11021     ConstantInt *Flag = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
11022     Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
11023
11024     // Built the IR for the btf_type_id intrinsic.
11025     //
11026     // In the above, we converted LValue argument to a pointer to LValue.
11027     // For example, the following
11028     //   int v;
11029     //   C1: __builtin_btf_type_id(v, flag);
11030     // will be converted to
11031     //   L1: llvm.bpf.btf.type.id(&v, flag)
11032     // This makes it hard to differentiate from
11033     //   C2: __builtin_btf_type_id(&v, flag);
11034     // to
11035     //   L2: llvm.bpf.btf.type.id(&v, flag)
11036     //
11037     // If both C1 and C2 are present in the code, the llvm may later
11038     // on do CSE on L1 and L2, which will result in incorrect tagged types.
11039     //
11040     // The C1->L1 transformation only happens if the argument of
11041     // __builtin_btf_type_id() is a LValue. So Let us put whether
11042     // the argument is an LValue or not into generated IR. This should
11043     // prevent potential CSE from causing debuginfo type loss.
11044     //
11045     // The generated IR intrinsics will hence look like
11046     //   L1: llvm.bpf.btf.type.id(&v, 1, flag) !di_type_for_{v};
11047     //   L2: llvm.bpf.btf.type.id(&v, 0, flag) !di_type_for_{&v};
11048     Constant *CV = ConstantInt::get(IntTy, IsLValue);
11049     llvm::Function *FnBtfTypeId = llvm::Intrinsic::getDeclaration(
11050         &CGM.getModule(), llvm::Intrinsic::bpf_btf_type_id,
11051         {FieldVal->getType(), CV->getType()});
11052     CallInst *Fn = Builder.CreateCall(FnBtfTypeId, {FieldVal, CV, FlagValue});
11053     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
11054     return Fn;
11055   }
11056   }
11057 }
11058
11059 llvm::Value *CodeGenFunction::
11060 BuildVector(ArrayRef<llvm::Value*> Ops) {
11061   assert((Ops.size() & (Ops.size() - 1)) == 0 &&
11062          "Not a power-of-two sized vector!");
11063   bool AllConstants = true;
11064   for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
11065     AllConstants &= isa<Constant>(Ops[i]);
11066
11067   // If this is a constant vector, create a ConstantVector.
11068   if (AllConstants) {
11069     SmallVector<llvm::Constant*, 16> CstOps;
11070     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
11071       CstOps.push_back(cast<Constant>(Ops[i]));
11072     return llvm::ConstantVector::get(CstOps);
11073   }
11074
11075   // Otherwise, insertelement the values to build the vector.
11076   Value *Result = llvm::UndefValue::get(
11077       llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
11078
11079   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
11080     Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
11081
11082   return Result;
11083 }
11084
11085 // Convert the mask from an integer type to a vector of i1.
11086 static Value *getMaskVecValue(CodeGenFunction &CGF, Value *Mask,
11087                               unsigned NumElts) {
11088
11089   auto *MaskTy = llvm::FixedVectorType::get(
11090       CGF.Builder.getInt1Ty(),
11091       cast<IntegerType>(Mask->getType())->getBitWidth());
11092   Value *MaskVec = CGF.Builder.CreateBitCast(Mask, MaskTy);
11093
11094   // If we have less than 8 elements, then the starting mask was an i8 and
11095   // we need to extract down to the right number of elements.
11096   if (NumElts < 8) {
11097     int Indices[4];
11098     for (unsigned i = 0; i != NumElts; ++i)
11099       Indices[i] = i;
11100     MaskVec = CGF.Builder.CreateShuffleVector(MaskVec, MaskVec,
11101                                              makeArrayRef(Indices, NumElts),
11102                                              "extract");
11103   }
11104   return MaskVec;
11105 }
11106
11107 static Value *EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
11108                                  Align Alignment) {
11109   // Cast the pointer to right type.
11110   Value *Ptr = CGF.Builder.CreateBitCast(Ops[0],
11111                                llvm::PointerType::getUnqual(Ops[1]->getType()));
11112
11113   Value *MaskVec = getMaskVecValue(
11114       CGF, Ops[2], cast<llvm::VectorType>(Ops[1]->getType())->getNumElements());
11115
11116   return CGF.Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
11117 }
11118
11119 static Value *EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
11120                                 Align Alignment) {
11121   // Cast the pointer to right type.
11122   Value *Ptr = CGF.Builder.CreateBitCast(Ops[0],
11123                                llvm::PointerType::getUnqual(Ops[1]->getType()));
11124
11125   Value *MaskVec = getMaskVecValue(
11126       CGF, Ops[2], cast<llvm::VectorType>(Ops[1]->getType())->getNumElements());
11127
11128   return CGF.Builder.CreateMaskedLoad(Ptr, Alignment, MaskVec, Ops[1]);
11129 }
11130
11131 static Value *EmitX86ExpandLoad(CodeGenFunction &CGF,
11132                                 ArrayRef<Value *> Ops) {
11133   auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
11134   llvm::Type *PtrTy = ResultTy->getElementType();
11135
11136   // Cast the pointer to element type.
11137   Value *Ptr = CGF.Builder.CreateBitCast(Ops[0],
11138                                          llvm::PointerType::getUnqual(PtrTy));
11139
11140   Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements());
11141
11142   llvm::Function *F = CGF.CGM.getIntrinsic(Intrinsic::masked_expandload,
11143                                            ResultTy);
11144   return CGF.Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
11145 }
11146
11147 static Value *EmitX86CompressExpand(CodeGenFunction &CGF,
11148                                     ArrayRef<Value *> Ops,
11149                                     bool IsCompress) {
11150   auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
11151
11152   Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements());
11153
11154   Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
11155                                  : Intrinsic::x86_avx512_mask_expand;
11156   llvm::Function *F = CGF.CGM.getIntrinsic(IID, ResultTy);
11157   return CGF.Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
11158 }
11159
11160 static Value *EmitX86CompressStore(CodeGenFunction &CGF,
11161                                    ArrayRef<Value *> Ops) {
11162   auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
11163   llvm::Type *PtrTy = ResultTy->getElementType();
11164
11165   // Cast the pointer to element type.
11166   Value *Ptr = CGF.Builder.CreateBitCast(Ops[0],
11167                                          llvm::PointerType::getUnqual(PtrTy));
11168
11169   Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements());
11170
11171   llvm::Function *F = CGF.CGM.getIntrinsic(Intrinsic::masked_compressstore,
11172                                            ResultTy);
11173   return CGF.Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
11174 }
11175
11176 static Value *EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc,
11177                               ArrayRef<Value *> Ops,
11178                               bool InvertLHS = false) {
11179   unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
11180   Value *LHS = getMaskVecValue(CGF, Ops[0], NumElts);
11181   Value *RHS = getMaskVecValue(CGF, Ops[1], NumElts);
11182
11183   if (InvertLHS)
11184     LHS = CGF.Builder.CreateNot(LHS);
11185
11186   return CGF.Builder.CreateBitCast(CGF.Builder.CreateBinOp(Opc, LHS, RHS),
11187                                    Ops[0]->getType());
11188 }
11189
11190 static Value *EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1,
11191                                  Value *Amt, bool IsRight) {
11192   llvm::Type *Ty = Op0->getType();
11193
11194   // Amount may be scalar immediate, in which case create a splat vector.
11195   // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
11196   // we only care about the lowest log2 bits anyway.
11197   if (Amt->getType() != Ty) {
11198     unsigned NumElts = cast<llvm::VectorType>(Ty)->getNumElements();
11199     Amt = CGF.Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
11200     Amt = CGF.Builder.CreateVectorSplat(NumElts, Amt);
11201   }
11202
11203   unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
11204   Function *F = CGF.CGM.getIntrinsic(IID, Ty);
11205   return CGF.Builder.CreateCall(F, {Op0, Op1, Amt});
11206 }
11207
11208 static Value *EmitX86vpcom(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
11209                            bool IsSigned) {
11210   Value *Op0 = Ops[0];
11211   Value *Op1 = Ops[1];
11212   llvm::Type *Ty = Op0->getType();
11213   uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
11214
11215   CmpInst::Predicate Pred;
11216   switch (Imm) {
11217   case 0x0:
11218     Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
11219     break;
11220   case 0x1:
11221     Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
11222     break;
11223   case 0x2:
11224     Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
11225     break;
11226   case 0x3:
11227     Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
11228     break;
11229   case 0x4:
11230     Pred = ICmpInst::ICMP_EQ;
11231     break;
11232   case 0x5:
11233     Pred = ICmpInst::ICMP_NE;
11234     break;
11235   case 0x6:
11236     return llvm::Constant::getNullValue(Ty); // FALSE
11237   case 0x7:
11238     return llvm::Constant::getAllOnesValue(Ty); // TRUE
11239   default:
11240     llvm_unreachable("Unexpected XOP vpcom/vpcomu predicate");
11241   }
11242
11243   Value *Cmp = CGF.Builder.CreateICmp(Pred, Op0, Op1);
11244   Value *Res = CGF.Builder.CreateSExt(Cmp, Ty);
11245   return Res;
11246 }
11247
11248 static Value *EmitX86Select(CodeGenFunction &CGF,
11249                             Value *Mask, Value *Op0, Value *Op1) {
11250
11251   // If the mask is all ones just return first argument.
11252   if (const auto *C = dyn_cast<Constant>(Mask))
11253     if (C->isAllOnesValue())
11254       return Op0;
11255
11256   Mask = getMaskVecValue(
11257       CGF, Mask, cast<llvm::VectorType>(Op0->getType())->getNumElements());
11258
11259   return CGF.Builder.CreateSelect(Mask, Op0, Op1);
11260 }
11261
11262 static Value *EmitX86ScalarSelect(CodeGenFunction &CGF,
11263                                   Value *Mask, Value *Op0, Value *Op1) {
11264   // If the mask is all ones just return first argument.
11265   if (const auto *C = dyn_cast<Constant>(Mask))
11266     if (C->isAllOnesValue())
11267       return Op0;
11268
11269   auto *MaskTy = llvm::FixedVectorType::get(
11270       CGF.Builder.getInt1Ty(), Mask->getType()->getIntegerBitWidth());
11271   Mask = CGF.Builder.CreateBitCast(Mask, MaskTy);
11272   Mask = CGF.Builder.CreateExtractElement(Mask, (uint64_t)0);
11273   return CGF.Builder.CreateSelect(Mask, Op0, Op1);
11274 }
11275
11276 static Value *EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp,
11277                                          unsigned NumElts, Value *MaskIn) {
11278   if (MaskIn) {
11279     const auto *C = dyn_cast<Constant>(MaskIn);
11280     if (!C || !C->isAllOnesValue())
11281       Cmp = CGF.Builder.CreateAnd(Cmp, getMaskVecValue(CGF, MaskIn, NumElts));
11282   }
11283
11284   if (NumElts < 8) {
11285     int Indices[8];
11286     for (unsigned i = 0; i != NumElts; ++i)
11287       Indices[i] = i;
11288     for (unsigned i = NumElts; i != 8; ++i)
11289       Indices[i] = i % NumElts + NumElts;
11290     Cmp = CGF.Builder.CreateShuffleVector(
11291         Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
11292   }
11293
11294   return CGF.Builder.CreateBitCast(Cmp,
11295                                    IntegerType::get(CGF.getLLVMContext(),
11296                                                     std::max(NumElts, 8U)));
11297 }
11298
11299 static Value *EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC,
11300                                    bool Signed, ArrayRef<Value *> Ops) {
11301   assert((Ops.size() == 2 || Ops.size() == 4) &&
11302          "Unexpected number of arguments");
11303   unsigned NumElts =
11304       cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
11305   Value *Cmp;
11306
11307   if (CC == 3) {
11308     Cmp = Constant::getNullValue(
11309         llvm::FixedVectorType::get(CGF.Builder.getInt1Ty(), NumElts));
11310   } else if (CC == 7) {
11311     Cmp = Constant::getAllOnesValue(
11312         llvm::FixedVectorType::get(CGF.Builder.getInt1Ty(), NumElts));
11313   } else {
11314     ICmpInst::Predicate Pred;
11315     switch (CC) {
11316     default: llvm_unreachable("Unknown condition code");
11317     case 0: Pred = ICmpInst::ICMP_EQ;  break;
11318     case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
11319     case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
11320     case 4: Pred = ICmpInst::ICMP_NE;  break;
11321     case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
11322     case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
11323     }
11324     Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]);
11325   }
11326
11327   Value *MaskIn = nullptr;
11328   if (Ops.size() == 4)
11329     MaskIn = Ops[3];
11330
11331   return EmitX86MaskedCompareResult(CGF, Cmp, NumElts, MaskIn);
11332 }
11333
11334 static Value *EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In) {
11335   Value *Zero = Constant::getNullValue(In->getType());
11336   return EmitX86MaskedCompare(CGF, 1, true, { In, Zero });
11337 }
11338
11339 static Value *EmitX86ConvertIntToFp(CodeGenFunction &CGF,
11340                                     ArrayRef<Value *> Ops, bool IsSigned) {
11341   unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
11342   llvm::Type *Ty = Ops[1]->getType();
11343
11344   Value *Res;
11345   if (Rnd != 4) {
11346     Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
11347                                  : Intrinsic::x86_avx512_uitofp_round;
11348     Function *F = CGF.CGM.getIntrinsic(IID, { Ty, Ops[0]->getType() });
11349     Res = CGF.Builder.CreateCall(F, { Ops[0], Ops[3] });
11350   } else {
11351     Res = IsSigned ? CGF.Builder.CreateSIToFP(Ops[0], Ty)
11352                    : CGF.Builder.CreateUIToFP(Ops[0], Ty);
11353   }
11354
11355   return EmitX86Select(CGF, Ops[2], Res, Ops[1]);
11356 }
11357
11358 static Value *EmitX86Abs(CodeGenFunction &CGF, ArrayRef<Value *> Ops) {
11359
11360   llvm::Type *Ty = Ops[0]->getType();
11361   Value *Zero = llvm::Constant::getNullValue(Ty);
11362   Value *Sub = CGF.Builder.CreateSub(Zero, Ops[0]);
11363   Value *Cmp = CGF.Builder.CreateICmp(ICmpInst::ICMP_SGT, Ops[0], Zero);
11364   Value *Res = CGF.Builder.CreateSelect(Cmp, Ops[0], Sub);
11365   return Res;
11366 }
11367
11368 static Value *EmitX86MinMax(CodeGenFunction &CGF, ICmpInst::Predicate Pred,
11369                             ArrayRef<Value *> Ops) {
11370   Value *Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]);
11371   Value *Res = CGF.Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
11372
11373   assert(Ops.size() == 2);
11374   return Res;
11375 }
11376
11377 // Lowers X86 FMA intrinsics to IR.
11378 static Value *EmitX86FMAExpr(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
11379                              unsigned BuiltinID, bool IsAddSub) {
11380
11381   bool Subtract = false;
11382   Intrinsic::ID IID = Intrinsic::not_intrinsic;
11383   switch (BuiltinID) {
11384   default: break;
11385   case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
11386     Subtract = true;
11387     LLVM_FALLTHROUGH;
11388   case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
11389   case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
11390   case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
11391     IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512; break;
11392   case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
11393     Subtract = true;
11394     LLVM_FALLTHROUGH;
11395   case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
11396   case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
11397   case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
11398     IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512; break;
11399   case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
11400     Subtract = true;
11401     LLVM_FALLTHROUGH;
11402   case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
11403   case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
11404   case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
11405     IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
11406     break;
11407   case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
11408     Subtract = true;
11409     LLVM_FALLTHROUGH;
11410   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
11411   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
11412   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
11413     IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
11414     break;
11415   }
11416
11417   Value *A = Ops[0];
11418   Value *B = Ops[1];
11419   Value *C = Ops[2];
11420
11421   if (Subtract)
11422     C = CGF.Builder.CreateFNeg(C);
11423
11424   Value *Res;
11425
11426   // Only handle in case of _MM_FROUND_CUR_DIRECTION/4 (no rounding).
11427   if (IID != Intrinsic::not_intrinsic &&
11428       (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 ||
11429        IsAddSub)) {
11430     Function *Intr = CGF.CGM.getIntrinsic(IID);
11431     Res = CGF.Builder.CreateCall(Intr, {A, B, C, Ops.back() });
11432   } else {
11433     llvm::Type *Ty = A->getType();
11434     Function *FMA;
11435     if (CGF.Builder.getIsFPConstrained()) {
11436       FMA = CGF.CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
11437       Res = CGF.Builder.CreateConstrainedFPCall(FMA, {A, B, C});
11438     } else {
11439       FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ty);
11440       Res = CGF.Builder.CreateCall(FMA, {A, B, C});
11441     }
11442   }
11443
11444   // Handle any required masking.
11445   Value *MaskFalseVal = nullptr;
11446   switch (BuiltinID) {
11447   case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
11448   case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
11449   case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
11450   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
11451     MaskFalseVal = Ops[0];
11452     break;
11453   case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
11454   case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
11455   case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
11456   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
11457     MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
11458     break;
11459   case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
11460   case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
11461   case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
11462   case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
11463   case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
11464   case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
11465   case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
11466   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
11467     MaskFalseVal = Ops[2];
11468     break;
11469   }
11470
11471   if (MaskFalseVal)
11472     return EmitX86Select(CGF, Ops[3], Res, MaskFalseVal);
11473
11474   return Res;
11475 }
11476
11477 static Value *
11478 EmitScalarFMAExpr(CodeGenFunction &CGF, MutableArrayRef<Value *> Ops,
11479                   Value *Upper, bool ZeroMask = false, unsigned PTIdx = 0,
11480                   bool NegAcc = false) {
11481   unsigned Rnd = 4;
11482   if (Ops.size() > 4)
11483     Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
11484
11485   if (NegAcc)
11486     Ops[2] = CGF.Builder.CreateFNeg(Ops[2]);
11487
11488   Ops[0] = CGF.Builder.CreateExtractElement(Ops[0], (uint64_t)0);
11489   Ops[1] = CGF.Builder.CreateExtractElement(Ops[1], (uint64_t)0);
11490   Ops[2] = CGF.Builder.CreateExtractElement(Ops[2], (uint64_t)0);
11491   Value *Res;
11492   if (Rnd != 4) {
11493     Intrinsic::ID IID = Ops[0]->getType()->getPrimitiveSizeInBits() == 32 ?
11494                         Intrinsic::x86_avx512_vfmadd_f32 :
11495                         Intrinsic::x86_avx512_vfmadd_f64;
11496     Res = CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IID),
11497                                  {Ops[0], Ops[1], Ops[2], Ops[4]});
11498   } else if (CGF.Builder.getIsFPConstrained()) {
11499     Function *FMA = CGF.CGM.getIntrinsic(
11500         Intrinsic::experimental_constrained_fma, Ops[0]->getType());
11501     Res = CGF.Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
11502   } else {
11503     Function *FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ops[0]->getType());
11504     Res = CGF.Builder.CreateCall(FMA, Ops.slice(0, 3));
11505   }
11506   // If we have more than 3 arguments, we need to do masking.
11507   if (Ops.size() > 3) {
11508     Value *PassThru = ZeroMask ? Constant::getNullValue(Res->getType())
11509                                : Ops[PTIdx];
11510
11511     // If we negated the accumulator and the its the PassThru value we need to
11512     // bypass the negate. Conveniently Upper should be the same thing in this
11513     // case.
11514     if (NegAcc && PTIdx == 2)
11515       PassThru = CGF.Builder.CreateExtractElement(Upper, (uint64_t)0);
11516
11517     Res = EmitX86ScalarSelect(CGF, Ops[3], Res, PassThru);
11518   }
11519   return CGF.Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
11520 }
11521
11522 static Value *EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned,
11523                            ArrayRef<Value *> Ops) {
11524   llvm::Type *Ty = Ops[0]->getType();
11525   // Arguments have a vXi32 type so cast to vXi64.
11526   Ty = llvm::FixedVectorType::get(CGF.Int64Ty,
11527                                   Ty->getPrimitiveSizeInBits() / 64);
11528   Value *LHS = CGF.Builder.CreateBitCast(Ops[0], Ty);
11529   Value *RHS = CGF.Builder.CreateBitCast(Ops[1], Ty);
11530
11531   if (IsSigned) {
11532     // Shift left then arithmetic shift right.
11533     Constant *ShiftAmt = ConstantInt::get(Ty, 32);
11534     LHS = CGF.Builder.CreateShl(LHS, ShiftAmt);
11535     LHS = CGF.Builder.CreateAShr(LHS, ShiftAmt);
11536     RHS = CGF.Builder.CreateShl(RHS, ShiftAmt);
11537     RHS = CGF.Builder.CreateAShr(RHS, ShiftAmt);
11538   } else {
11539     // Clear the upper bits.
11540     Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
11541     LHS = CGF.Builder.CreateAnd(LHS, Mask);
11542     RHS = CGF.Builder.CreateAnd(RHS, Mask);
11543   }
11544
11545   return CGF.Builder.CreateMul(LHS, RHS);
11546 }
11547
11548 // Emit a masked pternlog intrinsic. This only exists because the header has to
11549 // use a macro and we aren't able to pass the input argument to a pternlog
11550 // builtin and a select builtin without evaluating it twice.
11551 static Value *EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask,
11552                              ArrayRef<Value *> Ops) {
11553   llvm::Type *Ty = Ops[0]->getType();
11554
11555   unsigned VecWidth = Ty->getPrimitiveSizeInBits();
11556   unsigned EltWidth = Ty->getScalarSizeInBits();
11557   Intrinsic::ID IID;
11558   if (VecWidth == 128 && EltWidth == 32)
11559     IID = Intrinsic::x86_avx512_pternlog_d_128;
11560   else if (VecWidth == 256 && EltWidth == 32)
11561     IID = Intrinsic::x86_avx512_pternlog_d_256;
11562   else if (VecWidth == 512 && EltWidth == 32)
11563     IID = Intrinsic::x86_avx512_pternlog_d_512;
11564   else if (VecWidth == 128 && EltWidth == 64)
11565     IID = Intrinsic::x86_avx512_pternlog_q_128;
11566   else if (VecWidth == 256 && EltWidth == 64)
11567     IID = Intrinsic::x86_avx512_pternlog_q_256;
11568   else if (VecWidth == 512 && EltWidth == 64)
11569     IID = Intrinsic::x86_avx512_pternlog_q_512;
11570   else
11571     llvm_unreachable("Unexpected intrinsic");
11572
11573   Value *Ternlog = CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IID),
11574                                           Ops.drop_back());
11575   Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
11576   return EmitX86Select(CGF, Ops[4], Ternlog, PassThru);
11577 }
11578
11579 static Value *EmitX86SExtMask(CodeGenFunction &CGF, Value *Op,
11580                               llvm::Type *DstTy) {
11581   unsigned NumberOfElements = cast<llvm::VectorType>(DstTy)->getNumElements();
11582   Value *Mask = getMaskVecValue(CGF, Op, NumberOfElements);
11583   return CGF.Builder.CreateSExt(Mask, DstTy, "vpmovm2");
11584 }
11585
11586 // Emit addition or subtraction with signed/unsigned saturation.
11587 static Value *EmitX86AddSubSatExpr(CodeGenFunction &CGF,
11588                                    ArrayRef<Value *> Ops, bool IsSigned,
11589                                    bool IsAddition) {
11590   Intrinsic::ID IID =
11591       IsSigned ? (IsAddition ? Intrinsic::sadd_sat : Intrinsic::ssub_sat)
11592                : (IsAddition ? Intrinsic::uadd_sat : Intrinsic::usub_sat);
11593   llvm::Function *F = CGF.CGM.getIntrinsic(IID, Ops[0]->getType());
11594   return CGF.Builder.CreateCall(F, {Ops[0], Ops[1]});
11595 }
11596
11597 Value *CodeGenFunction::EmitX86CpuIs(const CallExpr *E) {
11598   const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
11599   StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
11600   return EmitX86CpuIs(CPUStr);
11601 }
11602
11603 // Convert F16 halfs to floats.
11604 static Value *EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF,
11605                                        ArrayRef<Value *> Ops,
11606                                        llvm::Type *DstTy) {
11607   assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
11608          "Unknown cvtph2ps intrinsic");
11609
11610   // If the SAE intrinsic doesn't use default rounding then we can't upgrade.
11611   if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
11612     Function *F =
11613         CGF.CGM.getIntrinsic(Intrinsic::x86_avx512_mask_vcvtph2ps_512);
11614     return CGF.Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
11615   }
11616
11617   unsigned NumDstElts = cast<llvm::VectorType>(DstTy)->getNumElements();
11618   Value *Src = Ops[0];
11619
11620   // Extract the subvector.
11621   if (NumDstElts != cast<llvm::VectorType>(Src->getType())->getNumElements()) {
11622     assert(NumDstElts == 4 && "Unexpected vector size");
11623     Src = CGF.Builder.CreateShuffleVector(Src, UndefValue::get(Src->getType()),
11624                                           ArrayRef<int>{0, 1, 2, 3});
11625   }
11626
11627   // Bitcast from vXi16 to vXf16.
11628   auto *HalfTy = llvm::FixedVectorType::get(
11629       llvm::Type::getHalfTy(CGF.getLLVMContext()), NumDstElts);
11630   Src = CGF.Builder.CreateBitCast(Src, HalfTy);
11631
11632   // Perform the fp-extension.
11633   Value *Res = CGF.Builder.CreateFPExt(Src, DstTy, "cvtph2ps");
11634
11635   if (Ops.size() >= 3)
11636     Res = EmitX86Select(CGF, Ops[2], Res, Ops[1]);
11637   return Res;
11638 }
11639
11640 // Convert a BF16 to a float.
11641 static Value *EmitX86CvtBF16ToFloatExpr(CodeGenFunction &CGF,
11642                                         const CallExpr *E,
11643                                         ArrayRef<Value *> Ops) {
11644   llvm::Type *Int32Ty = CGF.Builder.getInt32Ty();
11645   Value *ZeroExt = CGF.Builder.CreateZExt(Ops[0], Int32Ty);
11646   Value *Shl = CGF.Builder.CreateShl(ZeroExt, 16);
11647   llvm::Type *ResultType = CGF.ConvertType(E->getType());
11648   Value *BitCast = CGF.Builder.CreateBitCast(Shl, ResultType);
11649   return BitCast;
11650 }
11651
11652 Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
11653
11654   llvm::Type *Int32Ty = Builder.getInt32Ty();
11655
11656   // Matching the struct layout from the compiler-rt/libgcc structure that is
11657   // filled in:
11658   // unsigned int __cpu_vendor;
11659   // unsigned int __cpu_type;
11660   // unsigned int __cpu_subtype;
11661   // unsigned int __cpu_features[1];
11662   llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
11663                                           llvm::ArrayType::get(Int32Ty, 1));
11664
11665   // Grab the global __cpu_model.
11666   llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model");
11667   cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(true);
11668
11669   // Calculate the index needed to access the correct field based on the
11670   // range. Also adjust the expected value.
11671   unsigned Index;
11672   unsigned Value;
11673   std::tie(Index, Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
11674 #define X86_VENDOR(ENUM, STRING)                                               \
11675   .Case(STRING, {0u, static_cast<unsigned>(llvm::X86::ENUM)})
11676 #define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)                                        \
11677   .Case(ALIAS, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
11678 #define X86_CPU_TYPE(ENUM, STR)                                                \
11679   .Case(STR, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
11680 #define X86_CPU_SUBTYPE(ENUM, STR)                                             \
11681   .Case(STR, {2u, static_cast<unsigned>(llvm::X86::ENUM)})
11682 #include "llvm/Support/X86TargetParser.def"
11683                                .Default({0, 0});
11684   assert(Value != 0 && "Invalid CPUStr passed to CpuIs");
11685
11686   // Grab the appropriate field from __cpu_model.
11687   llvm::Value *Idxs[] = {ConstantInt::get(Int32Ty, 0),
11688                          ConstantInt::get(Int32Ty, Index)};
11689   llvm::Value *CpuValue = Builder.CreateGEP(STy, CpuModel, Idxs);
11690   CpuValue = Builder.CreateAlignedLoad(CpuValue, CharUnits::fromQuantity(4));
11691
11692   // Check the value of the field against the requested value.
11693   return Builder.CreateICmpEQ(CpuValue,
11694                                   llvm::ConstantInt::get(Int32Ty, Value));
11695 }
11696
11697 Value *CodeGenFunction::EmitX86CpuSupports(const CallExpr *E) {
11698   const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts();
11699   StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
11700   return EmitX86CpuSupports(FeatureStr);
11701 }
11702
11703 uint64_t
11704 CodeGenFunction::GetX86CpuSupportsMask(ArrayRef<StringRef> FeatureStrs) {
11705   // Processor features and mapping to processor feature value.
11706   uint64_t FeaturesMask = 0;
11707   for (const StringRef &FeatureStr : FeatureStrs) {
11708     unsigned Feature =
11709         StringSwitch<unsigned>(FeatureStr)
11710 #define X86_FEATURE_COMPAT(ENUM, STR) .Case(STR, llvm::X86::FEATURE_##ENUM)
11711 #include "llvm/Support/X86TargetParser.def"
11712         ;
11713     FeaturesMask |= (1ULL << Feature);
11714   }
11715   return FeaturesMask;
11716 }
11717
11718 Value *CodeGenFunction::EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs) {
11719   return EmitX86CpuSupports(GetX86CpuSupportsMask(FeatureStrs));
11720 }
11721
11722 llvm::Value *CodeGenFunction::EmitX86CpuSupports(uint64_t FeaturesMask) {
11723   uint32_t Features1 = Lo_32(FeaturesMask);
11724   uint32_t Features2 = Hi_32(FeaturesMask);
11725
11726   Value *Result = Builder.getTrue();
11727
11728   if (Features1 != 0) {
11729     // Matching the struct layout from the compiler-rt/libgcc structure that is
11730     // filled in:
11731     // unsigned int __cpu_vendor;
11732     // unsigned int __cpu_type;
11733     // unsigned int __cpu_subtype;
11734     // unsigned int __cpu_features[1];
11735     llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
11736                                             llvm::ArrayType::get(Int32Ty, 1));
11737
11738     // Grab the global __cpu_model.
11739     llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model");
11740     cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(true);
11741
11742     // Grab the first (0th) element from the field __cpu_features off of the
11743     // global in the struct STy.
11744     Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(3),
11745                      Builder.getInt32(0)};
11746     Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
11747     Value *Features =
11748         Builder.CreateAlignedLoad(CpuFeatures, CharUnits::fromQuantity(4));
11749
11750     // Check the value of the bit corresponding to the feature requested.
11751     Value *Mask = Builder.getInt32(Features1);
11752     Value *Bitset = Builder.CreateAnd(Features, Mask);
11753     Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
11754     Result = Builder.CreateAnd(Result, Cmp);
11755   }
11756
11757   if (Features2 != 0) {
11758     llvm::Constant *CpuFeatures2 = CGM.CreateRuntimeVariable(Int32Ty,
11759                                                              "__cpu_features2");
11760     cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(true);
11761
11762     Value *Features =
11763         Builder.CreateAlignedLoad(CpuFeatures2, CharUnits::fromQuantity(4));
11764
11765     // Check the value of the bit corresponding to the feature requested.
11766     Value *Mask = Builder.getInt32(Features2);
11767     Value *Bitset = Builder.CreateAnd(Features, Mask);
11768     Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
11769     Result = Builder.CreateAnd(Result, Cmp);
11770   }
11771
11772   return Result;
11773 }
11774
11775 Value *CodeGenFunction::EmitX86CpuInit() {
11776   llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
11777                                                     /*Variadic*/ false);
11778   llvm::FunctionCallee Func =
11779       CGM.CreateRuntimeFunction(FTy, "__cpu_indicator_init");
11780   cast<llvm::GlobalValue>(Func.getCallee())->setDSOLocal(true);
11781   cast<llvm::GlobalValue>(Func.getCallee())
11782       ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
11783   return Builder.CreateCall(Func);
11784 }
11785
11786 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
11787                                            const CallExpr *E) {
11788   if (BuiltinID == X86::BI__builtin_cpu_is)
11789     return EmitX86CpuIs(E);
11790   if (BuiltinID == X86::BI__builtin_cpu_supports)
11791     return EmitX86CpuSupports(E);
11792   if (BuiltinID == X86::BI__builtin_cpu_init)
11793     return EmitX86CpuInit();
11794
11795   SmallVector<Value*, 4> Ops;
11796
11797   // Find out if any arguments are required to be integer constant expressions.
11798   unsigned ICEArguments = 0;
11799   ASTContext::GetBuiltinTypeError Error;
11800   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
11801   assert(Error == ASTContext::GE_None && "Should not codegen an error");
11802
11803   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
11804     // If this is a normal argument, just emit it as a scalar.
11805     if ((ICEArguments & (1 << i)) == 0) {
11806       Ops.push_back(EmitScalarExpr(E->getArg(i)));
11807       continue;
11808     }
11809
11810     // If this is required to be a constant, constant fold it so that we know
11811     // that the generated intrinsic gets a ConstantInt.
11812     llvm::APSInt Result;
11813     bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
11814     assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
11815     Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
11816   }
11817
11818   // These exist so that the builtin that takes an immediate can be bounds
11819   // checked by clang to avoid passing bad immediates to the backend. Since
11820   // AVX has a larger immediate than SSE we would need separate builtins to
11821   // do the different bounds checking. Rather than create a clang specific
11822   // SSE only builtin, this implements eight separate builtins to match gcc
11823   // implementation.
11824   auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) {
11825     Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm));
11826     llvm::Function *F = CGM.getIntrinsic(ID);
11827     return Builder.CreateCall(F, Ops);
11828   };
11829
11830   // For the vector forms of FP comparisons, translate the builtins directly to
11831   // IR.
11832   // TODO: The builtins could be removed if the SSE header files used vector
11833   // extension comparisons directly (vector ordered/unordered may need
11834   // additional support via __builtin_isnan()).
11835   auto getVectorFCmpIR = [this, &Ops](CmpInst::Predicate Pred,
11836                                       bool IsSignaling) {
11837     Value *Cmp;
11838     if (IsSignaling)
11839       Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
11840     else
11841       Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
11842     llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
11843     llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
11844     Value *Sext = Builder.CreateSExt(Cmp, IntVecTy);
11845     return Builder.CreateBitCast(Sext, FPVecTy);
11846   };
11847
11848   switch (BuiltinID) {
11849   default: return nullptr;
11850   case X86::BI_mm_prefetch: {
11851     Value *Address = Ops[0];
11852     ConstantInt *C = cast<ConstantInt>(Ops[1]);
11853     Value *RW = ConstantInt::get(Int32Ty, (C->getZExtValue() >> 2) & 0x1);
11854     Value *Locality = ConstantInt::get(Int32Ty, C->getZExtValue() & 0x3);
11855     Value *Data = ConstantInt::get(Int32Ty, 1);
11856     Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
11857     return Builder.CreateCall(F, {Address, RW, Locality, Data});
11858   }
11859   case X86::BI_mm_clflush: {
11860     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush),
11861                               Ops[0]);
11862   }
11863   case X86::BI_mm_lfence: {
11864     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_lfence));
11865   }
11866   case X86::BI_mm_mfence: {
11867     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_mfence));
11868   }
11869   case X86::BI_mm_sfence: {
11870     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_sfence));
11871   }
11872   case X86::BI_mm_pause: {
11873     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_pause));
11874   }
11875   case X86::BI__rdtsc: {
11876     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtsc));
11877   }
11878   case X86::BI__builtin_ia32_rdtscp: {
11879     Value *Call = Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtscp));
11880     Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
11881                                       Ops[0]);
11882     return Builder.CreateExtractValue(Call, 0);
11883   }
11884   case X86::BI__builtin_ia32_lzcnt_u16:
11885   case X86::BI__builtin_ia32_lzcnt_u32:
11886   case X86::BI__builtin_ia32_lzcnt_u64: {
11887     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
11888     return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)});
11889   }
11890   case X86::BI__builtin_ia32_tzcnt_u16:
11891   case X86::BI__builtin_ia32_tzcnt_u32:
11892   case X86::BI__builtin_ia32_tzcnt_u64: {
11893     Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType());
11894     return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)});
11895   }
11896   case X86::BI__builtin_ia32_undef128:
11897   case X86::BI__builtin_ia32_undef256:
11898   case X86::BI__builtin_ia32_undef512:
11899     // The x86 definition of "undef" is not the same as the LLVM definition
11900     // (PR32176). We leave optimizing away an unnecessary zero constant to the
11901     // IR optimizer and backend.
11902     // TODO: If we had a "freeze" IR instruction to generate a fixed undef
11903     // value, we should use that here instead of a zero.
11904     return llvm::Constant::getNullValue(ConvertType(E->getType()));
11905   case X86::BI__builtin_ia32_vec_init_v8qi:
11906   case X86::BI__builtin_ia32_vec_init_v4hi:
11907   case X86::BI__builtin_ia32_vec_init_v2si:
11908     return Builder.CreateBitCast(BuildVector(Ops),
11909                                  llvm::Type::getX86_MMXTy(getLLVMContext()));
11910   case X86::BI__builtin_ia32_vec_ext_v2si:
11911   case X86::BI__builtin_ia32_vec_ext_v16qi:
11912   case X86::BI__builtin_ia32_vec_ext_v8hi:
11913   case X86::BI__builtin_ia32_vec_ext_v4si:
11914   case X86::BI__builtin_ia32_vec_ext_v4sf:
11915   case X86::BI__builtin_ia32_vec_ext_v2di:
11916   case X86::BI__builtin_ia32_vec_ext_v32qi:
11917   case X86::BI__builtin_ia32_vec_ext_v16hi:
11918   case X86::BI__builtin_ia32_vec_ext_v8si:
11919   case X86::BI__builtin_ia32_vec_ext_v4di: {
11920     unsigned NumElts =
11921         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
11922     uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
11923     Index &= NumElts - 1;
11924     // These builtins exist so we can ensure the index is an ICE and in range.
11925     // Otherwise we could just do this in the header file.
11926     return Builder.CreateExtractElement(Ops[0], Index);
11927   }
11928   case X86::BI__builtin_ia32_vec_set_v16qi:
11929   case X86::BI__builtin_ia32_vec_set_v8hi:
11930   case X86::BI__builtin_ia32_vec_set_v4si:
11931   case X86::BI__builtin_ia32_vec_set_v2di:
11932   case X86::BI__builtin_ia32_vec_set_v32qi:
11933   case X86::BI__builtin_ia32_vec_set_v16hi:
11934   case X86::BI__builtin_ia32_vec_set_v8si:
11935   case X86::BI__builtin_ia32_vec_set_v4di: {
11936     unsigned NumElts =
11937         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
11938     unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
11939     Index &= NumElts - 1;
11940     // These builtins exist so we can ensure the index is an ICE and in range.
11941     // Otherwise we could just do this in the header file.
11942     return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
11943   }
11944   case X86::BI_mm_setcsr:
11945   case X86::BI__builtin_ia32_ldmxcsr: {
11946     Address Tmp = CreateMemTemp(E->getArg(0)->getType());
11947     Builder.CreateStore(Ops[0], Tmp);
11948     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
11949                           Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy));
11950   }
11951   case X86::BI_mm_getcsr:
11952   case X86::BI__builtin_ia32_stmxcsr: {
11953     Address Tmp = CreateMemTemp(E->getType());
11954     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
11955                        Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy));
11956     return Builder.CreateLoad(Tmp, "stmxcsr");
11957   }
11958   case X86::BI__builtin_ia32_xsave:
11959   case X86::BI__builtin_ia32_xsave64:
11960   case X86::BI__builtin_ia32_xrstor:
11961   case X86::BI__builtin_ia32_xrstor64:
11962   case X86::BI__builtin_ia32_xsaveopt:
11963   case X86::BI__builtin_ia32_xsaveopt64:
11964   case X86::BI__builtin_ia32_xrstors:
11965   case X86::BI__builtin_ia32_xrstors64:
11966   case X86::BI__builtin_ia32_xsavec:
11967   case X86::BI__builtin_ia32_xsavec64:
11968   case X86::BI__builtin_ia32_xsaves:
11969   case X86::BI__builtin_ia32_xsaves64:
11970   case X86::BI__builtin_ia32_xsetbv:
11971   case X86::BI_xsetbv: {
11972     Intrinsic::ID ID;
11973 #define INTRINSIC_X86_XSAVE_ID(NAME) \
11974     case X86::BI__builtin_ia32_##NAME: \
11975       ID = Intrinsic::x86_##NAME; \
11976       break
11977     switch (BuiltinID) {
11978     default: llvm_unreachable("Unsupported intrinsic!");
11979     INTRINSIC_X86_XSAVE_ID(xsave);
11980     INTRINSIC_X86_XSAVE_ID(xsave64);
11981     INTRINSIC_X86_XSAVE_ID(xrstor);
11982     INTRINSIC_X86_XSAVE_ID(xrstor64);
11983     INTRINSIC_X86_XSAVE_ID(xsaveopt);
11984     INTRINSIC_X86_XSAVE_ID(xsaveopt64);
11985     INTRINSIC_X86_XSAVE_ID(xrstors);
11986     INTRINSIC_X86_XSAVE_ID(xrstors64);
11987     INTRINSIC_X86_XSAVE_ID(xsavec);
11988     INTRINSIC_X86_XSAVE_ID(xsavec64);
11989     INTRINSIC_X86_XSAVE_ID(xsaves);
11990     INTRINSIC_X86_XSAVE_ID(xsaves64);
11991     INTRINSIC_X86_XSAVE_ID(xsetbv);
11992     case X86::BI_xsetbv:
11993       ID = Intrinsic::x86_xsetbv;
11994       break;
11995     }
11996 #undef INTRINSIC_X86_XSAVE_ID
11997     Value *Mhi = Builder.CreateTrunc(
11998       Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty);
11999     Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty);
12000     Ops[1] = Mhi;
12001     Ops.push_back(Mlo);
12002     return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
12003   }
12004   case X86::BI__builtin_ia32_xgetbv:
12005   case X86::BI_xgetbv:
12006     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_xgetbv), Ops);
12007   case X86::BI__builtin_ia32_storedqudi128_mask:
12008   case X86::BI__builtin_ia32_storedqusi128_mask:
12009   case X86::BI__builtin_ia32_storedquhi128_mask:
12010   case X86::BI__builtin_ia32_storedquqi128_mask:
12011   case X86::BI__builtin_ia32_storeupd128_mask:
12012   case X86::BI__builtin_ia32_storeups128_mask:
12013   case X86::BI__builtin_ia32_storedqudi256_mask:
12014   case X86::BI__builtin_ia32_storedqusi256_mask:
12015   case X86::BI__builtin_ia32_storedquhi256_mask:
12016   case X86::BI__builtin_ia32_storedquqi256_mask:
12017   case X86::BI__builtin_ia32_storeupd256_mask:
12018   case X86::BI__builtin_ia32_storeups256_mask:
12019   case X86::BI__builtin_ia32_storedqudi512_mask:
12020   case X86::BI__builtin_ia32_storedqusi512_mask:
12021   case X86::BI__builtin_ia32_storedquhi512_mask:
12022   case X86::BI__builtin_ia32_storedquqi512_mask:
12023   case X86::BI__builtin_ia32_storeupd512_mask:
12024   case X86::BI__builtin_ia32_storeups512_mask:
12025     return EmitX86MaskedStore(*this, Ops, Align(1));
12026
12027   case X86::BI__builtin_ia32_storess128_mask:
12028   case X86::BI__builtin_ia32_storesd128_mask:
12029     return EmitX86MaskedStore(*this, Ops, Align(1));
12030
12031   case X86::BI__builtin_ia32_vpopcntb_128:
12032   case X86::BI__builtin_ia32_vpopcntd_128:
12033   case X86::BI__builtin_ia32_vpopcntq_128:
12034   case X86::BI__builtin_ia32_vpopcntw_128:
12035   case X86::BI__builtin_ia32_vpopcntb_256:
12036   case X86::BI__builtin_ia32_vpopcntd_256:
12037   case X86::BI__builtin_ia32_vpopcntq_256:
12038   case X86::BI__builtin_ia32_vpopcntw_256:
12039   case X86::BI__builtin_ia32_vpopcntb_512:
12040   case X86::BI__builtin_ia32_vpopcntd_512:
12041   case X86::BI__builtin_ia32_vpopcntq_512:
12042   case X86::BI__builtin_ia32_vpopcntw_512: {
12043     llvm::Type *ResultType = ConvertType(E->getType());
12044     llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
12045     return Builder.CreateCall(F, Ops);
12046   }
12047   case X86::BI__builtin_ia32_cvtmask2b128:
12048   case X86::BI__builtin_ia32_cvtmask2b256:
12049   case X86::BI__builtin_ia32_cvtmask2b512:
12050   case X86::BI__builtin_ia32_cvtmask2w128:
12051   case X86::BI__builtin_ia32_cvtmask2w256:
12052   case X86::BI__builtin_ia32_cvtmask2w512:
12053   case X86::BI__builtin_ia32_cvtmask2d128:
12054   case X86::BI__builtin_ia32_cvtmask2d256:
12055   case X86::BI__builtin_ia32_cvtmask2d512:
12056   case X86::BI__builtin_ia32_cvtmask2q128:
12057   case X86::BI__builtin_ia32_cvtmask2q256:
12058   case X86::BI__builtin_ia32_cvtmask2q512:
12059     return EmitX86SExtMask(*this, Ops[0], ConvertType(E->getType()));
12060
12061   case X86::BI__builtin_ia32_cvtb2mask128:
12062   case X86::BI__builtin_ia32_cvtb2mask256:
12063   case X86::BI__builtin_ia32_cvtb2mask512:
12064   case X86::BI__builtin_ia32_cvtw2mask128:
12065   case X86::BI__builtin_ia32_cvtw2mask256:
12066   case X86::BI__builtin_ia32_cvtw2mask512:
12067   case X86::BI__builtin_ia32_cvtd2mask128:
12068   case X86::BI__builtin_ia32_cvtd2mask256:
12069   case X86::BI__builtin_ia32_cvtd2mask512:
12070   case X86::BI__builtin_ia32_cvtq2mask128:
12071   case X86::BI__builtin_ia32_cvtq2mask256:
12072   case X86::BI__builtin_ia32_cvtq2mask512:
12073     return EmitX86ConvertToMask(*this, Ops[0]);
12074
12075   case X86::BI__builtin_ia32_cvtdq2ps512_mask:
12076   case X86::BI__builtin_ia32_cvtqq2ps512_mask:
12077   case X86::BI__builtin_ia32_cvtqq2pd512_mask:
12078     return EmitX86ConvertIntToFp(*this, Ops, /*IsSigned*/true);
12079   case X86::BI__builtin_ia32_cvtudq2ps512_mask:
12080   case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
12081   case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
12082     return EmitX86ConvertIntToFp(*this, Ops, /*IsSigned*/false);
12083
12084   case X86::BI__builtin_ia32_vfmaddss3:
12085   case X86::BI__builtin_ia32_vfmaddsd3:
12086   case X86::BI__builtin_ia32_vfmaddss3_mask:
12087   case X86::BI__builtin_ia32_vfmaddsd3_mask:
12088     return EmitScalarFMAExpr(*this, Ops, Ops[0]);
12089   case X86::BI__builtin_ia32_vfmaddss:
12090   case X86::BI__builtin_ia32_vfmaddsd:
12091     return EmitScalarFMAExpr(*this, Ops,
12092                              Constant::getNullValue(Ops[0]->getType()));
12093   case X86::BI__builtin_ia32_vfmaddss3_maskz:
12094   case X86::BI__builtin_ia32_vfmaddsd3_maskz:
12095     return EmitScalarFMAExpr(*this, Ops, Ops[0], /*ZeroMask*/true);
12096   case X86::BI__builtin_ia32_vfmaddss3_mask3:
12097   case X86::BI__builtin_ia32_vfmaddsd3_mask3:
12098     return EmitScalarFMAExpr(*this, Ops, Ops[2], /*ZeroMask*/false, 2);
12099   case X86::BI__builtin_ia32_vfmsubss3_mask3:
12100   case X86::BI__builtin_ia32_vfmsubsd3_mask3:
12101     return EmitScalarFMAExpr(*this, Ops, Ops[2], /*ZeroMask*/false, 2,
12102                              /*NegAcc*/true);
12103   case X86::BI__builtin_ia32_vfmaddps:
12104   case X86::BI__builtin_ia32_vfmaddpd:
12105   case X86::BI__builtin_ia32_vfmaddps256:
12106   case X86::BI__builtin_ia32_vfmaddpd256:
12107   case X86::BI__builtin_ia32_vfmaddps512_mask:
12108   case X86::BI__builtin_ia32_vfmaddps512_maskz:
12109   case X86::BI__builtin_ia32_vfmaddps512_mask3:
12110   case X86::BI__builtin_ia32_vfmsubps512_mask3:
12111   case X86::BI__builtin_ia32_vfmaddpd512_mask:
12112   case X86::BI__builtin_ia32_vfmaddpd512_maskz:
12113   case X86::BI__builtin_ia32_vfmaddpd512_mask3:
12114   case X86::BI__builtin_ia32_vfmsubpd512_mask3:
12115     return EmitX86FMAExpr(*this, Ops, BuiltinID, /*IsAddSub*/false);
12116   case X86::BI__builtin_ia32_vfmaddsubps512_mask:
12117   case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
12118   case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
12119   case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
12120   case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
12121   case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
12122   case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
12123   case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
12124     return EmitX86FMAExpr(*this, Ops, BuiltinID, /*IsAddSub*/true);
12125
12126   case X86::BI__builtin_ia32_movdqa32store128_mask:
12127   case X86::BI__builtin_ia32_movdqa64store128_mask:
12128   case X86::BI__builtin_ia32_storeaps128_mask:
12129   case X86::BI__builtin_ia32_storeapd128_mask:
12130   case X86::BI__builtin_ia32_movdqa32store256_mask:
12131   case X86::BI__builtin_ia32_movdqa64store256_mask:
12132   case X86::BI__builtin_ia32_storeaps256_mask:
12133   case X86::BI__builtin_ia32_storeapd256_mask:
12134   case X86::BI__builtin_ia32_movdqa32store512_mask:
12135   case X86::BI__builtin_ia32_movdqa64store512_mask:
12136   case X86::BI__builtin_ia32_storeaps512_mask:
12137   case X86::BI__builtin_ia32_storeapd512_mask:
12138     return EmitX86MaskedStore(
12139         *this, Ops,
12140         getContext().getTypeAlignInChars(E->getArg(1)->getType()).getAsAlign());
12141
12142   case X86::BI__builtin_ia32_loadups128_mask:
12143   case X86::BI__builtin_ia32_loadups256_mask:
12144   case X86::BI__builtin_ia32_loadups512_mask:
12145   case X86::BI__builtin_ia32_loadupd128_mask:
12146   case X86::BI__builtin_ia32_loadupd256_mask:
12147   case X86::BI__builtin_ia32_loadupd512_mask:
12148   case X86::BI__builtin_ia32_loaddquqi128_mask:
12149   case X86::BI__builtin_ia32_loaddquqi256_mask:
12150   case X86::BI__builtin_ia32_loaddquqi512_mask:
12151   case X86::BI__builtin_ia32_loaddquhi128_mask:
12152   case X86::BI__builtin_ia32_loaddquhi256_mask:
12153   case X86::BI__builtin_ia32_loaddquhi512_mask:
12154   case X86::BI__builtin_ia32_loaddqusi128_mask:
12155   case X86::BI__builtin_ia32_loaddqusi256_mask:
12156   case X86::BI__builtin_ia32_loaddqusi512_mask:
12157   case X86::BI__builtin_ia32_loaddqudi128_mask:
12158   case X86::BI__builtin_ia32_loaddqudi256_mask:
12159   case X86::BI__builtin_ia32_loaddqudi512_mask:
12160     return EmitX86MaskedLoad(*this, Ops, Align(1));
12161
12162   case X86::BI__builtin_ia32_loadss128_mask:
12163   case X86::BI__builtin_ia32_loadsd128_mask:
12164     return EmitX86MaskedLoad(*this, Ops, Align(1));
12165
12166   case X86::BI__builtin_ia32_loadaps128_mask:
12167   case X86::BI__builtin_ia32_loadaps256_mask:
12168   case X86::BI__builtin_ia32_loadaps512_mask:
12169   case X86::BI__builtin_ia32_loadapd128_mask:
12170   case X86::BI__builtin_ia32_loadapd256_mask:
12171   case X86::BI__builtin_ia32_loadapd512_mask:
12172   case X86::BI__builtin_ia32_movdqa32load128_mask:
12173   case X86::BI__builtin_ia32_movdqa32load256_mask:
12174   case X86::BI__builtin_ia32_movdqa32load512_mask:
12175   case X86::BI__builtin_ia32_movdqa64load128_mask:
12176   case X86::BI__builtin_ia32_movdqa64load256_mask:
12177   case X86::BI__builtin_ia32_movdqa64load512_mask:
12178     return EmitX86MaskedLoad(
12179         *this, Ops,
12180         getContext().getTypeAlignInChars(E->getArg(1)->getType()).getAsAlign());
12181
12182   case X86::BI__builtin_ia32_expandloaddf128_mask:
12183   case X86::BI__builtin_ia32_expandloaddf256_mask:
12184   case X86::BI__builtin_ia32_expandloaddf512_mask:
12185   case X86::BI__builtin_ia32_expandloadsf128_mask:
12186   case X86::BI__builtin_ia32_expandloadsf256_mask:
12187   case X86::BI__builtin_ia32_expandloadsf512_mask:
12188   case X86::BI__builtin_ia32_expandloaddi128_mask:
12189   case X86::BI__builtin_ia32_expandloaddi256_mask:
12190   case X86::BI__builtin_ia32_expandloaddi512_mask:
12191   case X86::BI__builtin_ia32_expandloadsi128_mask:
12192   case X86::BI__builtin_ia32_expandloadsi256_mask:
12193   case X86::BI__builtin_ia32_expandloadsi512_mask:
12194   case X86::BI__builtin_ia32_expandloadhi128_mask:
12195   case X86::BI__builtin_ia32_expandloadhi256_mask:
12196   case X86::BI__builtin_ia32_expandloadhi512_mask:
12197   case X86::BI__builtin_ia32_expandloadqi128_mask:
12198   case X86::BI__builtin_ia32_expandloadqi256_mask:
12199   case X86::BI__builtin_ia32_expandloadqi512_mask:
12200     return EmitX86ExpandLoad(*this, Ops);
12201
12202   case X86::BI__builtin_ia32_compressstoredf128_mask:
12203   case X86::BI__builtin_ia32_compressstoredf256_mask:
12204   case X86::BI__builtin_ia32_compressstoredf512_mask:
12205   case X86::BI__builtin_ia32_compressstoresf128_mask:
12206   case X86::BI__builtin_ia32_compressstoresf256_mask:
12207   case X86::BI__builtin_ia32_compressstoresf512_mask:
12208   case X86::BI__builtin_ia32_compressstoredi128_mask:
12209   case X86::BI__builtin_ia32_compressstoredi256_mask:
12210   case X86::BI__builtin_ia32_compressstoredi512_mask:
12211   case X86::BI__builtin_ia32_compressstoresi128_mask:
12212   case X86::BI__builtin_ia32_compressstoresi256_mask:
12213   case X86::BI__builtin_ia32_compressstoresi512_mask:
12214   case X86::BI__builtin_ia32_compressstorehi128_mask:
12215   case X86::BI__builtin_ia32_compressstorehi256_mask:
12216   case X86::BI__builtin_ia32_compressstorehi512_mask:
12217   case X86::BI__builtin_ia32_compressstoreqi128_mask:
12218   case X86::BI__builtin_ia32_compressstoreqi256_mask:
12219   case X86::BI__builtin_ia32_compressstoreqi512_mask:
12220     return EmitX86CompressStore(*this, Ops);
12221
12222   case X86::BI__builtin_ia32_expanddf128_mask:
12223   case X86::BI__builtin_ia32_expanddf256_mask:
12224   case X86::BI__builtin_ia32_expanddf512_mask:
12225   case X86::BI__builtin_ia32_expandsf128_mask:
12226   case X86::BI__builtin_ia32_expandsf256_mask:
12227   case X86::BI__builtin_ia32_expandsf512_mask:
12228   case X86::BI__builtin_ia32_expanddi128_mask:
12229   case X86::BI__builtin_ia32_expanddi256_mask:
12230   case X86::BI__builtin_ia32_expanddi512_mask:
12231   case X86::BI__builtin_ia32_expandsi128_mask:
12232   case X86::BI__builtin_ia32_expandsi256_mask:
12233   case X86::BI__builtin_ia32_expandsi512_mask:
12234   case X86::BI__builtin_ia32_expandhi128_mask:
12235   case X86::BI__builtin_ia32_expandhi256_mask:
12236   case X86::BI__builtin_ia32_expandhi512_mask:
12237   case X86::BI__builtin_ia32_expandqi128_mask:
12238   case X86::BI__builtin_ia32_expandqi256_mask:
12239   case X86::BI__builtin_ia32_expandqi512_mask:
12240     return EmitX86CompressExpand(*this, Ops, /*IsCompress*/false);
12241
12242   case X86::BI__builtin_ia32_compressdf128_mask:
12243   case X86::BI__builtin_ia32_compressdf256_mask:
12244   case X86::BI__builtin_ia32_compressdf512_mask:
12245   case X86::BI__builtin_ia32_compresssf128_mask:
12246   case X86::BI__builtin_ia32_compresssf256_mask:
12247   case X86::BI__builtin_ia32_compresssf512_mask:
12248   case X86::BI__builtin_ia32_compressdi128_mask:
12249   case X86::BI__builtin_ia32_compressdi256_mask:
12250   case X86::BI__builtin_ia32_compressdi512_mask:
12251   case X86::BI__builtin_ia32_compresssi128_mask:
12252   case X86::BI__builtin_ia32_compresssi256_mask:
12253   case X86::BI__builtin_ia32_compresssi512_mask:
12254   case X86::BI__builtin_ia32_compresshi128_mask:
12255   case X86::BI__builtin_ia32_compresshi256_mask:
12256   case X86::BI__builtin_ia32_compresshi512_mask:
12257   case X86::BI__builtin_ia32_compressqi128_mask:
12258   case X86::BI__builtin_ia32_compressqi256_mask:
12259   case X86::BI__builtin_ia32_compressqi512_mask:
12260     return EmitX86CompressExpand(*this, Ops, /*IsCompress*/true);
12261
12262   case X86::BI__builtin_ia32_gather3div2df:
12263   case X86::BI__builtin_ia32_gather3div2di:
12264   case X86::BI__builtin_ia32_gather3div4df:
12265   case X86::BI__builtin_ia32_gather3div4di:
12266   case X86::BI__builtin_ia32_gather3div4sf:
12267   case X86::BI__builtin_ia32_gather3div4si:
12268   case X86::BI__builtin_ia32_gather3div8sf:
12269   case X86::BI__builtin_ia32_gather3div8si:
12270   case X86::BI__builtin_ia32_gather3siv2df:
12271   case X86::BI__builtin_ia32_gather3siv2di:
12272   case X86::BI__builtin_ia32_gather3siv4df:
12273   case X86::BI__builtin_ia32_gather3siv4di:
12274   case X86::BI__builtin_ia32_gather3siv4sf:
12275   case X86::BI__builtin_ia32_gather3siv4si:
12276   case X86::BI__builtin_ia32_gather3siv8sf:
12277   case X86::BI__builtin_ia32_gather3siv8si:
12278   case X86::BI__builtin_ia32_gathersiv8df:
12279   case X86::BI__builtin_ia32_gathersiv16sf:
12280   case X86::BI__builtin_ia32_gatherdiv8df:
12281   case X86::BI__builtin_ia32_gatherdiv16sf:
12282   case X86::BI__builtin_ia32_gathersiv8di:
12283   case X86::BI__builtin_ia32_gathersiv16si:
12284   case X86::BI__builtin_ia32_gatherdiv8di:
12285   case X86::BI__builtin_ia32_gatherdiv16si: {
12286     Intrinsic::ID IID;
12287     switch (BuiltinID) {
12288     default: llvm_unreachable("Unexpected builtin");
12289     case X86::BI__builtin_ia32_gather3div2df:
12290       IID = Intrinsic::x86_avx512_mask_gather3div2_df;
12291       break;
12292     case X86::BI__builtin_ia32_gather3div2di:
12293       IID = Intrinsic::x86_avx512_mask_gather3div2_di;
12294       break;
12295     case X86::BI__builtin_ia32_gather3div4df:
12296       IID = Intrinsic::x86_avx512_mask_gather3div4_df;
12297       break;
12298     case X86::BI__builtin_ia32_gather3div4di:
12299       IID = Intrinsic::x86_avx512_mask_gather3div4_di;
12300       break;
12301     case X86::BI__builtin_ia32_gather3div4sf:
12302       IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
12303       break;
12304     case X86::BI__builtin_ia32_gather3div4si:
12305       IID = Intrinsic::x86_avx512_mask_gather3div4_si;
12306       break;
12307     case X86::BI__builtin_ia32_gather3div8sf:
12308       IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
12309       break;
12310     case X86::BI__builtin_ia32_gather3div8si:
12311       IID = Intrinsic::x86_avx512_mask_gather3div8_si;
12312       break;
12313     case X86::BI__builtin_ia32_gather3siv2df:
12314       IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
12315       break;
12316     case X86::BI__builtin_ia32_gather3siv2di:
12317       IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
12318       break;
12319     case X86::BI__builtin_ia32_gather3siv4df:
12320       IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
12321       break;
12322     case X86::BI__builtin_ia32_gather3siv4di:
12323       IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
12324       break;
12325     case X86::BI__builtin_ia32_gather3siv4sf:
12326       IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
12327       break;
12328     case X86::BI__builtin_ia32_gather3siv4si:
12329       IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
12330       break;
12331     case X86::BI__builtin_ia32_gather3siv8sf:
12332       IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
12333       break;
12334     case X86::BI__builtin_ia32_gather3siv8si:
12335       IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
12336       break;
12337     case X86::BI__builtin_ia32_gathersiv8df:
12338       IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
12339       break;
12340     case X86::BI__builtin_ia32_gathersiv16sf:
12341       IID = Intrinsic::x86_avx512_mask_gather_dps_512;
12342       break;
12343     case X86::BI__builtin_ia32_gatherdiv8df:
12344       IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
12345       break;
12346     case X86::BI__builtin_ia32_gatherdiv16sf:
12347       IID = Intrinsic::x86_avx512_mask_gather_qps_512;
12348       break;
12349     case X86::BI__builtin_ia32_gathersiv8di:
12350       IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
12351       break;
12352     case X86::BI__builtin_ia32_gathersiv16si:
12353       IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
12354       break;
12355     case X86::BI__builtin_ia32_gatherdiv8di:
12356       IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
12357       break;
12358     case X86::BI__builtin_ia32_gatherdiv16si:
12359       IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
12360       break;
12361     }
12362
12363     unsigned MinElts =
12364         std::min(cast<llvm::VectorType>(Ops[0]->getType())->getNumElements(),
12365                  cast<llvm::VectorType>(Ops[2]->getType())->getNumElements());
12366     Ops[3] = getMaskVecValue(*this, Ops[3], MinElts);
12367     Function *Intr = CGM.getIntrinsic(IID);
12368     return Builder.CreateCall(Intr, Ops);
12369   }
12370
12371   case X86::BI__builtin_ia32_scattersiv8df:
12372   case X86::BI__builtin_ia32_scattersiv16sf:
12373   case X86::BI__builtin_ia32_scatterdiv8df:
12374   case X86::BI__builtin_ia32_scatterdiv16sf:
12375   case X86::BI__builtin_ia32_scattersiv8di:
12376   case X86::BI__builtin_ia32_scattersiv16si:
12377   case X86::BI__builtin_ia32_scatterdiv8di:
12378   case X86::BI__builtin_ia32_scatterdiv16si:
12379   case X86::BI__builtin_ia32_scatterdiv2df:
12380   case X86::BI__builtin_ia32_scatterdiv2di:
12381   case X86::BI__builtin_ia32_scatterdiv4df:
12382   case X86::BI__builtin_ia32_scatterdiv4di:
12383   case X86::BI__builtin_ia32_scatterdiv4sf:
12384   case X86::BI__builtin_ia32_scatterdiv4si:
12385   case X86::BI__builtin_ia32_scatterdiv8sf:
12386   case X86::BI__builtin_ia32_scatterdiv8si:
12387   case X86::BI__builtin_ia32_scattersiv2df:
12388   case X86::BI__builtin_ia32_scattersiv2di:
12389   case X86::BI__builtin_ia32_scattersiv4df:
12390   case X86::BI__builtin_ia32_scattersiv4di:
12391   case X86::BI__builtin_ia32_scattersiv4sf:
12392   case X86::BI__builtin_ia32_scattersiv4si:
12393   case X86::BI__builtin_ia32_scattersiv8sf:
12394   case X86::BI__builtin_ia32_scattersiv8si: {
12395     Intrinsic::ID IID;
12396     switch (BuiltinID) {
12397     default: llvm_unreachable("Unexpected builtin");
12398     case X86::BI__builtin_ia32_scattersiv8df:
12399       IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
12400       break;
12401     case X86::BI__builtin_ia32_scattersiv16sf:
12402       IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
12403       break;
12404     case X86::BI__builtin_ia32_scatterdiv8df:
12405       IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
12406       break;
12407     case X86::BI__builtin_ia32_scatterdiv16sf:
12408       IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
12409       break;
12410     case X86::BI__builtin_ia32_scattersiv8di:
12411       IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
12412       break;
12413     case X86::BI__builtin_ia32_scattersiv16si:
12414       IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
12415       break;
12416     case X86::BI__builtin_ia32_scatterdiv8di:
12417       IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
12418       break;
12419     case X86::BI__builtin_ia32_scatterdiv16si:
12420       IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
12421       break;
12422     case X86::BI__builtin_ia32_scatterdiv2df:
12423       IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
12424       break;
12425     case X86::BI__builtin_ia32_scatterdiv2di:
12426       IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
12427       break;
12428     case X86::BI__builtin_ia32_scatterdiv4df:
12429       IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
12430       break;
12431     case X86::BI__builtin_ia32_scatterdiv4di:
12432       IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
12433       break;
12434     case X86::BI__builtin_ia32_scatterdiv4sf:
12435       IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
12436       break;
12437     case X86::BI__builtin_ia32_scatterdiv4si:
12438       IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
12439       break;
12440     case X86::BI__builtin_ia32_scatterdiv8sf:
12441       IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
12442       break;
12443     case X86::BI__builtin_ia32_scatterdiv8si:
12444       IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
12445       break;
12446     case X86::BI__builtin_ia32_scattersiv2df:
12447       IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
12448       break;
12449     case X86::BI__builtin_ia32_scattersiv2di:
12450       IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
12451       break;
12452     case X86::BI__builtin_ia32_scattersiv4df:
12453       IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
12454       break;
12455     case X86::BI__builtin_ia32_scattersiv4di:
12456       IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
12457       break;
12458     case X86::BI__builtin_ia32_scattersiv4sf:
12459       IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
12460       break;
12461     case X86::BI__builtin_ia32_scattersiv4si:
12462       IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
12463       break;
12464     case X86::BI__builtin_ia32_scattersiv8sf:
12465       IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
12466       break;
12467     case X86::BI__builtin_ia32_scattersiv8si:
12468       IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
12469       break;
12470     }
12471
12472     unsigned MinElts =
12473         std::min(cast<llvm::VectorType>(Ops[2]->getType())->getNumElements(),
12474                  cast<llvm::VectorType>(Ops[3]->getType())->getNumElements());
12475     Ops[1] = getMaskVecValue(*this, Ops[1], MinElts);
12476     Function *Intr = CGM.getIntrinsic(IID);
12477     return Builder.CreateCall(Intr, Ops);
12478   }
12479
12480   case X86::BI__builtin_ia32_vextractf128_pd256:
12481   case X86::BI__builtin_ia32_vextractf128_ps256:
12482   case X86::BI__builtin_ia32_vextractf128_si256:
12483   case X86::BI__builtin_ia32_extract128i256:
12484   case X86::BI__builtin_ia32_extractf64x4_mask:
12485   case X86::BI__builtin_ia32_extractf32x4_mask:
12486   case X86::BI__builtin_ia32_extracti64x4_mask:
12487   case X86::BI__builtin_ia32_extracti32x4_mask:
12488   case X86::BI__builtin_ia32_extractf32x8_mask:
12489   case X86::BI__builtin_ia32_extracti32x8_mask:
12490   case X86::BI__builtin_ia32_extractf32x4_256_mask:
12491   case X86::BI__builtin_ia32_extracti32x4_256_mask:
12492   case X86::BI__builtin_ia32_extractf64x2_256_mask:
12493   case X86::BI__builtin_ia32_extracti64x2_256_mask:
12494   case X86::BI__builtin_ia32_extractf64x2_512_mask:
12495   case X86::BI__builtin_ia32_extracti64x2_512_mask: {
12496     auto *DstTy = cast<llvm::VectorType>(ConvertType(E->getType()));
12497     unsigned NumElts = DstTy->getNumElements();
12498     unsigned SrcNumElts =
12499         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
12500     unsigned SubVectors = SrcNumElts / NumElts;
12501     unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
12502     assert(llvm::isPowerOf2_32(SubVectors) && "Expected power of 2 subvectors");
12503     Index &= SubVectors - 1; // Remove any extra bits.
12504     Index *= NumElts;
12505
12506     int Indices[16];
12507     for (unsigned i = 0; i != NumElts; ++i)
12508       Indices[i] = i + Index;
12509
12510     Value *Res = Builder.CreateShuffleVector(Ops[0],
12511                                              UndefValue::get(Ops[0]->getType()),
12512                                              makeArrayRef(Indices, NumElts),
12513                                              "extract");
12514
12515     if (Ops.size() == 4)
12516       Res = EmitX86Select(*this, Ops[3], Res, Ops[2]);
12517
12518     return Res;
12519   }
12520   case X86::BI__builtin_ia32_vinsertf128_pd256:
12521   case X86::BI__builtin_ia32_vinsertf128_ps256:
12522   case X86::BI__builtin_ia32_vinsertf128_si256:
12523   case X86::BI__builtin_ia32_insert128i256:
12524   case X86::BI__builtin_ia32_insertf64x4:
12525   case X86::BI__builtin_ia32_insertf32x4:
12526   case X86::BI__builtin_ia32_inserti64x4:
12527   case X86::BI__builtin_ia32_inserti32x4:
12528   case X86::BI__builtin_ia32_insertf32x8:
12529   case X86::BI__builtin_ia32_inserti32x8:
12530   case X86::BI__builtin_ia32_insertf32x4_256:
12531   case X86::BI__builtin_ia32_inserti32x4_256:
12532   case X86::BI__builtin_ia32_insertf64x2_256:
12533   case X86::BI__builtin_ia32_inserti64x2_256:
12534   case X86::BI__builtin_ia32_insertf64x2_512:
12535   case X86::BI__builtin_ia32_inserti64x2_512: {
12536     unsigned DstNumElts =
12537         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
12538     unsigned SrcNumElts =
12539         cast<llvm::VectorType>(Ops[1]->getType())->getNumElements();
12540     unsigned SubVectors = DstNumElts / SrcNumElts;
12541     unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
12542     assert(llvm::isPowerOf2_32(SubVectors) && "Expected power of 2 subvectors");
12543     Index &= SubVectors - 1; // Remove any extra bits.
12544     Index *= SrcNumElts;
12545
12546     int Indices[16];
12547     for (unsigned i = 0; i != DstNumElts; ++i)
12548       Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
12549
12550     Value *Op1 = Builder.CreateShuffleVector(Ops[1],
12551                                              UndefValue::get(Ops[1]->getType()),
12552                                              makeArrayRef(Indices, DstNumElts),
12553                                              "widen");
12554
12555     for (unsigned i = 0; i != DstNumElts; ++i) {
12556       if (i >= Index && i < (Index + SrcNumElts))
12557         Indices[i] = (i - Index) + DstNumElts;
12558       else
12559         Indices[i] = i;
12560     }
12561
12562     return Builder.CreateShuffleVector(Ops[0], Op1,
12563                                        makeArrayRef(Indices, DstNumElts),
12564                                        "insert");
12565   }
12566   case X86::BI__builtin_ia32_pmovqd512_mask:
12567   case X86::BI__builtin_ia32_pmovwb512_mask: {
12568     Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType());
12569     return EmitX86Select(*this, Ops[2], Res, Ops[1]);
12570   }
12571   case X86::BI__builtin_ia32_pmovdb512_mask:
12572   case X86::BI__builtin_ia32_pmovdw512_mask:
12573   case X86::BI__builtin_ia32_pmovqw512_mask: {
12574     if (const auto *C = dyn_cast<Constant>(Ops[2]))
12575       if (C->isAllOnesValue())
12576         return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
12577
12578     Intrinsic::ID IID;
12579     switch (BuiltinID) {
12580     default: llvm_unreachable("Unsupported intrinsic!");
12581     case X86::BI__builtin_ia32_pmovdb512_mask:
12582       IID = Intrinsic::x86_avx512_mask_pmov_db_512;
12583       break;
12584     case X86::BI__builtin_ia32_pmovdw512_mask:
12585       IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
12586       break;
12587     case X86::BI__builtin_ia32_pmovqw512_mask:
12588       IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
12589       break;
12590     }
12591
12592     Function *Intr = CGM.getIntrinsic(IID);
12593     return Builder.CreateCall(Intr, Ops);
12594   }
12595   case X86::BI__builtin_ia32_pblendw128:
12596   case X86::BI__builtin_ia32_blendpd:
12597   case X86::BI__builtin_ia32_blendps:
12598   case X86::BI__builtin_ia32_blendpd256:
12599   case X86::BI__builtin_ia32_blendps256:
12600   case X86::BI__builtin_ia32_pblendw256:
12601   case X86::BI__builtin_ia32_pblendd128:
12602   case X86::BI__builtin_ia32_pblendd256: {
12603     unsigned NumElts =
12604         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
12605     unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
12606
12607     int Indices[16];
12608     // If there are more than 8 elements, the immediate is used twice so make
12609     // sure we handle that.
12610     for (unsigned i = 0; i != NumElts; ++i)
12611       Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
12612
12613     return Builder.CreateShuffleVector(Ops[0], Ops[1],
12614                                        makeArrayRef(Indices, NumElts),
12615                                        "blend");
12616   }
12617   case X86::BI__builtin_ia32_pshuflw:
12618   case X86::BI__builtin_ia32_pshuflw256:
12619   case X86::BI__builtin_ia32_pshuflw512: {
12620     uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
12621     auto *Ty = cast<llvm::VectorType>(Ops[0]->getType());
12622     unsigned NumElts = Ty->getNumElements();
12623
12624     // Splat the 8-bits of immediate 4 times to help the loop wrap around.
12625     Imm = (Imm & 0xff) * 0x01010101;
12626
12627     int Indices[32];
12628     for (unsigned l = 0; l != NumElts; l += 8) {
12629       for (unsigned i = 0; i != 4; ++i) {
12630         Indices[l + i] = l + (Imm & 3);
12631         Imm >>= 2;
12632       }
12633       for (unsigned i = 4; i != 8; ++i)
12634         Indices[l + i] = l + i;
12635     }
12636
12637     return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
12638                                        makeArrayRef(Indices, NumElts),
12639                                        "pshuflw");
12640   }
12641   case X86::BI__builtin_ia32_pshufhw:
12642   case X86::BI__builtin_ia32_pshufhw256:
12643   case X86::BI__builtin_ia32_pshufhw512: {
12644     uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
12645     auto *Ty = cast<llvm::VectorType>(Ops[0]->getType());
12646     unsigned NumElts = Ty->getNumElements();
12647
12648     // Splat the 8-bits of immediate 4 times to help the loop wrap around.
12649     Imm = (Imm & 0xff) * 0x01010101;
12650
12651     int Indices[32];
12652     for (unsigned l = 0; l != NumElts; l += 8) {
12653       for (unsigned i = 0; i != 4; ++i)
12654         Indices[l + i] = l + i;
12655       for (unsigned i = 4; i != 8; ++i) {
12656         Indices[l + i] = l + 4 + (Imm & 3);
12657         Imm >>= 2;
12658       }
12659     }
12660
12661     return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
12662                                        makeArrayRef(Indices, NumElts),
12663                                        "pshufhw");
12664   }
12665   case X86::BI__builtin_ia32_pshufd:
12666   case X86::BI__builtin_ia32_pshufd256:
12667   case X86::BI__builtin_ia32_pshufd512:
12668   case X86::BI__builtin_ia32_vpermilpd:
12669   case X86::BI__builtin_ia32_vpermilps:
12670   case X86::BI__builtin_ia32_vpermilpd256:
12671   case X86::BI__builtin_ia32_vpermilps256:
12672   case X86::BI__builtin_ia32_vpermilpd512:
12673   case X86::BI__builtin_ia32_vpermilps512: {
12674     uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
12675     auto *Ty = cast<llvm::VectorType>(Ops[0]->getType());
12676     unsigned NumElts = Ty->getNumElements();
12677     unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
12678     unsigned NumLaneElts = NumElts / NumLanes;
12679
12680     // Splat the 8-bits of immediate 4 times to help the loop wrap around.
12681     Imm = (Imm & 0xff) * 0x01010101;
12682
12683     int Indices[16];
12684     for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
12685       for (unsigned i = 0; i != NumLaneElts; ++i) {
12686         Indices[i + l] = (Imm % NumLaneElts) + l;
12687         Imm /= NumLaneElts;
12688       }
12689     }
12690
12691     return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
12692                                        makeArrayRef(Indices, NumElts),
12693                                        "permil");
12694   }
12695   case X86::BI__builtin_ia32_shufpd:
12696   case X86::BI__builtin_ia32_shufpd256:
12697   case X86::BI__builtin_ia32_shufpd512:
12698   case X86::BI__builtin_ia32_shufps:
12699   case X86::BI__builtin_ia32_shufps256:
12700   case X86::BI__builtin_ia32_shufps512: {
12701     uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
12702     auto *Ty = cast<llvm::VectorType>(Ops[0]->getType());
12703     unsigned NumElts = Ty->getNumElements();
12704     unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
12705     unsigned NumLaneElts = NumElts / NumLanes;
12706
12707     // Splat the 8-bits of immediate 4 times to help the loop wrap around.
12708     Imm = (Imm & 0xff) * 0x01010101;
12709
12710     int Indices[16];
12711     for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
12712       for (unsigned i = 0; i != NumLaneElts; ++i) {
12713         unsigned Index = Imm % NumLaneElts;
12714         Imm /= NumLaneElts;
12715         if (i >= (NumLaneElts / 2))
12716           Index += NumElts;
12717         Indices[l + i] = l + Index;
12718       }
12719     }
12720
12721     return Builder.CreateShuffleVector(Ops[0], Ops[1],
12722                                        makeArrayRef(Indices, NumElts),
12723                                        "shufp");
12724   }
12725   case X86::BI__builtin_ia32_permdi256:
12726   case X86::BI__builtin_ia32_permdf256:
12727   case X86::BI__builtin_ia32_permdi512:
12728   case X86::BI__builtin_ia32_permdf512: {
12729     unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
12730     auto *Ty = cast<llvm::VectorType>(Ops[0]->getType());
12731     unsigned NumElts = Ty->getNumElements();
12732
12733     // These intrinsics operate on 256-bit lanes of four 64-bit elements.
12734     int Indices[8];
12735     for (unsigned l = 0; l != NumElts; l += 4)
12736       for (unsigned i = 0; i != 4; ++i)
12737         Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
12738
12739     return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
12740                                        makeArrayRef(Indices, NumElts),
12741                                        "perm");
12742   }
12743   case X86::BI__builtin_ia32_palignr128:
12744   case X86::BI__builtin_ia32_palignr256:
12745   case X86::BI__builtin_ia32_palignr512: {
12746     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
12747
12748     unsigned NumElts =
12749         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
12750     assert(NumElts % 16 == 0);
12751
12752     // If palignr is shifting the pair of vectors more than the size of two
12753     // lanes, emit zero.
12754     if (ShiftVal >= 32)
12755       return llvm::Constant::getNullValue(ConvertType(E->getType()));
12756
12757     // If palignr is shifting the pair of input vectors more than one lane,
12758     // but less than two lanes, convert to shifting in zeroes.
12759     if (ShiftVal > 16) {
12760       ShiftVal -= 16;
12761       Ops[1] = Ops[0];
12762       Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
12763     }
12764
12765     int Indices[64];
12766     // 256-bit palignr operates on 128-bit lanes so we need to handle that
12767     for (unsigned l = 0; l != NumElts; l += 16) {
12768       for (unsigned i = 0; i != 16; ++i) {
12769         unsigned Idx = ShiftVal + i;
12770         if (Idx >= 16)
12771           Idx += NumElts - 16; // End of lane, switch operand.
12772         Indices[l + i] = Idx + l;
12773       }
12774     }
12775
12776     return Builder.CreateShuffleVector(Ops[1], Ops[0],
12777                                        makeArrayRef(Indices, NumElts),
12778                                        "palignr");
12779   }
12780   case X86::BI__builtin_ia32_alignd128:
12781   case X86::BI__builtin_ia32_alignd256:
12782   case X86::BI__builtin_ia32_alignd512:
12783   case X86::BI__builtin_ia32_alignq128:
12784   case X86::BI__builtin_ia32_alignq256:
12785   case X86::BI__builtin_ia32_alignq512: {
12786     unsigned NumElts =
12787         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
12788     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
12789
12790     // Mask the shift amount to width of two vectors.
12791     ShiftVal &= (2 * NumElts) - 1;
12792
12793     int Indices[16];
12794     for (unsigned i = 0; i != NumElts; ++i)
12795       Indices[i] = i + ShiftVal;
12796
12797     return Builder.CreateShuffleVector(Ops[1], Ops[0],
12798                                        makeArrayRef(Indices, NumElts),
12799                                        "valign");
12800   }
12801   case X86::BI__builtin_ia32_shuf_f32x4_256:
12802   case X86::BI__builtin_ia32_shuf_f64x2_256:
12803   case X86::BI__builtin_ia32_shuf_i32x4_256:
12804   case X86::BI__builtin_ia32_shuf_i64x2_256:
12805   case X86::BI__builtin_ia32_shuf_f32x4:
12806   case X86::BI__builtin_ia32_shuf_f64x2:
12807   case X86::BI__builtin_ia32_shuf_i32x4:
12808   case X86::BI__builtin_ia32_shuf_i64x2: {
12809     unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
12810     auto *Ty = cast<llvm::VectorType>(Ops[0]->getType());
12811     unsigned NumElts = Ty->getNumElements();
12812     unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
12813     unsigned NumLaneElts = NumElts / NumLanes;
12814
12815     int Indices[16];
12816     for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
12817       unsigned Index = (Imm % NumLanes) * NumLaneElts;
12818       Imm /= NumLanes; // Discard the bits we just used.
12819       if (l >= (NumElts / 2))
12820         Index += NumElts; // Switch to other source.
12821       for (unsigned i = 0; i != NumLaneElts; ++i) {
12822         Indices[l + i] = Index + i;
12823       }
12824     }
12825
12826     return Builder.CreateShuffleVector(Ops[0], Ops[1],
12827                                        makeArrayRef(Indices, NumElts),
12828                                        "shuf");
12829   }
12830
12831   case X86::BI__builtin_ia32_vperm2f128_pd256:
12832   case X86::BI__builtin_ia32_vperm2f128_ps256:
12833   case X86::BI__builtin_ia32_vperm2f128_si256:
12834   case X86::BI__builtin_ia32_permti256: {
12835     unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
12836     unsigned NumElts =
12837         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
12838
12839     // This takes a very simple approach since there are two lanes and a
12840     // shuffle can have 2 inputs. So we reserve the first input for the first
12841     // lane and the second input for the second lane. This may result in
12842     // duplicate sources, but this can be dealt with in the backend.
12843
12844     Value *OutOps[2];
12845     int Indices[8];
12846     for (unsigned l = 0; l != 2; ++l) {
12847       // Determine the source for this lane.
12848       if (Imm & (1 << ((l * 4) + 3)))
12849         OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
12850       else if (Imm & (1 << ((l * 4) + 1)))
12851         OutOps[l] = Ops[1];
12852       else
12853         OutOps[l] = Ops[0];
12854
12855       for (unsigned i = 0; i != NumElts/2; ++i) {
12856         // Start with ith element of the source for this lane.
12857         unsigned Idx = (l * NumElts) + i;
12858         // If bit 0 of the immediate half is set, switch to the high half of
12859         // the source.
12860         if (Imm & (1 << (l * 4)))
12861           Idx += NumElts/2;
12862         Indices[(l * (NumElts/2)) + i] = Idx;
12863       }
12864     }
12865
12866     return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
12867                                        makeArrayRef(Indices, NumElts),
12868                                        "vperm");
12869   }
12870
12871   case X86::BI__builtin_ia32_pslldqi128_byteshift:
12872   case X86::BI__builtin_ia32_pslldqi256_byteshift:
12873   case X86::BI__builtin_ia32_pslldqi512_byteshift: {
12874     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
12875     auto *ResultType = cast<llvm::VectorType>(Ops[0]->getType());
12876     // Builtin type is vXi64 so multiply by 8 to get bytes.
12877     unsigned NumElts = ResultType->getNumElements() * 8;
12878
12879     // If pslldq is shifting the vector more than 15 bytes, emit zero.
12880     if (ShiftVal >= 16)
12881       return llvm::Constant::getNullValue(ResultType);
12882
12883     int Indices[64];
12884     // 256/512-bit pslldq operates on 128-bit lanes so we need to handle that
12885     for (unsigned l = 0; l != NumElts; l += 16) {
12886       for (unsigned i = 0; i != 16; ++i) {
12887         unsigned Idx = NumElts + i - ShiftVal;
12888         if (Idx < NumElts) Idx -= NumElts - 16; // end of lane, switch operand.
12889         Indices[l + i] = Idx + l;
12890       }
12891     }
12892
12893     auto *VecTy = llvm::FixedVectorType::get(Int8Ty, NumElts);
12894     Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast");
12895     Value *Zero = llvm::Constant::getNullValue(VecTy);
12896     Value *SV = Builder.CreateShuffleVector(Zero, Cast,
12897                                             makeArrayRef(Indices, NumElts),
12898                                             "pslldq");
12899     return Builder.CreateBitCast(SV, Ops[0]->getType(), "cast");
12900   }
12901   case X86::BI__builtin_ia32_psrldqi128_byteshift:
12902   case X86::BI__builtin_ia32_psrldqi256_byteshift:
12903   case X86::BI__builtin_ia32_psrldqi512_byteshift: {
12904     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
12905     auto *ResultType = cast<llvm::VectorType>(Ops[0]->getType());
12906     // Builtin type is vXi64 so multiply by 8 to get bytes.
12907     unsigned NumElts = ResultType->getNumElements() * 8;
12908
12909     // If psrldq is shifting the vector more than 15 bytes, emit zero.
12910     if (ShiftVal >= 16)
12911       return llvm::Constant::getNullValue(ResultType);
12912
12913     int Indices[64];
12914     // 256/512-bit psrldq operates on 128-bit lanes so we need to handle that
12915     for (unsigned l = 0; l != NumElts; l += 16) {
12916       for (unsigned i = 0; i != 16; ++i) {
12917         unsigned Idx = i + ShiftVal;
12918         if (Idx >= 16) Idx += NumElts - 16; // end of lane, switch operand.
12919         Indices[l + i] = Idx + l;
12920       }
12921     }
12922
12923     auto *VecTy = llvm::FixedVectorType::get(Int8Ty, NumElts);
12924     Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast");
12925     Value *Zero = llvm::Constant::getNullValue(VecTy);
12926     Value *SV = Builder.CreateShuffleVector(Cast, Zero,
12927                                             makeArrayRef(Indices, NumElts),
12928                                             "psrldq");
12929     return Builder.CreateBitCast(SV, ResultType, "cast");
12930   }
12931   case X86::BI__builtin_ia32_kshiftliqi:
12932   case X86::BI__builtin_ia32_kshiftlihi:
12933   case X86::BI__builtin_ia32_kshiftlisi:
12934   case X86::BI__builtin_ia32_kshiftlidi: {
12935     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
12936     unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
12937
12938     if (ShiftVal >= NumElts)
12939       return llvm::Constant::getNullValue(Ops[0]->getType());
12940
12941     Value *In = getMaskVecValue(*this, Ops[0], NumElts);
12942
12943     int Indices[64];
12944     for (unsigned i = 0; i != NumElts; ++i)
12945       Indices[i] = NumElts + i - ShiftVal;
12946
12947     Value *Zero = llvm::Constant::getNullValue(In->getType());
12948     Value *SV = Builder.CreateShuffleVector(Zero, In,
12949                                             makeArrayRef(Indices, NumElts),
12950                                             "kshiftl");
12951     return Builder.CreateBitCast(SV, Ops[0]->getType());
12952   }
12953   case X86::BI__builtin_ia32_kshiftriqi:
12954   case X86::BI__builtin_ia32_kshiftrihi:
12955   case X86::BI__builtin_ia32_kshiftrisi:
12956   case X86::BI__builtin_ia32_kshiftridi: {
12957     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
12958     unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
12959
12960     if (ShiftVal >= NumElts)
12961       return llvm::Constant::getNullValue(Ops[0]->getType());
12962
12963     Value *In = getMaskVecValue(*this, Ops[0], NumElts);
12964
12965     int Indices[64];
12966     for (unsigned i = 0; i != NumElts; ++i)
12967       Indices[i] = i + ShiftVal;
12968
12969     Value *Zero = llvm::Constant::getNullValue(In->getType());
12970     Value *SV = Builder.CreateShuffleVector(In, Zero,
12971                                             makeArrayRef(Indices, NumElts),
12972                                             "kshiftr");
12973     return Builder.CreateBitCast(SV, Ops[0]->getType());
12974   }
12975   case X86::BI__builtin_ia32_movnti:
12976   case X86::BI__builtin_ia32_movnti64:
12977   case X86::BI__builtin_ia32_movntsd:
12978   case X86::BI__builtin_ia32_movntss: {
12979     llvm::MDNode *Node = llvm::MDNode::get(
12980         getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
12981
12982     Value *Ptr = Ops[0];
12983     Value *Src = Ops[1];
12984
12985     // Extract the 0'th element of the source vector.
12986     if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
12987         BuiltinID == X86::BI__builtin_ia32_movntss)
12988       Src = Builder.CreateExtractElement(Src, (uint64_t)0, "extract");
12989
12990     // Convert the type of the pointer to a pointer to the stored type.
12991     Value *BC = Builder.CreateBitCast(
12992         Ptr, llvm::PointerType::getUnqual(Src->getType()), "cast");
12993
12994     // Unaligned nontemporal store of the scalar value.
12995     StoreInst *SI = Builder.CreateDefaultAlignedStore(Src, BC);
12996     SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
12997     SI->setAlignment(llvm::Align(1));
12998     return SI;
12999   }
13000   // Rotate is a special case of funnel shift - 1st 2 args are the same.
13001   case X86::BI__builtin_ia32_vprotb:
13002   case X86::BI__builtin_ia32_vprotw:
13003   case X86::BI__builtin_ia32_vprotd:
13004   case X86::BI__builtin_ia32_vprotq:
13005   case X86::BI__builtin_ia32_vprotbi:
13006   case X86::BI__builtin_ia32_vprotwi:
13007   case X86::BI__builtin_ia32_vprotdi:
13008   case X86::BI__builtin_ia32_vprotqi:
13009   case X86::BI__builtin_ia32_prold128:
13010   case X86::BI__builtin_ia32_prold256:
13011   case X86::BI__builtin_ia32_prold512:
13012   case X86::BI__builtin_ia32_prolq128:
13013   case X86::BI__builtin_ia32_prolq256:
13014   case X86::BI__builtin_ia32_prolq512:
13015   case X86::BI__builtin_ia32_prolvd128:
13016   case X86::BI__builtin_ia32_prolvd256:
13017   case X86::BI__builtin_ia32_prolvd512:
13018   case X86::BI__builtin_ia32_prolvq128:
13019   case X86::BI__builtin_ia32_prolvq256:
13020   case X86::BI__builtin_ia32_prolvq512:
13021     return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], false);
13022   case X86::BI__builtin_ia32_prord128:
13023   case X86::BI__builtin_ia32_prord256:
13024   case X86::BI__builtin_ia32_prord512:
13025   case X86::BI__builtin_ia32_prorq128:
13026   case X86::BI__builtin_ia32_prorq256:
13027   case X86::BI__builtin_ia32_prorq512:
13028   case X86::BI__builtin_ia32_prorvd128:
13029   case X86::BI__builtin_ia32_prorvd256:
13030   case X86::BI__builtin_ia32_prorvd512:
13031   case X86::BI__builtin_ia32_prorvq128:
13032   case X86::BI__builtin_ia32_prorvq256:
13033   case X86::BI__builtin_ia32_prorvq512:
13034     return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], true);
13035   case X86::BI__builtin_ia32_selectb_128:
13036   case X86::BI__builtin_ia32_selectb_256:
13037   case X86::BI__builtin_ia32_selectb_512:
13038   case X86::BI__builtin_ia32_selectw_128:
13039   case X86::BI__builtin_ia32_selectw_256:
13040   case X86::BI__builtin_ia32_selectw_512:
13041   case X86::BI__builtin_ia32_selectd_128:
13042   case X86::BI__builtin_ia32_selectd_256:
13043   case X86::BI__builtin_ia32_selectd_512:
13044   case X86::BI__builtin_ia32_selectq_128:
13045   case X86::BI__builtin_ia32_selectq_256:
13046   case X86::BI__builtin_ia32_selectq_512:
13047   case X86::BI__builtin_ia32_selectps_128:
13048   case X86::BI__builtin_ia32_selectps_256:
13049   case X86::BI__builtin_ia32_selectps_512:
13050   case X86::BI__builtin_ia32_selectpd_128:
13051   case X86::BI__builtin_ia32_selectpd_256:
13052   case X86::BI__builtin_ia32_selectpd_512:
13053     return EmitX86Select(*this, Ops[0], Ops[1], Ops[2]);
13054   case X86::BI__builtin_ia32_selectss_128:
13055   case X86::BI__builtin_ia32_selectsd_128: {
13056     Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
13057     Value *B = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
13058     A = EmitX86ScalarSelect(*this, Ops[0], A, B);
13059     return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
13060   }
13061   case X86::BI__builtin_ia32_cmpb128_mask:
13062   case X86::BI__builtin_ia32_cmpb256_mask:
13063   case X86::BI__builtin_ia32_cmpb512_mask:
13064   case X86::BI__builtin_ia32_cmpw128_mask:
13065   case X86::BI__builtin_ia32_cmpw256_mask:
13066   case X86::BI__builtin_ia32_cmpw512_mask:
13067   case X86::BI__builtin_ia32_cmpd128_mask:
13068   case X86::BI__builtin_ia32_cmpd256_mask:
13069   case X86::BI__builtin_ia32_cmpd512_mask:
13070   case X86::BI__builtin_ia32_cmpq128_mask:
13071   case X86::BI__builtin_ia32_cmpq256_mask:
13072   case X86::BI__builtin_ia32_cmpq512_mask: {
13073     unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
13074     return EmitX86MaskedCompare(*this, CC, true, Ops);
13075   }
13076   case X86::BI__builtin_ia32_ucmpb128_mask:
13077   case X86::BI__builtin_ia32_ucmpb256_mask:
13078   case X86::BI__builtin_ia32_ucmpb512_mask:
13079   case X86::BI__builtin_ia32_ucmpw128_mask:
13080   case X86::BI__builtin_ia32_ucmpw256_mask:
13081   case X86::BI__builtin_ia32_ucmpw512_mask:
13082   case X86::BI__builtin_ia32_ucmpd128_mask:
13083   case X86::BI__builtin_ia32_ucmpd256_mask:
13084   case X86::BI__builtin_ia32_ucmpd512_mask:
13085   case X86::BI__builtin_ia32_ucmpq128_mask:
13086   case X86::BI__builtin_ia32_ucmpq256_mask:
13087   case X86::BI__builtin_ia32_ucmpq512_mask: {
13088     unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
13089     return EmitX86MaskedCompare(*this, CC, false, Ops);
13090   }
13091   case X86::BI__builtin_ia32_vpcomb:
13092   case X86::BI__builtin_ia32_vpcomw:
13093   case X86::BI__builtin_ia32_vpcomd:
13094   case X86::BI__builtin_ia32_vpcomq:
13095     return EmitX86vpcom(*this, Ops, true);
13096   case X86::BI__builtin_ia32_vpcomub:
13097   case X86::BI__builtin_ia32_vpcomuw:
13098   case X86::BI__builtin_ia32_vpcomud:
13099   case X86::BI__builtin_ia32_vpcomuq:
13100     return EmitX86vpcom(*this, Ops, false);
13101
13102   case X86::BI__builtin_ia32_kortestcqi:
13103   case X86::BI__builtin_ia32_kortestchi:
13104   case X86::BI__builtin_ia32_kortestcsi:
13105   case X86::BI__builtin_ia32_kortestcdi: {
13106     Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops);
13107     Value *C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
13108     Value *Cmp = Builder.CreateICmpEQ(Or, C);
13109     return Builder.CreateZExt(Cmp, ConvertType(E->getType()));
13110   }
13111   case X86::BI__builtin_ia32_kortestzqi:
13112   case X86::BI__builtin_ia32_kortestzhi:
13113   case X86::BI__builtin_ia32_kortestzsi:
13114   case X86::BI__builtin_ia32_kortestzdi: {
13115     Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops);
13116     Value *C = llvm::Constant::getNullValue(Ops[0]->getType());
13117     Value *Cmp = Builder.CreateICmpEQ(Or, C);
13118     return Builder.CreateZExt(Cmp, ConvertType(E->getType()));
13119   }
13120
13121   case X86::BI__builtin_ia32_ktestcqi:
13122   case X86::BI__builtin_ia32_ktestzqi:
13123   case X86::BI__builtin_ia32_ktestchi:
13124   case X86::BI__builtin_ia32_ktestzhi:
13125   case X86::BI__builtin_ia32_ktestcsi:
13126   case X86::BI__builtin_ia32_ktestzsi:
13127   case X86::BI__builtin_ia32_ktestcdi:
13128   case X86::BI__builtin_ia32_ktestzdi: {
13129     Intrinsic::ID IID;
13130     switch (BuiltinID) {
13131     default: llvm_unreachable("Unsupported intrinsic!");
13132     case X86::BI__builtin_ia32_ktestcqi:
13133       IID = Intrinsic::x86_avx512_ktestc_b;
13134       break;
13135     case X86::BI__builtin_ia32_ktestzqi:
13136       IID = Intrinsic::x86_avx512_ktestz_b;
13137       break;
13138     case X86::BI__builtin_ia32_ktestchi:
13139       IID = Intrinsic::x86_avx512_ktestc_w;
13140       break;
13141     case X86::BI__builtin_ia32_ktestzhi:
13142       IID = Intrinsic::x86_avx512_ktestz_w;
13143       break;
13144     case X86::BI__builtin_ia32_ktestcsi:
13145       IID = Intrinsic::x86_avx512_ktestc_d;
13146       break;
13147     case X86::BI__builtin_ia32_ktestzsi:
13148       IID = Intrinsic::x86_avx512_ktestz_d;
13149       break;
13150     case X86::BI__builtin_ia32_ktestcdi:
13151       IID = Intrinsic::x86_avx512_ktestc_q;
13152       break;
13153     case X86::BI__builtin_ia32_ktestzdi:
13154       IID = Intrinsic::x86_avx512_ktestz_q;
13155       break;
13156     }
13157
13158     unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13159     Value *LHS = getMaskVecValue(*this, Ops[0], NumElts);
13160     Value *RHS = getMaskVecValue(*this, Ops[1], NumElts);
13161     Function *Intr = CGM.getIntrinsic(IID);
13162     return Builder.CreateCall(Intr, {LHS, RHS});
13163   }
13164
13165   case X86::BI__builtin_ia32_kaddqi:
13166   case X86::BI__builtin_ia32_kaddhi:
13167   case X86::BI__builtin_ia32_kaddsi:
13168   case X86::BI__builtin_ia32_kadddi: {
13169     Intrinsic::ID IID;
13170     switch (BuiltinID) {
13171     default: llvm_unreachable("Unsupported intrinsic!");
13172     case X86::BI__builtin_ia32_kaddqi:
13173       IID = Intrinsic::x86_avx512_kadd_b;
13174       break;
13175     case X86::BI__builtin_ia32_kaddhi:
13176       IID = Intrinsic::x86_avx512_kadd_w;
13177       break;
13178     case X86::BI__builtin_ia32_kaddsi:
13179       IID = Intrinsic::x86_avx512_kadd_d;
13180       break;
13181     case X86::BI__builtin_ia32_kadddi:
13182       IID = Intrinsic::x86_avx512_kadd_q;
13183       break;
13184     }
13185
13186     unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13187     Value *LHS = getMaskVecValue(*this, Ops[0], NumElts);
13188     Value *RHS = getMaskVecValue(*this, Ops[1], NumElts);
13189     Function *Intr = CGM.getIntrinsic(IID);
13190     Value *Res = Builder.CreateCall(Intr, {LHS, RHS});
13191     return Builder.CreateBitCast(Res, Ops[0]->getType());
13192   }
13193   case X86::BI__builtin_ia32_kandqi:
13194   case X86::BI__builtin_ia32_kandhi:
13195   case X86::BI__builtin_ia32_kandsi:
13196   case X86::BI__builtin_ia32_kanddi:
13197     return EmitX86MaskLogic(*this, Instruction::And, Ops);
13198   case X86::BI__builtin_ia32_kandnqi:
13199   case X86::BI__builtin_ia32_kandnhi:
13200   case X86::BI__builtin_ia32_kandnsi:
13201   case X86::BI__builtin_ia32_kandndi:
13202     return EmitX86MaskLogic(*this, Instruction::And, Ops, true);
13203   case X86::BI__builtin_ia32_korqi:
13204   case X86::BI__builtin_ia32_korhi:
13205   case X86::BI__builtin_ia32_korsi:
13206   case X86::BI__builtin_ia32_kordi:
13207     return EmitX86MaskLogic(*this, Instruction::Or, Ops);
13208   case X86::BI__builtin_ia32_kxnorqi:
13209   case X86::BI__builtin_ia32_kxnorhi:
13210   case X86::BI__builtin_ia32_kxnorsi:
13211   case X86::BI__builtin_ia32_kxnordi:
13212     return EmitX86MaskLogic(*this, Instruction::Xor, Ops, true);
13213   case X86::BI__builtin_ia32_kxorqi:
13214   case X86::BI__builtin_ia32_kxorhi:
13215   case X86::BI__builtin_ia32_kxorsi:
13216   case X86::BI__builtin_ia32_kxordi:
13217     return EmitX86MaskLogic(*this, Instruction::Xor,  Ops);
13218   case X86::BI__builtin_ia32_knotqi:
13219   case X86::BI__builtin_ia32_knothi:
13220   case X86::BI__builtin_ia32_knotsi:
13221   case X86::BI__builtin_ia32_knotdi: {
13222     unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13223     Value *Res = getMaskVecValue(*this, Ops[0], NumElts);
13224     return Builder.CreateBitCast(Builder.CreateNot(Res),
13225                                  Ops[0]->getType());
13226   }
13227   case X86::BI__builtin_ia32_kmovb:
13228   case X86::BI__builtin_ia32_kmovw:
13229   case X86::BI__builtin_ia32_kmovd:
13230   case X86::BI__builtin_ia32_kmovq: {
13231     // Bitcast to vXi1 type and then back to integer. This gets the mask
13232     // register type into the IR, but might be optimized out depending on
13233     // what's around it.
13234     unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13235     Value *Res = getMaskVecValue(*this, Ops[0], NumElts);
13236     return Builder.CreateBitCast(Res, Ops[0]->getType());
13237   }
13238
13239   case X86::BI__builtin_ia32_kunpckdi:
13240   case X86::BI__builtin_ia32_kunpcksi:
13241   case X86::BI__builtin_ia32_kunpckhi: {
13242     unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13243     Value *LHS = getMaskVecValue(*this, Ops[0], NumElts);
13244     Value *RHS = getMaskVecValue(*this, Ops[1], NumElts);
13245     int Indices[64];
13246     for (unsigned i = 0; i != NumElts; ++i)
13247       Indices[i] = i;
13248
13249     // First extract half of each vector. This gives better codegen than
13250     // doing it in a single shuffle.
13251     LHS = Builder.CreateShuffleVector(LHS, LHS,
13252                                       makeArrayRef(Indices, NumElts / 2));
13253     RHS = Builder.CreateShuffleVector(RHS, RHS,
13254                                       makeArrayRef(Indices, NumElts / 2));
13255     // Concat the vectors.
13256     // NOTE: Operands are swapped to match the intrinsic definition.
13257     Value *Res = Builder.CreateShuffleVector(RHS, LHS,
13258                                              makeArrayRef(Indices, NumElts));
13259     return Builder.CreateBitCast(Res, Ops[0]->getType());
13260   }
13261
13262   case X86::BI__builtin_ia32_vplzcntd_128:
13263   case X86::BI__builtin_ia32_vplzcntd_256:
13264   case X86::BI__builtin_ia32_vplzcntd_512:
13265   case X86::BI__builtin_ia32_vplzcntq_128:
13266   case X86::BI__builtin_ia32_vplzcntq_256:
13267   case X86::BI__builtin_ia32_vplzcntq_512: {
13268     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
13269     return Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)});
13270   }
13271   case X86::BI__builtin_ia32_sqrtss:
13272   case X86::BI__builtin_ia32_sqrtsd: {
13273     Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
13274     Function *F;
13275     if (Builder.getIsFPConstrained()) {
13276       F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt,
13277                            A->getType());
13278       A = Builder.CreateConstrainedFPCall(F, {A});
13279     } else {
13280       F = CGM.getIntrinsic(Intrinsic::sqrt, A->getType());
13281       A = Builder.CreateCall(F, {A});
13282     }
13283     return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
13284   }
13285   case X86::BI__builtin_ia32_sqrtsd_round_mask:
13286   case X86::BI__builtin_ia32_sqrtss_round_mask: {
13287     unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
13288     // Support only if the rounding mode is 4 (AKA CUR_DIRECTION),
13289     // otherwise keep the intrinsic.
13290     if (CC != 4) {
13291       Intrinsic::ID IID = BuiltinID == X86::BI__builtin_ia32_sqrtsd_round_mask ?
13292                           Intrinsic::x86_avx512_mask_sqrt_sd :
13293                           Intrinsic::x86_avx512_mask_sqrt_ss;
13294       return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
13295     }
13296     Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
13297     Function *F;
13298     if (Builder.getIsFPConstrained()) {
13299       F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt,
13300                            A->getType());
13301       A = Builder.CreateConstrainedFPCall(F, A);
13302     } else {
13303       F = CGM.getIntrinsic(Intrinsic::sqrt, A->getType());
13304       A = Builder.CreateCall(F, A);
13305     }
13306     Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
13307     A = EmitX86ScalarSelect(*this, Ops[3], A, Src);
13308     return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
13309   }
13310   case X86::BI__builtin_ia32_sqrtpd256:
13311   case X86::BI__builtin_ia32_sqrtpd:
13312   case X86::BI__builtin_ia32_sqrtps256:
13313   case X86::BI__builtin_ia32_sqrtps:
13314   case X86::BI__builtin_ia32_sqrtps512:
13315   case X86::BI__builtin_ia32_sqrtpd512: {
13316     if (Ops.size() == 2) {
13317       unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
13318       // Support only if the rounding mode is 4 (AKA CUR_DIRECTION),
13319       // otherwise keep the intrinsic.
13320       if (CC != 4) {
13321         Intrinsic::ID IID = BuiltinID == X86::BI__builtin_ia32_sqrtps512 ?
13322                             Intrinsic::x86_avx512_sqrt_ps_512 :
13323                             Intrinsic::x86_avx512_sqrt_pd_512;
13324         return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
13325       }
13326     }
13327     if (Builder.getIsFPConstrained()) {
13328       Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt,
13329                                      Ops[0]->getType());
13330       return Builder.CreateConstrainedFPCall(F, Ops[0]);
13331     } else {
13332       Function *F = CGM.getIntrinsic(Intrinsic::sqrt, Ops[0]->getType());
13333       return Builder.CreateCall(F, Ops[0]);
13334     }
13335   }
13336   case X86::BI__builtin_ia32_pabsb128:
13337   case X86::BI__builtin_ia32_pabsw128:
13338   case X86::BI__builtin_ia32_pabsd128:
13339   case X86::BI__builtin_ia32_pabsb256:
13340   case X86::BI__builtin_ia32_pabsw256:
13341   case X86::BI__builtin_ia32_pabsd256:
13342   case X86::BI__builtin_ia32_pabsq128:
13343   case X86::BI__builtin_ia32_pabsq256:
13344   case X86::BI__builtin_ia32_pabsb512:
13345   case X86::BI__builtin_ia32_pabsw512:
13346   case X86::BI__builtin_ia32_pabsd512:
13347   case X86::BI__builtin_ia32_pabsq512:
13348     return EmitX86Abs(*this, Ops);
13349
13350   case X86::BI__builtin_ia32_pmaxsb128:
13351   case X86::BI__builtin_ia32_pmaxsw128:
13352   case X86::BI__builtin_ia32_pmaxsd128:
13353   case X86::BI__builtin_ia32_pmaxsq128:
13354   case X86::BI__builtin_ia32_pmaxsb256:
13355   case X86::BI__builtin_ia32_pmaxsw256:
13356   case X86::BI__builtin_ia32_pmaxsd256:
13357   case X86::BI__builtin_ia32_pmaxsq256:
13358   case X86::BI__builtin_ia32_pmaxsb512:
13359   case X86::BI__builtin_ia32_pmaxsw512:
13360   case X86::BI__builtin_ia32_pmaxsd512:
13361   case X86::BI__builtin_ia32_pmaxsq512:
13362     return EmitX86MinMax(*this, ICmpInst::ICMP_SGT, Ops);
13363   case X86::BI__builtin_ia32_pmaxub128:
13364   case X86::BI__builtin_ia32_pmaxuw128:
13365   case X86::BI__builtin_ia32_pmaxud128:
13366   case X86::BI__builtin_ia32_pmaxuq128:
13367   case X86::BI__builtin_ia32_pmaxub256:
13368   case X86::BI__builtin_ia32_pmaxuw256:
13369   case X86::BI__builtin_ia32_pmaxud256:
13370   case X86::BI__builtin_ia32_pmaxuq256:
13371   case X86::BI__builtin_ia32_pmaxub512:
13372   case X86::BI__builtin_ia32_pmaxuw512:
13373   case X86::BI__builtin_ia32_pmaxud512:
13374   case X86::BI__builtin_ia32_pmaxuq512:
13375     return EmitX86MinMax(*this, ICmpInst::ICMP_UGT, Ops);
13376   case X86::BI__builtin_ia32_pminsb128:
13377   case X86::BI__builtin_ia32_pminsw128:
13378   case X86::BI__builtin_ia32_pminsd128:
13379   case X86::BI__builtin_ia32_pminsq128:
13380   case X86::BI__builtin_ia32_pminsb256:
13381   case X86::BI__builtin_ia32_pminsw256:
13382   case X86::BI__builtin_ia32_pminsd256:
13383   case X86::BI__builtin_ia32_pminsq256:
13384   case X86::BI__builtin_ia32_pminsb512:
13385   case X86::BI__builtin_ia32_pminsw512:
13386   case X86::BI__builtin_ia32_pminsd512:
13387   case X86::BI__builtin_ia32_pminsq512:
13388     return EmitX86MinMax(*this, ICmpInst::ICMP_SLT, Ops);
13389   case X86::BI__builtin_ia32_pminub128:
13390   case X86::BI__builtin_ia32_pminuw128:
13391   case X86::BI__builtin_ia32_pminud128:
13392   case X86::BI__builtin_ia32_pminuq128:
13393   case X86::BI__builtin_ia32_pminub256:
13394   case X86::BI__builtin_ia32_pminuw256:
13395   case X86::BI__builtin_ia32_pminud256:
13396   case X86::BI__builtin_ia32_pminuq256:
13397   case X86::BI__builtin_ia32_pminub512:
13398   case X86::BI__builtin_ia32_pminuw512:
13399   case X86::BI__builtin_ia32_pminud512:
13400   case X86::BI__builtin_ia32_pminuq512:
13401     return EmitX86MinMax(*this, ICmpInst::ICMP_ULT, Ops);
13402
13403   case X86::BI__builtin_ia32_pmuludq128:
13404   case X86::BI__builtin_ia32_pmuludq256:
13405   case X86::BI__builtin_ia32_pmuludq512:
13406     return EmitX86Muldq(*this, /*IsSigned*/false, Ops);
13407
13408   case X86::BI__builtin_ia32_pmuldq128:
13409   case X86::BI__builtin_ia32_pmuldq256:
13410   case X86::BI__builtin_ia32_pmuldq512:
13411     return EmitX86Muldq(*this, /*IsSigned*/true, Ops);
13412
13413   case X86::BI__builtin_ia32_pternlogd512_mask:
13414   case X86::BI__builtin_ia32_pternlogq512_mask:
13415   case X86::BI__builtin_ia32_pternlogd128_mask:
13416   case X86::BI__builtin_ia32_pternlogd256_mask:
13417   case X86::BI__builtin_ia32_pternlogq128_mask:
13418   case X86::BI__builtin_ia32_pternlogq256_mask:
13419     return EmitX86Ternlog(*this, /*ZeroMask*/false, Ops);
13420
13421   case X86::BI__builtin_ia32_pternlogd512_maskz:
13422   case X86::BI__builtin_ia32_pternlogq512_maskz:
13423   case X86::BI__builtin_ia32_pternlogd128_maskz:
13424   case X86::BI__builtin_ia32_pternlogd256_maskz:
13425   case X86::BI__builtin_ia32_pternlogq128_maskz:
13426   case X86::BI__builtin_ia32_pternlogq256_maskz:
13427     return EmitX86Ternlog(*this, /*ZeroMask*/true, Ops);
13428
13429   case X86::BI__builtin_ia32_vpshldd128:
13430   case X86::BI__builtin_ia32_vpshldd256:
13431   case X86::BI__builtin_ia32_vpshldd512:
13432   case X86::BI__builtin_ia32_vpshldq128:
13433   case X86::BI__builtin_ia32_vpshldq256:
13434   case X86::BI__builtin_ia32_vpshldq512:
13435   case X86::BI__builtin_ia32_vpshldw128:
13436   case X86::BI__builtin_ia32_vpshldw256:
13437   case X86::BI__builtin_ia32_vpshldw512:
13438     return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false);
13439
13440   case X86::BI__builtin_ia32_vpshrdd128:
13441   case X86::BI__builtin_ia32_vpshrdd256:
13442   case X86::BI__builtin_ia32_vpshrdd512:
13443   case X86::BI__builtin_ia32_vpshrdq128:
13444   case X86::BI__builtin_ia32_vpshrdq256:
13445   case X86::BI__builtin_ia32_vpshrdq512:
13446   case X86::BI__builtin_ia32_vpshrdw128:
13447   case X86::BI__builtin_ia32_vpshrdw256:
13448   case X86::BI__builtin_ia32_vpshrdw512:
13449     // Ops 0 and 1 are swapped.
13450     return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true);
13451
13452   case X86::BI__builtin_ia32_vpshldvd128:
13453   case X86::BI__builtin_ia32_vpshldvd256:
13454   case X86::BI__builtin_ia32_vpshldvd512:
13455   case X86::BI__builtin_ia32_vpshldvq128:
13456   case X86::BI__builtin_ia32_vpshldvq256:
13457   case X86::BI__builtin_ia32_vpshldvq512:
13458   case X86::BI__builtin_ia32_vpshldvw128:
13459   case X86::BI__builtin_ia32_vpshldvw256:
13460   case X86::BI__builtin_ia32_vpshldvw512:
13461     return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false);
13462
13463   case X86::BI__builtin_ia32_vpshrdvd128:
13464   case X86::BI__builtin_ia32_vpshrdvd256:
13465   case X86::BI__builtin_ia32_vpshrdvd512:
13466   case X86::BI__builtin_ia32_vpshrdvq128:
13467   case X86::BI__builtin_ia32_vpshrdvq256:
13468   case X86::BI__builtin_ia32_vpshrdvq512:
13469   case X86::BI__builtin_ia32_vpshrdvw128:
13470   case X86::BI__builtin_ia32_vpshrdvw256:
13471   case X86::BI__builtin_ia32_vpshrdvw512:
13472     // Ops 0 and 1 are swapped.
13473     return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true);
13474
13475   // 3DNow!
13476   case X86::BI__builtin_ia32_pswapdsf:
13477   case X86::BI__builtin_ia32_pswapdsi: {
13478     llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext());
13479     Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast");
13480     llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd);
13481     return Builder.CreateCall(F, Ops, "pswapd");
13482   }
13483   case X86::BI__builtin_ia32_rdrand16_step:
13484   case X86::BI__builtin_ia32_rdrand32_step:
13485   case X86::BI__builtin_ia32_rdrand64_step:
13486   case X86::BI__builtin_ia32_rdseed16_step:
13487   case X86::BI__builtin_ia32_rdseed32_step:
13488   case X86::BI__builtin_ia32_rdseed64_step: {
13489     Intrinsic::ID ID;
13490     switch (BuiltinID) {
13491     default: llvm_unreachable("Unsupported intrinsic!");
13492     case X86::BI__builtin_ia32_rdrand16_step:
13493       ID = Intrinsic::x86_rdrand_16;
13494       break;
13495     case X86::BI__builtin_ia32_rdrand32_step:
13496       ID = Intrinsic::x86_rdrand_32;
13497       break;
13498     case X86::BI__builtin_ia32_rdrand64_step:
13499       ID = Intrinsic::x86_rdrand_64;
13500       break;
13501     case X86::BI__builtin_ia32_rdseed16_step:
13502       ID = Intrinsic::x86_rdseed_16;
13503       break;
13504     case X86::BI__builtin_ia32_rdseed32_step:
13505       ID = Intrinsic::x86_rdseed_32;
13506       break;
13507     case X86::BI__builtin_ia32_rdseed64_step:
13508       ID = Intrinsic::x86_rdseed_64;
13509       break;
13510     }
13511
13512     Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
13513     Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0),
13514                                       Ops[0]);
13515     return Builder.CreateExtractValue(Call, 1);
13516   }
13517   case X86::BI__builtin_ia32_addcarryx_u32:
13518   case X86::BI__builtin_ia32_addcarryx_u64:
13519   case X86::BI__builtin_ia32_subborrow_u32:
13520   case X86::BI__builtin_ia32_subborrow_u64: {
13521     Intrinsic::ID IID;
13522     switch (BuiltinID) {
13523     default: llvm_unreachable("Unsupported intrinsic!");
13524     case X86::BI__builtin_ia32_addcarryx_u32:
13525       IID = Intrinsic::x86_addcarry_32;
13526       break;
13527     case X86::BI__builtin_ia32_addcarryx_u64:
13528       IID = Intrinsic::x86_addcarry_64;
13529       break;
13530     case X86::BI__builtin_ia32_subborrow_u32:
13531       IID = Intrinsic::x86_subborrow_32;
13532       break;
13533     case X86::BI__builtin_ia32_subborrow_u64:
13534       IID = Intrinsic::x86_subborrow_64;
13535       break;
13536     }
13537
13538     Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID),
13539                                      { Ops[0], Ops[1], Ops[2] });
13540     Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
13541                                       Ops[3]);
13542     return Builder.CreateExtractValue(Call, 0);
13543   }
13544
13545   case X86::BI__builtin_ia32_fpclassps128_mask:
13546   case X86::BI__builtin_ia32_fpclassps256_mask:
13547   case X86::BI__builtin_ia32_fpclassps512_mask:
13548   case X86::BI__builtin_ia32_fpclasspd128_mask:
13549   case X86::BI__builtin_ia32_fpclasspd256_mask:
13550   case X86::BI__builtin_ia32_fpclasspd512_mask: {
13551     unsigned NumElts =
13552         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
13553     Value *MaskIn = Ops[2];
13554     Ops.erase(&Ops[2]);
13555
13556     Intrinsic::ID ID;
13557     switch (BuiltinID) {
13558     default: llvm_unreachable("Unsupported intrinsic!");
13559     case X86::BI__builtin_ia32_fpclassps128_mask:
13560       ID = Intrinsic::x86_avx512_fpclass_ps_128;
13561       break;
13562     case X86::BI__builtin_ia32_fpclassps256_mask:
13563       ID = Intrinsic::x86_avx512_fpclass_ps_256;
13564       break;
13565     case X86::BI__builtin_ia32_fpclassps512_mask:
13566       ID = Intrinsic::x86_avx512_fpclass_ps_512;
13567       break;
13568     case X86::BI__builtin_ia32_fpclasspd128_mask:
13569       ID = Intrinsic::x86_avx512_fpclass_pd_128;
13570       break;
13571     case X86::BI__builtin_ia32_fpclasspd256_mask:
13572       ID = Intrinsic::x86_avx512_fpclass_pd_256;
13573       break;
13574     case X86::BI__builtin_ia32_fpclasspd512_mask:
13575       ID = Intrinsic::x86_avx512_fpclass_pd_512;
13576       break;
13577     }
13578
13579     Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
13580     return EmitX86MaskedCompareResult(*this, Fpclass, NumElts, MaskIn);
13581   }
13582
13583   case X86::BI__builtin_ia32_vp2intersect_q_512:
13584   case X86::BI__builtin_ia32_vp2intersect_q_256:
13585   case X86::BI__builtin_ia32_vp2intersect_q_128:
13586   case X86::BI__builtin_ia32_vp2intersect_d_512:
13587   case X86::BI__builtin_ia32_vp2intersect_d_256:
13588   case X86::BI__builtin_ia32_vp2intersect_d_128: {
13589     unsigned NumElts =
13590         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
13591     Intrinsic::ID ID;
13592
13593     switch (BuiltinID) {
13594     default: llvm_unreachable("Unsupported intrinsic!");
13595     case X86::BI__builtin_ia32_vp2intersect_q_512:
13596       ID = Intrinsic::x86_avx512_vp2intersect_q_512;
13597       break;
13598     case X86::BI__builtin_ia32_vp2intersect_q_256:
13599       ID = Intrinsic::x86_avx512_vp2intersect_q_256;
13600       break;
13601     case X86::BI__builtin_ia32_vp2intersect_q_128:
13602       ID = Intrinsic::x86_avx512_vp2intersect_q_128;
13603       break;
13604     case X86::BI__builtin_ia32_vp2intersect_d_512:
13605       ID = Intrinsic::x86_avx512_vp2intersect_d_512;
13606       break;
13607     case X86::BI__builtin_ia32_vp2intersect_d_256:
13608       ID = Intrinsic::x86_avx512_vp2intersect_d_256;
13609       break;
13610     case X86::BI__builtin_ia32_vp2intersect_d_128:
13611       ID = Intrinsic::x86_avx512_vp2intersect_d_128;
13612       break;
13613     }
13614
13615     Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID), {Ops[0], Ops[1]});
13616     Value *Result = Builder.CreateExtractValue(Call, 0);
13617     Result = EmitX86MaskedCompareResult(*this, Result, NumElts, nullptr);
13618     Builder.CreateDefaultAlignedStore(Result, Ops[2]);
13619
13620     Result = Builder.CreateExtractValue(Call, 1);
13621     Result = EmitX86MaskedCompareResult(*this, Result, NumElts, nullptr);
13622     return Builder.CreateDefaultAlignedStore(Result, Ops[3]);
13623   }
13624
13625   case X86::BI__builtin_ia32_vpmultishiftqb128:
13626   case X86::BI__builtin_ia32_vpmultishiftqb256:
13627   case X86::BI__builtin_ia32_vpmultishiftqb512: {
13628     Intrinsic::ID ID;
13629     switch (BuiltinID) {
13630     default: llvm_unreachable("Unsupported intrinsic!");
13631     case X86::BI__builtin_ia32_vpmultishiftqb128:
13632       ID = Intrinsic::x86_avx512_pmultishift_qb_128;
13633       break;
13634     case X86::BI__builtin_ia32_vpmultishiftqb256:
13635       ID = Intrinsic::x86_avx512_pmultishift_qb_256;
13636       break;
13637     case X86::BI__builtin_ia32_vpmultishiftqb512:
13638       ID = Intrinsic::x86_avx512_pmultishift_qb_512;
13639       break;
13640     }
13641
13642     return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
13643   }
13644
13645   case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
13646   case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
13647   case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
13648     unsigned NumElts =
13649         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
13650     Value *MaskIn = Ops[2];
13651     Ops.erase(&Ops[2]);
13652
13653     Intrinsic::ID ID;
13654     switch (BuiltinID) {
13655     default: llvm_unreachable("Unsupported intrinsic!");
13656     case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
13657       ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
13658       break;
13659     case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
13660       ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
13661       break;
13662     case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
13663       ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
13664       break;
13665     }
13666
13667     Value *Shufbit = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
13668     return EmitX86MaskedCompareResult(*this, Shufbit, NumElts, MaskIn);
13669   }
13670
13671   // packed comparison intrinsics
13672   case X86::BI__builtin_ia32_cmpeqps:
13673   case X86::BI__builtin_ia32_cmpeqpd:
13674     return getVectorFCmpIR(CmpInst::FCMP_OEQ, /*IsSignaling*/false);
13675   case X86::BI__builtin_ia32_cmpltps:
13676   case X86::BI__builtin_ia32_cmpltpd:
13677     return getVectorFCmpIR(CmpInst::FCMP_OLT, /*IsSignaling*/true);
13678   case X86::BI__builtin_ia32_cmpleps:
13679   case X86::BI__builtin_ia32_cmplepd:
13680     return getVectorFCmpIR(CmpInst::FCMP_OLE, /*IsSignaling*/true);
13681   case X86::BI__builtin_ia32_cmpunordps:
13682   case X86::BI__builtin_ia32_cmpunordpd:
13683     return getVectorFCmpIR(CmpInst::FCMP_UNO, /*IsSignaling*/false);
13684   case X86::BI__builtin_ia32_cmpneqps:
13685   case X86::BI__builtin_ia32_cmpneqpd:
13686     return getVectorFCmpIR(CmpInst::FCMP_UNE, /*IsSignaling*/false);
13687   case X86::BI__builtin_ia32_cmpnltps:
13688   case X86::BI__builtin_ia32_cmpnltpd:
13689     return getVectorFCmpIR(CmpInst::FCMP_UGE, /*IsSignaling*/true);
13690   case X86::BI__builtin_ia32_cmpnleps:
13691   case X86::BI__builtin_ia32_cmpnlepd:
13692     return getVectorFCmpIR(CmpInst::FCMP_UGT, /*IsSignaling*/true);
13693   case X86::BI__builtin_ia32_cmpordps:
13694   case X86::BI__builtin_ia32_cmpordpd:
13695     return getVectorFCmpIR(CmpInst::FCMP_ORD, /*IsSignaling*/false);
13696   case X86::BI__builtin_ia32_cmpps:
13697   case X86::BI__builtin_ia32_cmpps256:
13698   case X86::BI__builtin_ia32_cmppd:
13699   case X86::BI__builtin_ia32_cmppd256:
13700   case X86::BI__builtin_ia32_cmpps128_mask:
13701   case X86::BI__builtin_ia32_cmpps256_mask:
13702   case X86::BI__builtin_ia32_cmpps512_mask:
13703   case X86::BI__builtin_ia32_cmppd128_mask:
13704   case X86::BI__builtin_ia32_cmppd256_mask:
13705   case X86::BI__builtin_ia32_cmppd512_mask: {
13706     // Lowering vector comparisons to fcmp instructions, while
13707     // ignoring signalling behaviour requested
13708     // ignoring rounding mode requested
13709     // This is is only possible as long as FENV_ACCESS is not implemented.
13710     // See also: https://reviews.llvm.org/D45616
13711
13712     // The third argument is the comparison condition, and integer in the
13713     // range [0, 31]
13714     unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
13715
13716     // Lowering to IR fcmp instruction.
13717     // Ignoring requested signaling behaviour,
13718     // e.g. both _CMP_GT_OS & _CMP_GT_OQ are translated to FCMP_OGT.
13719     FCmpInst::Predicate Pred;
13720     bool IsSignaling;
13721     // Predicates for 16-31 repeat the 0-15 predicates. Only the signalling
13722     // behavior is inverted. We'll handle that after the switch.
13723     switch (CC & 0xf) {
13724     case 0x00: Pred = FCmpInst::FCMP_OEQ;   IsSignaling = false; break;
13725     case 0x01: Pred = FCmpInst::FCMP_OLT;   IsSignaling = true;  break;
13726     case 0x02: Pred = FCmpInst::FCMP_OLE;   IsSignaling = true;  break;
13727     case 0x03: Pred = FCmpInst::FCMP_UNO;   IsSignaling = false; break;
13728     case 0x04: Pred = FCmpInst::FCMP_UNE;   IsSignaling = false; break;
13729     case 0x05: Pred = FCmpInst::FCMP_UGE;   IsSignaling = true;  break;
13730     case 0x06: Pred = FCmpInst::FCMP_UGT;   IsSignaling = true;  break;
13731     case 0x07: Pred = FCmpInst::FCMP_ORD;   IsSignaling = false; break;
13732     case 0x08: Pred = FCmpInst::FCMP_UEQ;   IsSignaling = false; break;
13733     case 0x09: Pred = FCmpInst::FCMP_ULT;   IsSignaling = true;  break;
13734     case 0x0a: Pred = FCmpInst::FCMP_ULE;   IsSignaling = true;  break;
13735     case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling = false; break;
13736     case 0x0c: Pred = FCmpInst::FCMP_ONE;   IsSignaling = false; break;
13737     case 0x0d: Pred = FCmpInst::FCMP_OGE;   IsSignaling = true;  break;
13738     case 0x0e: Pred = FCmpInst::FCMP_OGT;   IsSignaling = true;  break;
13739     case 0x0f: Pred = FCmpInst::FCMP_TRUE;  IsSignaling = false; break;
13740     default: llvm_unreachable("Unhandled CC");
13741     }
13742
13743     // Invert the signalling behavior for 16-31.
13744     if (CC & 0x10)
13745       IsSignaling = !IsSignaling;
13746
13747     // If the predicate is true or false and we're using constrained intrinsics,
13748     // we don't have a compare intrinsic we can use. Just use the legacy X86
13749     // specific intrinsic.
13750     if ((Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE) &&
13751         Builder.getIsFPConstrained()) {
13752
13753       Intrinsic::ID IID;
13754       switch (BuiltinID) {
13755       default: llvm_unreachable("Unexpected builtin");
13756       case X86::BI__builtin_ia32_cmpps:
13757         IID = Intrinsic::x86_sse_cmp_ps;
13758         break;
13759       case X86::BI__builtin_ia32_cmpps256:
13760         IID = Intrinsic::x86_avx_cmp_ps_256;
13761         break;
13762       case X86::BI__builtin_ia32_cmppd:
13763         IID = Intrinsic::x86_sse2_cmp_pd;
13764         break;
13765       case X86::BI__builtin_ia32_cmppd256:
13766         IID = Intrinsic::x86_avx_cmp_pd_256;
13767         break;
13768       case X86::BI__builtin_ia32_cmpps512_mask:
13769         IID = Intrinsic::x86_avx512_cmp_ps_512;
13770         break;
13771       case X86::BI__builtin_ia32_cmppd512_mask:
13772         IID = Intrinsic::x86_avx512_cmp_pd_512;
13773         break;
13774       case X86::BI__builtin_ia32_cmpps128_mask:
13775         IID = Intrinsic::x86_avx512_cmp_ps_128;
13776         break;
13777       case X86::BI__builtin_ia32_cmpps256_mask:
13778         IID = Intrinsic::x86_avx512_cmp_ps_256;
13779         break;
13780       case X86::BI__builtin_ia32_cmppd128_mask:
13781         IID = Intrinsic::x86_avx512_cmp_pd_128;
13782         break;
13783       case X86::BI__builtin_ia32_cmppd256_mask:
13784         IID = Intrinsic::x86_avx512_cmp_pd_256;
13785         break;
13786       }
13787
13788       Function *Intr = CGM.getIntrinsic(IID);
13789       if (cast<llvm::VectorType>(Intr->getReturnType())
13790               ->getElementType()
13791               ->isIntegerTy(1)) {
13792         unsigned NumElts =
13793             cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
13794         Value *MaskIn = Ops[3];
13795         Ops.erase(&Ops[3]);
13796
13797         Value *Cmp = Builder.CreateCall(Intr, Ops);
13798         return EmitX86MaskedCompareResult(*this, Cmp, NumElts, MaskIn);
13799       }
13800
13801       return Builder.CreateCall(Intr, Ops);
13802     }
13803
13804     // Builtins without the _mask suffix return a vector of integers
13805     // of the same width as the input vectors
13806     switch (BuiltinID) {
13807     case X86::BI__builtin_ia32_cmpps512_mask:
13808     case X86::BI__builtin_ia32_cmppd512_mask:
13809     case X86::BI__builtin_ia32_cmpps128_mask:
13810     case X86::BI__builtin_ia32_cmpps256_mask:
13811     case X86::BI__builtin_ia32_cmppd128_mask:
13812     case X86::BI__builtin_ia32_cmppd256_mask: {
13813       // FIXME: Support SAE.
13814       unsigned NumElts =
13815           cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
13816       Value *Cmp;
13817       if (IsSignaling)
13818         Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
13819       else
13820         Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
13821       return EmitX86MaskedCompareResult(*this, Cmp, NumElts, Ops[3]);
13822     }
13823     default:
13824       return getVectorFCmpIR(Pred, IsSignaling);
13825     }
13826   }
13827
13828   // SSE scalar comparison intrinsics
13829   case X86::BI__builtin_ia32_cmpeqss:
13830     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
13831   case X86::BI__builtin_ia32_cmpltss:
13832     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
13833   case X86::BI__builtin_ia32_cmpless:
13834     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
13835   case X86::BI__builtin_ia32_cmpunordss:
13836     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
13837   case X86::BI__builtin_ia32_cmpneqss:
13838     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
13839   case X86::BI__builtin_ia32_cmpnltss:
13840     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
13841   case X86::BI__builtin_ia32_cmpnless:
13842     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
13843   case X86::BI__builtin_ia32_cmpordss:
13844     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
13845   case X86::BI__builtin_ia32_cmpeqsd:
13846     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
13847   case X86::BI__builtin_ia32_cmpltsd:
13848     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
13849   case X86::BI__builtin_ia32_cmplesd:
13850     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
13851   case X86::BI__builtin_ia32_cmpunordsd:
13852     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
13853   case X86::BI__builtin_ia32_cmpneqsd:
13854     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
13855   case X86::BI__builtin_ia32_cmpnltsd:
13856     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
13857   case X86::BI__builtin_ia32_cmpnlesd:
13858     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
13859   case X86::BI__builtin_ia32_cmpordsd:
13860     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
13861
13862   // f16c half2float intrinsics
13863   case X86::BI__builtin_ia32_vcvtph2ps:
13864   case X86::BI__builtin_ia32_vcvtph2ps256:
13865   case X86::BI__builtin_ia32_vcvtph2ps_mask:
13866   case X86::BI__builtin_ia32_vcvtph2ps256_mask:
13867   case X86::BI__builtin_ia32_vcvtph2ps512_mask:
13868     return EmitX86CvtF16ToFloatExpr(*this, Ops, ConvertType(E->getType()));
13869
13870 // AVX512 bf16 intrinsics
13871   case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
13872     Ops[2] = getMaskVecValue(
13873         *this, Ops[2],
13874         cast<llvm::VectorType>(Ops[0]->getType())->getNumElements());
13875     Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
13876     return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
13877   }
13878   case X86::BI__builtin_ia32_cvtsbf162ss_32:
13879     return EmitX86CvtBF16ToFloatExpr(*this, E, Ops);
13880
13881   case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
13882   case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
13883     Intrinsic::ID IID;
13884     switch (BuiltinID) {
13885     default: llvm_unreachable("Unsupported intrinsic!");
13886     case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
13887       IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
13888       break;
13889     case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
13890       IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
13891       break;
13892     }
13893     Value *Res = Builder.CreateCall(CGM.getIntrinsic(IID), Ops[0]);
13894     return EmitX86Select(*this, Ops[2], Res, Ops[1]);
13895   }
13896
13897   case X86::BI__emul:
13898   case X86::BI__emulu: {
13899     llvm::Type *Int64Ty = llvm::IntegerType::get(getLLVMContext(), 64);
13900     bool isSigned = (BuiltinID == X86::BI__emul);
13901     Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned);
13902     Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned);
13903     return Builder.CreateMul(LHS, RHS, "", !isSigned, isSigned);
13904   }
13905   case X86::BI__mulh:
13906   case X86::BI__umulh:
13907   case X86::BI_mul128:
13908   case X86::BI_umul128: {
13909     llvm::Type *ResType = ConvertType(E->getType());
13910     llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
13911
13912     bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
13913     Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
13914     Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
13915
13916     Value *MulResult, *HigherBits;
13917     if (IsSigned) {
13918       MulResult = Builder.CreateNSWMul(LHS, RHS);
13919       HigherBits = Builder.CreateAShr(MulResult, 64);
13920     } else {
13921       MulResult = Builder.CreateNUWMul(LHS, RHS);
13922       HigherBits = Builder.CreateLShr(MulResult, 64);
13923     }
13924     HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
13925
13926     if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
13927       return HigherBits;
13928
13929     Address HighBitsAddress = EmitPointerWithAlignment(E->getArg(2));
13930     Builder.CreateStore(HigherBits, HighBitsAddress);
13931     return Builder.CreateIntCast(MulResult, ResType, IsSigned);
13932   }
13933
13934   case X86::BI__faststorefence: {
13935     return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
13936                                llvm::SyncScope::System);
13937   }
13938   case X86::BI__shiftleft128:
13939   case X86::BI__shiftright128: {
13940     // FIXME: Once fshl/fshr no longer add an unneeded and and cmov, do this:
13941     // llvm::Function *F = CGM.getIntrinsic(
13942     //   BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
13943     //   Int64Ty);
13944     // Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
13945     // return Builder.CreateCall(F, Ops);
13946     llvm::Type *Int128Ty = Builder.getInt128Ty();
13947     Value *HighPart128 =
13948         Builder.CreateShl(Builder.CreateZExt(Ops[1], Int128Ty), 64);
13949     Value *LowPart128 = Builder.CreateZExt(Ops[0], Int128Ty);
13950     Value *Val = Builder.CreateOr(HighPart128, LowPart128);
13951     Value *Amt = Builder.CreateAnd(Builder.CreateZExt(Ops[2], Int128Ty),
13952                                    llvm::ConstantInt::get(Int128Ty, 0x3f));
13953     Value *Res;
13954     if (BuiltinID == X86::BI__shiftleft128)
13955       Res = Builder.CreateLShr(Builder.CreateShl(Val, Amt), 64);
13956     else
13957       Res = Builder.CreateLShr(Val, Amt);
13958     return Builder.CreateTrunc(Res, Int64Ty);
13959   }
13960   case X86::BI_ReadWriteBarrier:
13961   case X86::BI_ReadBarrier:
13962   case X86::BI_WriteBarrier: {
13963     return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
13964                                llvm::SyncScope::SingleThread);
13965   }
13966   case X86::BI_BitScanForward:
13967   case X86::BI_BitScanForward64:
13968     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
13969   case X86::BI_BitScanReverse:
13970   case X86::BI_BitScanReverse64:
13971     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
13972
13973   case X86::BI_InterlockedAnd64:
13974     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
13975   case X86::BI_InterlockedExchange64:
13976     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
13977   case X86::BI_InterlockedExchangeAdd64:
13978     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
13979   case X86::BI_InterlockedExchangeSub64:
13980     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
13981   case X86::BI_InterlockedOr64:
13982     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
13983   case X86::BI_InterlockedXor64:
13984     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
13985   case X86::BI_InterlockedDecrement64:
13986     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
13987   case X86::BI_InterlockedIncrement64:
13988     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
13989   case X86::BI_InterlockedCompareExchange128: {
13990     // InterlockedCompareExchange128 doesn't directly refer to 128bit ints,
13991     // instead it takes pointers to 64bit ints for Destination and
13992     // ComparandResult, and exchange is taken as two 64bit ints (high & low).
13993     // The previous value is written to ComparandResult, and success is
13994     // returned.
13995
13996     llvm::Type *Int128Ty = Builder.getInt128Ty();
13997     llvm::Type *Int128PtrTy = Int128Ty->getPointerTo();
13998
13999     Value *Destination =
14000         Builder.CreateBitCast(Ops[0], Int128PtrTy);
14001     Value *ExchangeHigh128 = Builder.CreateZExt(Ops[1], Int128Ty);
14002     Value *ExchangeLow128 = Builder.CreateZExt(Ops[2], Int128Ty);
14003     Address ComparandResult(Builder.CreateBitCast(Ops[3], Int128PtrTy),
14004                             getContext().toCharUnitsFromBits(128));
14005
14006     Value *Exchange = Builder.CreateOr(
14007         Builder.CreateShl(ExchangeHigh128, 64, "", false, false),
14008         ExchangeLow128);
14009
14010     Value *Comparand = Builder.CreateLoad(ComparandResult);
14011
14012     AtomicCmpXchgInst *CXI =
14013         Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
14014                                     AtomicOrdering::SequentiallyConsistent,
14015                                     AtomicOrdering::SequentiallyConsistent);
14016     CXI->setVolatile(true);
14017
14018     // Write the result back to the inout pointer.
14019     Builder.CreateStore(Builder.CreateExtractValue(CXI, 0), ComparandResult);
14020
14021     // Get the success boolean and zero extend it to i8.
14022     Value *Success = Builder.CreateExtractValue(CXI, 1);
14023     return Builder.CreateZExt(Success, ConvertType(E->getType()));
14024   }
14025
14026   case X86::BI_AddressOfReturnAddress: {
14027     Function *F =
14028         CGM.getIntrinsic(Intrinsic::addressofreturnaddress, AllocaInt8PtrTy);
14029     return Builder.CreateCall(F);
14030   }
14031   case X86::BI__stosb: {
14032     // We treat __stosb as a volatile memset - it may not generate "rep stosb"
14033     // instruction, but it will create a memset that won't be optimized away.
14034     return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], Align(1), true);
14035   }
14036   case X86::BI__ud2:
14037     // llvm.trap makes a ud2a instruction on x86.
14038     return EmitTrapCall(Intrinsic::trap);
14039   case X86::BI__int2c: {
14040     // This syscall signals a driver assertion failure in x86 NT kernels.
14041     llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
14042     llvm::InlineAsm *IA =
14043         llvm::InlineAsm::get(FTy, "int $$0x2c", "", /*hasSideEffects=*/true);
14044     llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
14045         getLLVMContext(), llvm::AttributeList::FunctionIndex,
14046         llvm::Attribute::NoReturn);
14047     llvm::CallInst *CI = Builder.CreateCall(IA);
14048     CI->setAttributes(NoReturnAttr);
14049     return CI;
14050   }
14051   case X86::BI__readfsbyte:
14052   case X86::BI__readfsword:
14053   case X86::BI__readfsdword:
14054   case X86::BI__readfsqword: {
14055     llvm::Type *IntTy = ConvertType(E->getType());
14056     Value *Ptr =
14057         Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(IntTy, 257));
14058     LoadInst *Load = Builder.CreateAlignedLoad(
14059         IntTy, Ptr, getContext().getTypeAlignInChars(E->getType()));
14060     Load->setVolatile(true);
14061     return Load;
14062   }
14063   case X86::BI__readgsbyte:
14064   case X86::BI__readgsword:
14065   case X86::BI__readgsdword:
14066   case X86::BI__readgsqword: {
14067     llvm::Type *IntTy = ConvertType(E->getType());
14068     Value *Ptr =
14069         Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(IntTy, 256));
14070     LoadInst *Load = Builder.CreateAlignedLoad(
14071         IntTy, Ptr, getContext().getTypeAlignInChars(E->getType()));
14072     Load->setVolatile(true);
14073     return Load;
14074   }
14075   case X86::BI__builtin_ia32_paddsb512:
14076   case X86::BI__builtin_ia32_paddsw512:
14077   case X86::BI__builtin_ia32_paddsb256:
14078   case X86::BI__builtin_ia32_paddsw256:
14079   case X86::BI__builtin_ia32_paddsb128:
14080   case X86::BI__builtin_ia32_paddsw128:
14081     return EmitX86AddSubSatExpr(*this, Ops, true, true);
14082   case X86::BI__builtin_ia32_paddusb512:
14083   case X86::BI__builtin_ia32_paddusw512:
14084   case X86::BI__builtin_ia32_paddusb256:
14085   case X86::BI__builtin_ia32_paddusw256:
14086   case X86::BI__builtin_ia32_paddusb128:
14087   case X86::BI__builtin_ia32_paddusw128:
14088     return EmitX86AddSubSatExpr(*this, Ops, false, true);
14089   case X86::BI__builtin_ia32_psubsb512:
14090   case X86::BI__builtin_ia32_psubsw512:
14091   case X86::BI__builtin_ia32_psubsb256:
14092   case X86::BI__builtin_ia32_psubsw256:
14093   case X86::BI__builtin_ia32_psubsb128:
14094   case X86::BI__builtin_ia32_psubsw128:
14095     return EmitX86AddSubSatExpr(*this, Ops, true, false);
14096   case X86::BI__builtin_ia32_psubusb512:
14097   case X86::BI__builtin_ia32_psubusw512:
14098   case X86::BI__builtin_ia32_psubusb256:
14099   case X86::BI__builtin_ia32_psubusw256:
14100   case X86::BI__builtin_ia32_psubusb128:
14101   case X86::BI__builtin_ia32_psubusw128:
14102     return EmitX86AddSubSatExpr(*this, Ops, false, false);
14103   }
14104 }
14105
14106 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
14107                                            const CallExpr *E) {
14108   SmallVector<Value*, 4> Ops;
14109
14110   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
14111     Ops.push_back(EmitScalarExpr(E->getArg(i)));
14112
14113   Intrinsic::ID ID = Intrinsic::not_intrinsic;
14114
14115   switch (BuiltinID) {
14116   default: return nullptr;
14117
14118   // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we
14119   // call __builtin_readcyclecounter.
14120   case PPC::BI__builtin_ppc_get_timebase:
14121     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter));
14122
14123   // vec_ld, vec_xl_be, vec_lvsl, vec_lvsr
14124   case PPC::BI__builtin_altivec_lvx:
14125   case PPC::BI__builtin_altivec_lvxl:
14126   case PPC::BI__builtin_altivec_lvebx:
14127   case PPC::BI__builtin_altivec_lvehx:
14128   case PPC::BI__builtin_altivec_lvewx:
14129   case PPC::BI__builtin_altivec_lvsl:
14130   case PPC::BI__builtin_altivec_lvsr:
14131   case PPC::BI__builtin_vsx_lxvd2x:
14132   case PPC::BI__builtin_vsx_lxvw4x:
14133   case PPC::BI__builtin_vsx_lxvd2x_be:
14134   case PPC::BI__builtin_vsx_lxvw4x_be:
14135   case PPC::BI__builtin_vsx_lxvl:
14136   case PPC::BI__builtin_vsx_lxvll:
14137   {
14138     if(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
14139        BuiltinID == PPC::BI__builtin_vsx_lxvll){
14140       Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy);
14141     }else {
14142       Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
14143       Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]);
14144       Ops.pop_back();
14145     }
14146
14147     switch (BuiltinID) {
14148     default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!");
14149     case PPC::BI__builtin_altivec_lvx:
14150       ID = Intrinsic::ppc_altivec_lvx;
14151       break;
14152     case PPC::BI__builtin_altivec_lvxl:
14153       ID = Intrinsic::ppc_altivec_lvxl;
14154       break;
14155     case PPC::BI__builtin_altivec_lvebx:
14156       ID = Intrinsic::ppc_altivec_lvebx;
14157       break;
14158     case PPC::BI__builtin_altivec_lvehx:
14159       ID = Intrinsic::ppc_altivec_lvehx;
14160       break;
14161     case PPC::BI__builtin_altivec_lvewx:
14162       ID = Intrinsic::ppc_altivec_lvewx;
14163       break;
14164     case PPC::BI__builtin_altivec_lvsl:
14165       ID = Intrinsic::ppc_altivec_lvsl;
14166       break;
14167     case PPC::BI__builtin_altivec_lvsr:
14168       ID = Intrinsic::ppc_altivec_lvsr;
14169       break;
14170     case PPC::BI__builtin_vsx_lxvd2x:
14171       ID = Intrinsic::ppc_vsx_lxvd2x;
14172       break;
14173     case PPC::BI__builtin_vsx_lxvw4x:
14174       ID = Intrinsic::ppc_vsx_lxvw4x;
14175       break;
14176     case PPC::BI__builtin_vsx_lxvd2x_be:
14177       ID = Intrinsic::ppc_vsx_lxvd2x_be;
14178       break;
14179     case PPC::BI__builtin_vsx_lxvw4x_be:
14180       ID = Intrinsic::ppc_vsx_lxvw4x_be;
14181       break;
14182     case PPC::BI__builtin_vsx_lxvl:
14183       ID = Intrinsic::ppc_vsx_lxvl;
14184       break;
14185     case PPC::BI__builtin_vsx_lxvll:
14186       ID = Intrinsic::ppc_vsx_lxvll;
14187       break;
14188     }
14189     llvm::Function *F = CGM.getIntrinsic(ID);
14190     return Builder.CreateCall(F, Ops, "");
14191   }
14192
14193   // vec_st, vec_xst_be
14194   case PPC::BI__builtin_altivec_stvx:
14195   case PPC::BI__builtin_altivec_stvxl:
14196   case PPC::BI__builtin_altivec_stvebx:
14197   case PPC::BI__builtin_altivec_stvehx:
14198   case PPC::BI__builtin_altivec_stvewx:
14199   case PPC::BI__builtin_vsx_stxvd2x:
14200   case PPC::BI__builtin_vsx_stxvw4x:
14201   case PPC::BI__builtin_vsx_stxvd2x_be:
14202   case PPC::BI__builtin_vsx_stxvw4x_be:
14203   case PPC::BI__builtin_vsx_stxvl:
14204   case PPC::BI__builtin_vsx_stxvll:
14205   {
14206     if(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
14207       BuiltinID == PPC::BI__builtin_vsx_stxvll ){
14208       Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
14209     }else {
14210       Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
14211       Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]);
14212       Ops.pop_back();
14213     }
14214
14215     switch (BuiltinID) {
14216     default: llvm_unreachable("Unsupported st intrinsic!");
14217     case PPC::BI__builtin_altivec_stvx:
14218       ID = Intrinsic::ppc_altivec_stvx;
14219       break;
14220     case PPC::BI__builtin_altivec_stvxl:
14221       ID = Intrinsic::ppc_altivec_stvxl;
14222       break;
14223     case PPC::BI__builtin_altivec_stvebx:
14224       ID = Intrinsic::ppc_altivec_stvebx;
14225       break;
14226     case PPC::BI__builtin_altivec_stvehx:
14227       ID = Intrinsic::ppc_altivec_stvehx;
14228       break;
14229     case PPC::BI__builtin_altivec_stvewx:
14230       ID = Intrinsic::ppc_altivec_stvewx;
14231       break;
14232     case PPC::BI__builtin_vsx_stxvd2x:
14233       ID = Intrinsic::ppc_vsx_stxvd2x;
14234       break;
14235     case PPC::BI__builtin_vsx_stxvw4x:
14236       ID = Intrinsic::ppc_vsx_stxvw4x;
14237       break;
14238     case PPC::BI__builtin_vsx_stxvd2x_be:
14239       ID = Intrinsic::ppc_vsx_stxvd2x_be;
14240       break;
14241     case PPC::BI__builtin_vsx_stxvw4x_be:
14242       ID = Intrinsic::ppc_vsx_stxvw4x_be;
14243       break;
14244     case PPC::BI__builtin_vsx_stxvl:
14245       ID = Intrinsic::ppc_vsx_stxvl;
14246       break;
14247     case PPC::BI__builtin_vsx_stxvll:
14248       ID = Intrinsic::ppc_vsx_stxvll;
14249       break;
14250     }
14251     llvm::Function *F = CGM.getIntrinsic(ID);
14252     return Builder.CreateCall(F, Ops, "");
14253   }
14254   // Square root
14255   case PPC::BI__builtin_vsx_xvsqrtsp:
14256   case PPC::BI__builtin_vsx_xvsqrtdp: {
14257     llvm::Type *ResultType = ConvertType(E->getType());
14258     Value *X = EmitScalarExpr(E->getArg(0));
14259     if (Builder.getIsFPConstrained()) {
14260       llvm::Function *F = CGM.getIntrinsic(
14261           Intrinsic::experimental_constrained_sqrt, ResultType);
14262       return Builder.CreateConstrainedFPCall(F, X);
14263     } else {
14264       llvm::Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType);
14265       return Builder.CreateCall(F, X);
14266     }
14267   }
14268   // Count leading zeros
14269   case PPC::BI__builtin_altivec_vclzb:
14270   case PPC::BI__builtin_altivec_vclzh:
14271   case PPC::BI__builtin_altivec_vclzw:
14272   case PPC::BI__builtin_altivec_vclzd: {
14273     llvm::Type *ResultType = ConvertType(E->getType());
14274     Value *X = EmitScalarExpr(E->getArg(0));
14275     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
14276     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
14277     return Builder.CreateCall(F, {X, Undef});
14278   }
14279   case PPC::BI__builtin_altivec_vctzb:
14280   case PPC::BI__builtin_altivec_vctzh:
14281   case PPC::BI__builtin_altivec_vctzw:
14282   case PPC::BI__builtin_altivec_vctzd: {
14283     llvm::Type *ResultType = ConvertType(E->getType());
14284     Value *X = EmitScalarExpr(E->getArg(0));
14285     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
14286     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
14287     return Builder.CreateCall(F, {X, Undef});
14288   }
14289   case PPC::BI__builtin_altivec_vpopcntb:
14290   case PPC::BI__builtin_altivec_vpopcnth:
14291   case PPC::BI__builtin_altivec_vpopcntw:
14292   case PPC::BI__builtin_altivec_vpopcntd: {
14293     llvm::Type *ResultType = ConvertType(E->getType());
14294     Value *X = EmitScalarExpr(E->getArg(0));
14295     llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
14296     return Builder.CreateCall(F, X);
14297   }
14298   // Copy sign
14299   case PPC::BI__builtin_vsx_xvcpsgnsp:
14300   case PPC::BI__builtin_vsx_xvcpsgndp: {
14301     llvm::Type *ResultType = ConvertType(E->getType());
14302     Value *X = EmitScalarExpr(E->getArg(0));
14303     Value *Y = EmitScalarExpr(E->getArg(1));
14304     ID = Intrinsic::copysign;
14305     llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
14306     return Builder.CreateCall(F, {X, Y});
14307   }
14308   // Rounding/truncation
14309   case PPC::BI__builtin_vsx_xvrspip:
14310   case PPC::BI__builtin_vsx_xvrdpip:
14311   case PPC::BI__builtin_vsx_xvrdpim:
14312   case PPC::BI__builtin_vsx_xvrspim:
14313   case PPC::BI__builtin_vsx_xvrdpi:
14314   case PPC::BI__builtin_vsx_xvrspi:
14315   case PPC::BI__builtin_vsx_xvrdpic:
14316   case PPC::BI__builtin_vsx_xvrspic:
14317   case PPC::BI__builtin_vsx_xvrdpiz:
14318   case PPC::BI__builtin_vsx_xvrspiz: {
14319     llvm::Type *ResultType = ConvertType(E->getType());
14320     Value *X = EmitScalarExpr(E->getArg(0));
14321     if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
14322         BuiltinID == PPC::BI__builtin_vsx_xvrspim)
14323       ID = Builder.getIsFPConstrained()
14324                ? Intrinsic::experimental_constrained_floor
14325                : Intrinsic::floor;
14326     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
14327              BuiltinID == PPC::BI__builtin_vsx_xvrspi)
14328       ID = Builder.getIsFPConstrained()
14329                ? Intrinsic::experimental_constrained_round
14330                : Intrinsic::round;
14331     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
14332              BuiltinID == PPC::BI__builtin_vsx_xvrspic)
14333       ID = Builder.getIsFPConstrained()
14334                ? Intrinsic::experimental_constrained_nearbyint
14335                : Intrinsic::nearbyint;
14336     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
14337              BuiltinID == PPC::BI__builtin_vsx_xvrspip)
14338       ID = Builder.getIsFPConstrained()
14339                ? Intrinsic::experimental_constrained_ceil
14340                : Intrinsic::ceil;
14341     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
14342              BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
14343       ID = Builder.getIsFPConstrained()
14344                ? Intrinsic::experimental_constrained_trunc
14345                : Intrinsic::trunc;
14346     llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
14347     return Builder.getIsFPConstrained() ? Builder.CreateConstrainedFPCall(F, X)
14348                                         : Builder.CreateCall(F, X);
14349   }
14350
14351   // Absolute value
14352   case PPC::BI__builtin_vsx_xvabsdp:
14353   case PPC::BI__builtin_vsx_xvabssp: {
14354     llvm::Type *ResultType = ConvertType(E->getType());
14355     Value *X = EmitScalarExpr(E->getArg(0));
14356     llvm::Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
14357     return Builder.CreateCall(F, X);
14358   }
14359
14360   // FMA variations
14361   case PPC::BI__builtin_vsx_xvmaddadp:
14362   case PPC::BI__builtin_vsx_xvmaddasp:
14363   case PPC::BI__builtin_vsx_xvnmaddadp:
14364   case PPC::BI__builtin_vsx_xvnmaddasp:
14365   case PPC::BI__builtin_vsx_xvmsubadp:
14366   case PPC::BI__builtin_vsx_xvmsubasp:
14367   case PPC::BI__builtin_vsx_xvnmsubadp:
14368   case PPC::BI__builtin_vsx_xvnmsubasp: {
14369     llvm::Type *ResultType = ConvertType(E->getType());
14370     Value *X = EmitScalarExpr(E->getArg(0));
14371     Value *Y = EmitScalarExpr(E->getArg(1));
14372     Value *Z = EmitScalarExpr(E->getArg(2));
14373     llvm::Function *F;
14374     if (Builder.getIsFPConstrained())
14375       F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
14376     else
14377       F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
14378     switch (BuiltinID) {
14379       case PPC::BI__builtin_vsx_xvmaddadp:
14380       case PPC::BI__builtin_vsx_xvmaddasp:
14381         if (Builder.getIsFPConstrained())
14382           return Builder.CreateConstrainedFPCall(F, {X, Y, Z});
14383         else
14384           return Builder.CreateCall(F, {X, Y, Z});
14385       case PPC::BI__builtin_vsx_xvnmaddadp:
14386       case PPC::BI__builtin_vsx_xvnmaddasp:
14387         if (Builder.getIsFPConstrained())
14388           return Builder.CreateFNeg(
14389               Builder.CreateConstrainedFPCall(F, {X, Y, Z}), "neg");
14390         else
14391           return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}), "neg");
14392       case PPC::BI__builtin_vsx_xvmsubadp:
14393       case PPC::BI__builtin_vsx_xvmsubasp:
14394         if (Builder.getIsFPConstrained())
14395           return Builder.CreateConstrainedFPCall(
14396               F, {X, Y, Builder.CreateFNeg(Z, "neg")});
14397         else
14398           return Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")});
14399       case PPC::BI__builtin_vsx_xvnmsubadp:
14400       case PPC::BI__builtin_vsx_xvnmsubasp:
14401         if (Builder.getIsFPConstrained())
14402           return Builder.CreateFNeg(
14403               Builder.CreateConstrainedFPCall(
14404                   F, {X, Y, Builder.CreateFNeg(Z, "neg")}),
14405               "neg");
14406         else
14407           return Builder.CreateFNeg(
14408               Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")}),
14409               "neg");
14410     }
14411     llvm_unreachable("Unknown FMA operation");
14412     return nullptr; // Suppress no-return warning
14413   }
14414
14415   case PPC::BI__builtin_vsx_insertword: {
14416     llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxinsertw);
14417
14418     // Third argument is a compile time constant int. It must be clamped to
14419     // to the range [0, 12].
14420     ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
14421     assert(ArgCI &&
14422            "Third arg to xxinsertw intrinsic must be constant integer");
14423     const int64_t MaxIndex = 12;
14424     int64_t Index = clamp(ArgCI->getSExtValue(), 0, MaxIndex);
14425
14426     // The builtin semantics don't exactly match the xxinsertw instructions
14427     // semantics (which ppc_vsx_xxinsertw follows). The builtin extracts the
14428     // word from the first argument, and inserts it in the second argument. The
14429     // instruction extracts the word from its second input register and inserts
14430     // it into its first input register, so swap the first and second arguments.
14431     std::swap(Ops[0], Ops[1]);
14432
14433     // Need to cast the second argument from a vector of unsigned int to a
14434     // vector of long long.
14435     Ops[1] =
14436         Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(Int64Ty, 2));
14437
14438     if (getTarget().isLittleEndian()) {
14439       // Reverse the double words in the vector we will extract from.
14440       Ops[0] =
14441           Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2));
14442       Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ArrayRef<int>{1, 0});
14443
14444       // Reverse the index.
14445       Index = MaxIndex - Index;
14446     }
14447
14448     // Intrinsic expects the first arg to be a vector of int.
14449     Ops[0] =
14450         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4));
14451     Ops[2] = ConstantInt::getSigned(Int32Ty, Index);
14452     return Builder.CreateCall(F, Ops);
14453   }
14454
14455   case PPC::BI__builtin_vsx_extractuword: {
14456     llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
14457
14458     // Intrinsic expects the first argument to be a vector of doublewords.
14459     Ops[0] =
14460         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2));
14461
14462     // The second argument is a compile time constant int that needs to
14463     // be clamped to the range [0, 12].
14464     ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[1]);
14465     assert(ArgCI &&
14466            "Second Arg to xxextractuw intrinsic must be a constant integer!");
14467     const int64_t MaxIndex = 12;
14468     int64_t Index = clamp(ArgCI->getSExtValue(), 0, MaxIndex);
14469
14470     if (getTarget().isLittleEndian()) {
14471       // Reverse the index.
14472       Index = MaxIndex - Index;
14473       Ops[1] = ConstantInt::getSigned(Int32Ty, Index);
14474
14475       // Emit the call, then reverse the double words of the results vector.
14476       Value *Call = Builder.CreateCall(F, Ops);
14477
14478       Value *ShuffleCall =
14479           Builder.CreateShuffleVector(Call, Call, ArrayRef<int>{1, 0});
14480       return ShuffleCall;
14481     } else {
14482       Ops[1] = ConstantInt::getSigned(Int32Ty, Index);
14483       return Builder.CreateCall(F, Ops);
14484     }
14485   }
14486
14487   case PPC::BI__builtin_vsx_xxpermdi: {
14488     ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
14489     assert(ArgCI && "Third arg must be constant integer!");
14490
14491     unsigned Index = ArgCI->getZExtValue();
14492     Ops[0] =
14493         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2));
14494     Ops[1] =
14495         Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(Int64Ty, 2));
14496
14497     // Account for endianness by treating this as just a shuffle. So we use the
14498     // same indices for both LE and BE in order to produce expected results in
14499     // both cases.
14500     int ElemIdx0 = (Index & 2) >> 1;
14501     int ElemIdx1 = 2 + (Index & 1);
14502
14503     int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
14504     Value *ShuffleCall =
14505         Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleElts);
14506     QualType BIRetType = E->getType();
14507     auto RetTy = ConvertType(BIRetType);
14508     return Builder.CreateBitCast(ShuffleCall, RetTy);
14509   }
14510
14511   case PPC::BI__builtin_vsx_xxsldwi: {
14512     ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
14513     assert(ArgCI && "Third argument must be a compile time constant");
14514     unsigned Index = ArgCI->getZExtValue() & 0x3;
14515     Ops[0] =
14516         Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4));
14517     Ops[1] =
14518         Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(Int32Ty, 4));
14519
14520     // Create a shuffle mask
14521     int ElemIdx0;
14522     int ElemIdx1;
14523     int ElemIdx2;
14524     int ElemIdx3;
14525     if (getTarget().isLittleEndian()) {
14526       // Little endian element N comes from element 8+N-Index of the
14527       // concatenated wide vector (of course, using modulo arithmetic on
14528       // the total number of elements).
14529       ElemIdx0 = (8 - Index) % 8;
14530       ElemIdx1 = (9 - Index) % 8;
14531       ElemIdx2 = (10 - Index) % 8;
14532       ElemIdx3 = (11 - Index) % 8;
14533     } else {
14534       // Big endian ElemIdx<N> = Index + N
14535       ElemIdx0 = Index;
14536       ElemIdx1 = Index + 1;
14537       ElemIdx2 = Index + 2;
14538       ElemIdx3 = Index + 3;
14539     }
14540
14541     int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
14542     Value *ShuffleCall =
14543         Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleElts);
14544     QualType BIRetType = E->getType();
14545     auto RetTy = ConvertType(BIRetType);
14546     return Builder.CreateBitCast(ShuffleCall, RetTy);
14547   }
14548
14549   case PPC::BI__builtin_pack_vector_int128: {
14550     bool isLittleEndian = getTarget().isLittleEndian();
14551     Value *UndefValue =
14552         llvm::UndefValue::get(llvm::FixedVectorType::get(Ops[0]->getType(), 2));
14553     Value *Res = Builder.CreateInsertElement(
14554         UndefValue, Ops[0], (uint64_t)(isLittleEndian ? 1 : 0));
14555     Res = Builder.CreateInsertElement(Res, Ops[1],
14556                                       (uint64_t)(isLittleEndian ? 0 : 1));
14557     return Builder.CreateBitCast(Res, ConvertType(E->getType()));
14558   }
14559
14560   case PPC::BI__builtin_unpack_vector_int128: {
14561     ConstantInt *Index = cast<ConstantInt>(Ops[1]);
14562     Value *Unpacked = Builder.CreateBitCast(
14563         Ops[0], llvm::FixedVectorType::get(ConvertType(E->getType()), 2));
14564
14565     if (getTarget().isLittleEndian())
14566       Index = ConstantInt::get(Index->getType(), 1 - Index->getZExtValue());
14567
14568     return Builder.CreateExtractElement(Unpacked, Index);
14569   }
14570   }
14571 }
14572
14573 namespace {
14574 // If \p E is not null pointer, insert address space cast to match return
14575 // type of \p E if necessary.
14576 Value *EmitAMDGPUDispatchPtr(CodeGenFunction &CGF,
14577                              const CallExpr *E = nullptr) {
14578   auto *F = CGF.CGM.getIntrinsic(Intrinsic::amdgcn_dispatch_ptr);
14579   auto *Call = CGF.Builder.CreateCall(F);
14580   Call->addAttribute(
14581       AttributeList::ReturnIndex,
14582       Attribute::getWithDereferenceableBytes(Call->getContext(), 64));
14583   Call->addAttribute(AttributeList::ReturnIndex,
14584                      Attribute::getWithAlignment(Call->getContext(), Align(4)));
14585   if (!E)
14586     return Call;
14587   QualType BuiltinRetType = E->getType();
14588   auto *RetTy = cast<llvm::PointerType>(CGF.ConvertType(BuiltinRetType));
14589   if (RetTy == Call->getType())
14590     return Call;
14591   return CGF.Builder.CreateAddrSpaceCast(Call, RetTy);
14592 }
14593
14594 // \p Index is 0, 1, and 2 for x, y, and z dimension, respectively.
14595 Value *EmitAMDGPUWorkGroupSize(CodeGenFunction &CGF, unsigned Index) {
14596   const unsigned XOffset = 4;
14597   auto *DP = EmitAMDGPUDispatchPtr(CGF);
14598   // Indexing the HSA kernel_dispatch_packet struct.
14599   auto *Offset = llvm::ConstantInt::get(CGF.Int32Ty, XOffset + Index * 2);
14600   auto *GEP = CGF.Builder.CreateGEP(DP, Offset);
14601   auto *DstTy =
14602       CGF.Int16Ty->getPointerTo(GEP->getType()->getPointerAddressSpace());
14603   auto *Cast = CGF.Builder.CreateBitCast(GEP, DstTy);
14604   auto *LD = CGF.Builder.CreateLoad(Address(Cast, CharUnits::fromQuantity(2)));
14605   llvm::MDBuilder MDHelper(CGF.getLLVMContext());
14606   llvm::MDNode *RNode = MDHelper.createRange(APInt(16, 1),
14607       APInt(16, CGF.getTarget().getMaxOpenCLWorkGroupSize() + 1));
14608   LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
14609   LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
14610       llvm::MDNode::get(CGF.getLLVMContext(), None));
14611   return LD;
14612 }
14613 } // namespace
14614
14615 // For processing memory ordering and memory scope arguments of various
14616 // amdgcn builtins.
14617 // \p Order takes a C++11 comptabile memory-ordering specifier and converts
14618 // it into LLVM's memory ordering specifier using atomic C ABI, and writes
14619 // to \p AO. \p Scope takes a const char * and converts it into AMDGCN
14620 // specific SyncScopeID and writes it to \p SSID.
14621 bool CodeGenFunction::ProcessOrderScopeAMDGCN(Value *Order, Value *Scope,
14622                                               llvm::AtomicOrdering &AO,
14623                                               llvm::SyncScope::ID &SSID) {
14624   if (isa<llvm::ConstantInt>(Order)) {
14625     int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
14626
14627     // Map C11/C++11 memory ordering to LLVM memory ordering
14628     switch (static_cast<llvm::AtomicOrderingCABI>(ord)) {
14629     case llvm::AtomicOrderingCABI::acquire:
14630       AO = llvm::AtomicOrdering::Acquire;
14631       break;
14632     case llvm::AtomicOrderingCABI::release:
14633       AO = llvm::AtomicOrdering::Release;
14634       break;
14635     case llvm::AtomicOrderingCABI::acq_rel:
14636       AO = llvm::AtomicOrdering::AcquireRelease;
14637       break;
14638     case llvm::AtomicOrderingCABI::seq_cst:
14639       AO = llvm::AtomicOrdering::SequentiallyConsistent;
14640       break;
14641     case llvm::AtomicOrderingCABI::consume:
14642     case llvm::AtomicOrderingCABI::relaxed:
14643       break;
14644     }
14645
14646     StringRef scp;
14647     llvm::getConstantStringInfo(Scope, scp);
14648     SSID = getLLVMContext().getOrInsertSyncScopeID(scp);
14649     return true;
14650   }
14651   return false;
14652 }
14653
14654 Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
14655                                               const CallExpr *E) {
14656   llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
14657   llvm::SyncScope::ID SSID;
14658   switch (BuiltinID) {
14659   case AMDGPU::BI__builtin_amdgcn_div_scale:
14660   case AMDGPU::BI__builtin_amdgcn_div_scalef: {
14661     // Translate from the intrinsics's struct return to the builtin's out
14662     // argument.
14663
14664     Address FlagOutPtr = EmitPointerWithAlignment(E->getArg(3));
14665
14666     llvm::Value *X = EmitScalarExpr(E->getArg(0));
14667     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
14668     llvm::Value *Z = EmitScalarExpr(E->getArg(2));
14669
14670     llvm::Function *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale,
14671                                            X->getType());
14672
14673     llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z});
14674
14675     llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0);
14676     llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
14677
14678     llvm::Type *RealFlagType
14679       = FlagOutPtr.getPointer()->getType()->getPointerElementType();
14680
14681     llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
14682     Builder.CreateStore(FlagExt, FlagOutPtr);
14683     return Result;
14684   }
14685   case AMDGPU::BI__builtin_amdgcn_div_fmas:
14686   case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
14687     llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
14688     llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
14689     llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
14690     llvm::Value *Src3 = EmitScalarExpr(E->getArg(3));
14691
14692     llvm::Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas,
14693                                       Src0->getType());
14694     llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
14695     return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
14696   }
14697
14698   case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
14699     return emitBinaryBuiltin(*this, E, Intrinsic::amdgcn_ds_swizzle);
14700   case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
14701     return emitBinaryBuiltin(*this, E, Intrinsic::amdgcn_mov_dpp8);
14702   case AMDGPU::BI__builtin_amdgcn_mov_dpp:
14703   case AMDGPU::BI__builtin_amdgcn_update_dpp: {
14704     llvm::SmallVector<llvm::Value *, 6> Args;
14705     for (unsigned I = 0; I != E->getNumArgs(); ++I)
14706       Args.push_back(EmitScalarExpr(E->getArg(I)));
14707     assert(Args.size() == 5 || Args.size() == 6);
14708     if (Args.size() == 5)
14709       Args.insert(Args.begin(), llvm::UndefValue::get(Args[0]->getType()));
14710     Function *F =
14711         CGM.getIntrinsic(Intrinsic::amdgcn_update_dpp, Args[0]->getType());
14712     return Builder.CreateCall(F, Args);
14713   }
14714   case AMDGPU::BI__builtin_amdgcn_div_fixup:
14715   case AMDGPU::BI__builtin_amdgcn_div_fixupf:
14716   case AMDGPU::BI__builtin_amdgcn_div_fixuph:
14717     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_div_fixup);
14718   case AMDGPU::BI__builtin_amdgcn_trig_preop:
14719   case AMDGPU::BI__builtin_amdgcn_trig_preopf:
14720     return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_trig_preop);
14721   case AMDGPU::BI__builtin_amdgcn_rcp:
14722   case AMDGPU::BI__builtin_amdgcn_rcpf:
14723   case AMDGPU::BI__builtin_amdgcn_rcph:
14724     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rcp);
14725   case AMDGPU::BI__builtin_amdgcn_sqrt:
14726   case AMDGPU::BI__builtin_amdgcn_sqrtf:
14727   case AMDGPU::BI__builtin_amdgcn_sqrth:
14728     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sqrt);
14729   case AMDGPU::BI__builtin_amdgcn_rsq:
14730   case AMDGPU::BI__builtin_amdgcn_rsqf:
14731   case AMDGPU::BI__builtin_amdgcn_rsqh:
14732     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq);
14733   case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
14734   case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
14735     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq_clamp);
14736   case AMDGPU::BI__builtin_amdgcn_sinf:
14737   case AMDGPU::BI__builtin_amdgcn_sinh:
14738     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sin);
14739   case AMDGPU::BI__builtin_amdgcn_cosf:
14740   case AMDGPU::BI__builtin_amdgcn_cosh:
14741     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_cos);
14742   case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
14743     return EmitAMDGPUDispatchPtr(*this, E);
14744   case AMDGPU::BI__builtin_amdgcn_log_clampf:
14745     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log_clamp);
14746   case AMDGPU::BI__builtin_amdgcn_ldexp:
14747   case AMDGPU::BI__builtin_amdgcn_ldexpf:
14748   case AMDGPU::BI__builtin_amdgcn_ldexph:
14749     return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp);
14750   case AMDGPU::BI__builtin_amdgcn_frexp_mant:
14751   case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
14752   case AMDGPU::BI__builtin_amdgcn_frexp_manth:
14753     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_mant);
14754   case AMDGPU::BI__builtin_amdgcn_frexp_exp:
14755   case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
14756     Value *Src0 = EmitScalarExpr(E->getArg(0));
14757     Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
14758                                 { Builder.getInt32Ty(), Src0->getType() });
14759     return Builder.CreateCall(F, Src0);
14760   }
14761   case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
14762     Value *Src0 = EmitScalarExpr(E->getArg(0));
14763     Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
14764                                 { Builder.getInt16Ty(), Src0->getType() });
14765     return Builder.CreateCall(F, Src0);
14766   }
14767   case AMDGPU::BI__builtin_amdgcn_fract:
14768   case AMDGPU::BI__builtin_amdgcn_fractf:
14769   case AMDGPU::BI__builtin_amdgcn_fracth:
14770     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_fract);
14771   case AMDGPU::BI__builtin_amdgcn_lerp:
14772     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_lerp);
14773   case AMDGPU::BI__builtin_amdgcn_ubfe:
14774     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_ubfe);
14775   case AMDGPU::BI__builtin_amdgcn_sbfe:
14776     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_sbfe);
14777   case AMDGPU::BI__builtin_amdgcn_uicmp:
14778   case AMDGPU::BI__builtin_amdgcn_uicmpl:
14779   case AMDGPU::BI__builtin_amdgcn_sicmp:
14780   case AMDGPU::BI__builtin_amdgcn_sicmpl: {
14781     llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
14782     llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
14783     llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
14784
14785     // FIXME-GFX10: How should 32 bit mask be handled?
14786     Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_icmp,
14787       { Builder.getInt64Ty(), Src0->getType() });
14788     return Builder.CreateCall(F, { Src0, Src1, Src2 });
14789   }
14790   case AMDGPU::BI__builtin_amdgcn_fcmp:
14791   case AMDGPU::BI__builtin_amdgcn_fcmpf: {
14792     llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
14793     llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
14794     llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
14795
14796     // FIXME-GFX10: How should 32 bit mask be handled?
14797     Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_fcmp,
14798       { Builder.getInt64Ty(), Src0->getType() });
14799     return Builder.CreateCall(F, { Src0, Src1, Src2 });
14800   }
14801   case AMDGPU::BI__builtin_amdgcn_class:
14802   case AMDGPU::BI__builtin_amdgcn_classf:
14803   case AMDGPU::BI__builtin_amdgcn_classh:
14804     return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_class);
14805   case AMDGPU::BI__builtin_amdgcn_fmed3f:
14806   case AMDGPU::BI__builtin_amdgcn_fmed3h:
14807     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_fmed3);
14808   case AMDGPU::BI__builtin_amdgcn_ds_append:
14809   case AMDGPU::BI__builtin_amdgcn_ds_consume: {
14810     Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
14811       Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
14812     Value *Src0 = EmitScalarExpr(E->getArg(0));
14813     Function *F = CGM.getIntrinsic(Intrin, { Src0->getType() });
14814     return Builder.CreateCall(F, { Src0, Builder.getFalse() });
14815   }
14816   case AMDGPU::BI__builtin_amdgcn_read_exec: {
14817     CallInst *CI = cast<CallInst>(
14818       EmitSpecialRegisterBuiltin(*this, E, Int64Ty, Int64Ty, NormalRead, "exec"));
14819     CI->setConvergent();
14820     return CI;
14821   }
14822   case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
14823   case AMDGPU::BI__builtin_amdgcn_read_exec_hi: {
14824     StringRef RegName = BuiltinID == AMDGPU::BI__builtin_amdgcn_read_exec_lo ?
14825       "exec_lo" : "exec_hi";
14826     CallInst *CI = cast<CallInst>(
14827       EmitSpecialRegisterBuiltin(*this, E, Int32Ty, Int32Ty, NormalRead, RegName));
14828     CI->setConvergent();
14829     return CI;
14830   }
14831   // amdgcn workitem
14832   case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
14833     return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_x, 0, 1024);
14834   case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
14835     return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_y, 0, 1024);
14836   case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
14837     return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_z, 0, 1024);
14838
14839   // amdgcn workgroup size
14840   case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
14841     return EmitAMDGPUWorkGroupSize(*this, 0);
14842   case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
14843     return EmitAMDGPUWorkGroupSize(*this, 1);
14844   case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
14845     return EmitAMDGPUWorkGroupSize(*this, 2);
14846
14847   // r600 intrinsics
14848   case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
14849   case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
14850     return emitUnaryBuiltin(*this, E, Intrinsic::r600_recipsqrt_ieee);
14851   case AMDGPU::BI__builtin_r600_read_tidig_x:
14852     return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_x, 0, 1024);
14853   case AMDGPU::BI__builtin_r600_read_tidig_y:
14854     return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_y, 0, 1024);
14855   case AMDGPU::BI__builtin_r600_read_tidig_z:
14856     return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_z, 0, 1024);
14857   case AMDGPU::BI__builtin_amdgcn_alignbit: {
14858     llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
14859     llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
14860     llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
14861     Function *F = CGM.getIntrinsic(Intrinsic::fshr, Src0->getType());
14862     return Builder.CreateCall(F, { Src0, Src1, Src2 });
14863   }
14864
14865   case AMDGPU::BI__builtin_amdgcn_fence: {
14866     if (ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(0)),
14867                                 EmitScalarExpr(E->getArg(1)), AO, SSID))
14868       return Builder.CreateFence(AO, SSID);
14869     LLVM_FALLTHROUGH;
14870   }
14871   case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
14872   case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
14873   case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
14874   case AMDGPU::BI__builtin_amdgcn_atomic_dec64: {
14875     unsigned BuiltinAtomicOp;
14876     llvm::Type *ResultType = ConvertType(E->getType());
14877
14878     switch (BuiltinID) {
14879     case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
14880     case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
14881       BuiltinAtomicOp = Intrinsic::amdgcn_atomic_inc;
14882       break;
14883     case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
14884     case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
14885       BuiltinAtomicOp = Intrinsic::amdgcn_atomic_dec;
14886       break;
14887     }
14888
14889     Value *Ptr = EmitScalarExpr(E->getArg(0));
14890     Value *Val = EmitScalarExpr(E->getArg(1));
14891
14892     llvm::Function *F =
14893         CGM.getIntrinsic(BuiltinAtomicOp, {ResultType, Ptr->getType()});
14894
14895     if (ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(2)),
14896                                 EmitScalarExpr(E->getArg(3)), AO, SSID)) {
14897
14898       // llvm.amdgcn.atomic.inc and llvm.amdgcn.atomic.dec expects ordering and
14899       // scope as unsigned values
14900       Value *MemOrder = Builder.getInt32(static_cast<int>(AO));
14901       Value *MemScope = Builder.getInt32(static_cast<int>(SSID));
14902
14903       QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
14904       bool Volatile =
14905           PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
14906       Value *IsVolatile = Builder.getInt1(static_cast<bool>(Volatile));
14907
14908       return Builder.CreateCall(F, {Ptr, Val, MemOrder, MemScope, IsVolatile});
14909     }
14910     LLVM_FALLTHROUGH;
14911   }
14912   default:
14913     return nullptr;
14914   }
14915 }
14916
14917 /// Handle a SystemZ function in which the final argument is a pointer
14918 /// to an int that receives the post-instruction CC value.  At the LLVM level
14919 /// this is represented as a function that returns a {result, cc} pair.
14920 static Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF,
14921                                          unsigned IntrinsicID,
14922                                          const CallExpr *E) {
14923   unsigned NumArgs = E->getNumArgs() - 1;
14924   SmallVector<Value *, 8> Args(NumArgs);
14925   for (unsigned I = 0; I < NumArgs; ++I)
14926     Args[I] = CGF.EmitScalarExpr(E->getArg(I));
14927   Address CCPtr = CGF.EmitPointerWithAlignment(E->getArg(NumArgs));
14928   Function *F = CGF.CGM.getIntrinsic(IntrinsicID);
14929   Value *Call = CGF.Builder.CreateCall(F, Args);
14930   Value *CC = CGF.Builder.CreateExtractValue(Call, 1);
14931   CGF.Builder.CreateStore(CC, CCPtr);
14932   return CGF.Builder.CreateExtractValue(Call, 0);
14933 }
14934
14935 Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID,
14936                                                const CallExpr *E) {
14937   switch (BuiltinID) {
14938   case SystemZ::BI__builtin_tbegin: {
14939     Value *TDB = EmitScalarExpr(E->getArg(0));
14940     Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
14941     Function *F = CGM.getIntrinsic(Intrinsic::s390_tbegin);
14942     return Builder.CreateCall(F, {TDB, Control});
14943   }
14944   case SystemZ::BI__builtin_tbegin_nofloat: {
14945     Value *TDB = EmitScalarExpr(E->getArg(0));
14946     Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
14947     Function *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat);
14948     return Builder.CreateCall(F, {TDB, Control});
14949   }
14950   case SystemZ::BI__builtin_tbeginc: {
14951     Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy);
14952     Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08);
14953     Function *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc);
14954     return Builder.CreateCall(F, {TDB, Control});
14955   }
14956   case SystemZ::BI__builtin_tabort: {
14957     Value *Data = EmitScalarExpr(E->getArg(0));
14958     Function *F = CGM.getIntrinsic(Intrinsic::s390_tabort);
14959     return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort"));
14960   }
14961   case SystemZ::BI__builtin_non_tx_store: {
14962     Value *Address = EmitScalarExpr(E->getArg(0));
14963     Value *Data = EmitScalarExpr(E->getArg(1));
14964     Function *F = CGM.getIntrinsic(Intrinsic::s390_ntstg);
14965     return Builder.CreateCall(F, {Data, Address});
14966   }
14967
14968   // Vector builtins.  Note that most vector builtins are mapped automatically
14969   // to target-specific LLVM intrinsics.  The ones handled specially here can
14970   // be represented via standard LLVM IR, which is preferable to enable common
14971   // LLVM optimizations.
14972
14973   case SystemZ::BI__builtin_s390_vpopctb:
14974   case SystemZ::BI__builtin_s390_vpopcth:
14975   case SystemZ::BI__builtin_s390_vpopctf:
14976   case SystemZ::BI__builtin_s390_vpopctg: {
14977     llvm::Type *ResultType = ConvertType(E->getType());
14978     Value *X = EmitScalarExpr(E->getArg(0));
14979     Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
14980     return Builder.CreateCall(F, X);
14981   }
14982
14983   case SystemZ::BI__builtin_s390_vclzb:
14984   case SystemZ::BI__builtin_s390_vclzh:
14985   case SystemZ::BI__builtin_s390_vclzf:
14986   case SystemZ::BI__builtin_s390_vclzg: {
14987     llvm::Type *ResultType = ConvertType(E->getType());
14988     Value *X = EmitScalarExpr(E->getArg(0));
14989     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
14990     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
14991     return Builder.CreateCall(F, {X, Undef});
14992   }
14993
14994   case SystemZ::BI__builtin_s390_vctzb:
14995   case SystemZ::BI__builtin_s390_vctzh:
14996   case SystemZ::BI__builtin_s390_vctzf:
14997   case SystemZ::BI__builtin_s390_vctzg: {
14998     llvm::Type *ResultType = ConvertType(E->getType());
14999     Value *X = EmitScalarExpr(E->getArg(0));
15000     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
15001     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
15002     return Builder.CreateCall(F, {X, Undef});
15003   }
15004
15005   case SystemZ::BI__builtin_s390_vfsqsb:
15006   case SystemZ::BI__builtin_s390_vfsqdb: {
15007     llvm::Type *ResultType = ConvertType(E->getType());
15008     Value *X = EmitScalarExpr(E->getArg(0));
15009     if (Builder.getIsFPConstrained()) {
15010       Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt, ResultType);
15011       return Builder.CreateConstrainedFPCall(F, { X });
15012     } else {
15013       Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType);
15014       return Builder.CreateCall(F, X);
15015     }
15016   }
15017   case SystemZ::BI__builtin_s390_vfmasb:
15018   case SystemZ::BI__builtin_s390_vfmadb: {
15019     llvm::Type *ResultType = ConvertType(E->getType());
15020     Value *X = EmitScalarExpr(E->getArg(0));
15021     Value *Y = EmitScalarExpr(E->getArg(1));
15022     Value *Z = EmitScalarExpr(E->getArg(2));
15023     if (Builder.getIsFPConstrained()) {
15024       Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
15025       return Builder.CreateConstrainedFPCall(F, {X, Y, Z});
15026     } else {
15027       Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
15028       return Builder.CreateCall(F, {X, Y, Z});
15029     }
15030   }
15031   case SystemZ::BI__builtin_s390_vfmssb:
15032   case SystemZ::BI__builtin_s390_vfmsdb: {
15033     llvm::Type *ResultType = ConvertType(E->getType());
15034     Value *X = EmitScalarExpr(E->getArg(0));
15035     Value *Y = EmitScalarExpr(E->getArg(1));
15036     Value *Z = EmitScalarExpr(E->getArg(2));
15037     if (Builder.getIsFPConstrained()) {
15038       Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
15039       return Builder.CreateConstrainedFPCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")});
15040     } else {
15041       Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
15042       return Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")});
15043     }
15044   }
15045   case SystemZ::BI__builtin_s390_vfnmasb:
15046   case SystemZ::BI__builtin_s390_vfnmadb: {
15047     llvm::Type *ResultType = ConvertType(E->getType());
15048     Value *X = EmitScalarExpr(E->getArg(0));
15049     Value *Y = EmitScalarExpr(E->getArg(1));
15050     Value *Z = EmitScalarExpr(E->getArg(2));
15051     if (Builder.getIsFPConstrained()) {
15052       Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
15053       return Builder.CreateFNeg(Builder.CreateConstrainedFPCall(F, {X, Y,  Z}), "neg");
15054     } else {
15055       Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
15056       return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}), "neg");
15057     }
15058   }
15059   case SystemZ::BI__builtin_s390_vfnmssb:
15060   case SystemZ::BI__builtin_s390_vfnmsdb: {
15061     llvm::Type *ResultType = ConvertType(E->getType());
15062     Value *X = EmitScalarExpr(E->getArg(0));
15063     Value *Y = EmitScalarExpr(E->getArg(1));
15064     Value *Z = EmitScalarExpr(E->getArg(2));
15065     if (Builder.getIsFPConstrained()) {
15066       Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
15067       Value *NegZ = Builder.CreateFNeg(Z, "sub");
15068       return Builder.CreateFNeg(Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
15069     } else {
15070       Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
15071       Value *NegZ = Builder.CreateFNeg(Z, "neg");
15072       return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, NegZ}));
15073     }
15074   }
15075   case SystemZ::BI__builtin_s390_vflpsb:
15076   case SystemZ::BI__builtin_s390_vflpdb: {
15077     llvm::Type *ResultType = ConvertType(E->getType());
15078     Value *X = EmitScalarExpr(E->getArg(0));
15079     Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
15080     return Builder.CreateCall(F, X);
15081   }
15082   case SystemZ::BI__builtin_s390_vflnsb:
15083   case SystemZ::BI__builtin_s390_vflndb: {
15084     llvm::Type *ResultType = ConvertType(E->getType());
15085     Value *X = EmitScalarExpr(E->getArg(0));
15086     Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
15087     return Builder.CreateFNeg(Builder.CreateCall(F, X), "neg");
15088   }
15089   case SystemZ::BI__builtin_s390_vfisb:
15090   case SystemZ::BI__builtin_s390_vfidb: {
15091     llvm::Type *ResultType = ConvertType(E->getType());
15092     Value *X = EmitScalarExpr(E->getArg(0));
15093     // Constant-fold the M4 and M5 mask arguments.
15094     llvm::APSInt M4, M5;
15095     bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext());
15096     bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext());
15097     assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?");
15098     (void)IsConstM4; (void)IsConstM5;
15099     // Check whether this instance can be represented via a LLVM standard
15100     // intrinsic.  We only support some combinations of M4 and M5.
15101     Intrinsic::ID ID = Intrinsic::not_intrinsic;
15102     Intrinsic::ID CI;
15103     switch (M4.getZExtValue()) {
15104     default: break;
15105     case 0:  // IEEE-inexact exception allowed
15106       switch (M5.getZExtValue()) {
15107       default: break;
15108       case 0: ID = Intrinsic::rint;
15109               CI = Intrinsic::experimental_constrained_rint; break;
15110       }
15111       break;
15112     case 4:  // IEEE-inexact exception suppressed
15113       switch (M5.getZExtValue()) {
15114       default: break;
15115       case 0: ID = Intrinsic::nearbyint;
15116               CI = Intrinsic::experimental_constrained_nearbyint; break;
15117       case 1: ID = Intrinsic::round;
15118               CI = Intrinsic::experimental_constrained_round; break;
15119       case 5: ID = Intrinsic::trunc;
15120               CI = Intrinsic::experimental_constrained_trunc; break;
15121       case 6: ID = Intrinsic::ceil;
15122               CI = Intrinsic::experimental_constrained_ceil; break;
15123       case 7: ID = Intrinsic::floor;
15124               CI = Intrinsic::experimental_constrained_floor; break;
15125       }
15126       break;
15127     }
15128     if (ID != Intrinsic::not_intrinsic) {
15129       if (Builder.getIsFPConstrained()) {
15130         Function *F = CGM.getIntrinsic(CI, ResultType);
15131         return Builder.CreateConstrainedFPCall(F, X);
15132       } else {
15133         Function *F = CGM.getIntrinsic(ID, ResultType);
15134         return Builder.CreateCall(F, X);
15135       }
15136     }
15137     switch (BuiltinID) { // FIXME: constrained version?
15138       case SystemZ::BI__builtin_s390_vfisb: ID = Intrinsic::s390_vfisb; break;
15139       case SystemZ::BI__builtin_s390_vfidb: ID = Intrinsic::s390_vfidb; break;
15140       default: llvm_unreachable("Unknown BuiltinID");
15141     }
15142     Function *F = CGM.getIntrinsic(ID);
15143     Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
15144     Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5);
15145     return Builder.CreateCall(F, {X, M4Value, M5Value});
15146   }
15147   case SystemZ::BI__builtin_s390_vfmaxsb:
15148   case SystemZ::BI__builtin_s390_vfmaxdb: {
15149     llvm::Type *ResultType = ConvertType(E->getType());
15150     Value *X = EmitScalarExpr(E->getArg(0));
15151     Value *Y = EmitScalarExpr(E->getArg(1));
15152     // Constant-fold the M4 mask argument.
15153     llvm::APSInt M4;
15154     bool IsConstM4 = E->getArg(2)->isIntegerConstantExpr(M4, getContext());
15155     assert(IsConstM4 && "Constant arg isn't actually constant?");
15156     (void)IsConstM4;
15157     // Check whether this instance can be represented via a LLVM standard
15158     // intrinsic.  We only support some values of M4.
15159     Intrinsic::ID ID = Intrinsic::not_intrinsic;
15160     Intrinsic::ID CI;
15161     switch (M4.getZExtValue()) {
15162     default: break;
15163     case 4: ID = Intrinsic::maxnum;
15164             CI = Intrinsic::experimental_constrained_maxnum; break;
15165     }
15166     if (ID != Intrinsic::not_intrinsic) {
15167       if (Builder.getIsFPConstrained()) {
15168         Function *F = CGM.getIntrinsic(CI, ResultType);
15169         return Builder.CreateConstrainedFPCall(F, {X, Y});
15170       } else {
15171         Function *F = CGM.getIntrinsic(ID, ResultType);
15172         return Builder.CreateCall(F, {X, Y});
15173       }
15174     }
15175     switch (BuiltinID) {
15176       case SystemZ::BI__builtin_s390_vfmaxsb: ID = Intrinsic::s390_vfmaxsb; break;
15177       case SystemZ::BI__builtin_s390_vfmaxdb: ID = Intrinsic::s390_vfmaxdb; break;
15178       default: llvm_unreachable("Unknown BuiltinID");
15179     }
15180     Function *F = CGM.getIntrinsic(ID);
15181     Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
15182     return Builder.CreateCall(F, {X, Y, M4Value});
15183   }
15184   case SystemZ::BI__builtin_s390_vfminsb:
15185   case SystemZ::BI__builtin_s390_vfmindb: {
15186     llvm::Type *ResultType = ConvertType(E->getType());
15187     Value *X = EmitScalarExpr(E->getArg(0));
15188     Value *Y = EmitScalarExpr(E->getArg(1));
15189     // Constant-fold the M4 mask argument.
15190     llvm::APSInt M4;
15191     bool IsConstM4 = E->getArg(2)->isIntegerConstantExpr(M4, getContext());
15192     assert(IsConstM4 && "Constant arg isn't actually constant?");
15193     (void)IsConstM4;
15194     // Check whether this instance can be represented via a LLVM standard
15195     // intrinsic.  We only support some values of M4.
15196     Intrinsic::ID ID = Intrinsic::not_intrinsic;
15197     Intrinsic::ID CI;
15198     switch (M4.getZExtValue()) {
15199     default: break;
15200     case 4: ID = Intrinsic::minnum;
15201             CI = Intrinsic::experimental_constrained_minnum; break;
15202     }
15203     if (ID != Intrinsic::not_intrinsic) {
15204       if (Builder.getIsFPConstrained()) {
15205         Function *F = CGM.getIntrinsic(CI, ResultType);
15206         return Builder.CreateConstrainedFPCall(F, {X, Y});
15207       } else {
15208         Function *F = CGM.getIntrinsic(ID, ResultType);
15209         return Builder.CreateCall(F, {X, Y});
15210       }
15211     }
15212     switch (BuiltinID) {
15213       case SystemZ::BI__builtin_s390_vfminsb: ID = Intrinsic::s390_vfminsb; break;
15214       case SystemZ::BI__builtin_s390_vfmindb: ID = Intrinsic::s390_vfmindb; break;
15215       default: llvm_unreachable("Unknown BuiltinID");
15216     }
15217     Function *F = CGM.getIntrinsic(ID);
15218     Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
15219     return Builder.CreateCall(F, {X, Y, M4Value});
15220   }
15221
15222   case SystemZ::BI__builtin_s390_vlbrh:
15223   case SystemZ::BI__builtin_s390_vlbrf:
15224   case SystemZ::BI__builtin_s390_vlbrg: {
15225     llvm::Type *ResultType = ConvertType(E->getType());
15226     Value *X = EmitScalarExpr(E->getArg(0));
15227     Function *F = CGM.getIntrinsic(Intrinsic::bswap, ResultType);
15228     return Builder.CreateCall(F, X);
15229   }
15230
15231   // Vector intrinsics that output the post-instruction CC value.
15232
15233 #define INTRINSIC_WITH_CC(NAME) \
15234     case SystemZ::BI__builtin_##NAME: \
15235       return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
15236
15237   INTRINSIC_WITH_CC(s390_vpkshs);
15238   INTRINSIC_WITH_CC(s390_vpksfs);
15239   INTRINSIC_WITH_CC(s390_vpksgs);
15240
15241   INTRINSIC_WITH_CC(s390_vpklshs);
15242   INTRINSIC_WITH_CC(s390_vpklsfs);
15243   INTRINSIC_WITH_CC(s390_vpklsgs);
15244
15245   INTRINSIC_WITH_CC(s390_vceqbs);
15246   INTRINSIC_WITH_CC(s390_vceqhs);
15247   INTRINSIC_WITH_CC(s390_vceqfs);
15248   INTRINSIC_WITH_CC(s390_vceqgs);
15249
15250   INTRINSIC_WITH_CC(s390_vchbs);
15251   INTRINSIC_WITH_CC(s390_vchhs);
15252   INTRINSIC_WITH_CC(s390_vchfs);
15253   INTRINSIC_WITH_CC(s390_vchgs);
15254
15255   INTRINSIC_WITH_CC(s390_vchlbs);
15256   INTRINSIC_WITH_CC(s390_vchlhs);
15257   INTRINSIC_WITH_CC(s390_vchlfs);
15258   INTRINSIC_WITH_CC(s390_vchlgs);
15259
15260   INTRINSIC_WITH_CC(s390_vfaebs);
15261   INTRINSIC_WITH_CC(s390_vfaehs);
15262   INTRINSIC_WITH_CC(s390_vfaefs);
15263
15264   INTRINSIC_WITH_CC(s390_vfaezbs);
15265   INTRINSIC_WITH_CC(s390_vfaezhs);
15266   INTRINSIC_WITH_CC(s390_vfaezfs);
15267
15268   INTRINSIC_WITH_CC(s390_vfeebs);
15269   INTRINSIC_WITH_CC(s390_vfeehs);
15270   INTRINSIC_WITH_CC(s390_vfeefs);
15271
15272   INTRINSIC_WITH_CC(s390_vfeezbs);
15273   INTRINSIC_WITH_CC(s390_vfeezhs);
15274   INTRINSIC_WITH_CC(s390_vfeezfs);
15275
15276   INTRINSIC_WITH_CC(s390_vfenebs);
15277   INTRINSIC_WITH_CC(s390_vfenehs);
15278   INTRINSIC_WITH_CC(s390_vfenefs);
15279
15280   INTRINSIC_WITH_CC(s390_vfenezbs);
15281   INTRINSIC_WITH_CC(s390_vfenezhs);
15282   INTRINSIC_WITH_CC(s390_vfenezfs);
15283
15284   INTRINSIC_WITH_CC(s390_vistrbs);
15285   INTRINSIC_WITH_CC(s390_vistrhs);
15286   INTRINSIC_WITH_CC(s390_vistrfs);
15287
15288   INTRINSIC_WITH_CC(s390_vstrcbs);
15289   INTRINSIC_WITH_CC(s390_vstrchs);
15290   INTRINSIC_WITH_CC(s390_vstrcfs);
15291
15292   INTRINSIC_WITH_CC(s390_vstrczbs);
15293   INTRINSIC_WITH_CC(s390_vstrczhs);
15294   INTRINSIC_WITH_CC(s390_vstrczfs);
15295
15296   INTRINSIC_WITH_CC(s390_vfcesbs);
15297   INTRINSIC_WITH_CC(s390_vfcedbs);
15298   INTRINSIC_WITH_CC(s390_vfchsbs);
15299   INTRINSIC_WITH_CC(s390_vfchdbs);
15300   INTRINSIC_WITH_CC(s390_vfchesbs);
15301   INTRINSIC_WITH_CC(s390_vfchedbs);
15302
15303   INTRINSIC_WITH_CC(s390_vftcisb);
15304   INTRINSIC_WITH_CC(s390_vftcidb);
15305
15306   INTRINSIC_WITH_CC(s390_vstrsb);
15307   INTRINSIC_WITH_CC(s390_vstrsh);
15308   INTRINSIC_WITH_CC(s390_vstrsf);
15309
15310   INTRINSIC_WITH_CC(s390_vstrszb);
15311   INTRINSIC_WITH_CC(s390_vstrszh);
15312   INTRINSIC_WITH_CC(s390_vstrszf);
15313
15314 #undef INTRINSIC_WITH_CC
15315
15316   default:
15317     return nullptr;
15318   }
15319 }
15320
15321 namespace {
15322 // Helper classes for mapping MMA builtins to particular LLVM intrinsic variant.
15323 struct NVPTXMmaLdstInfo {
15324   unsigned NumResults;  // Number of elements to load/store
15325   // Intrinsic IDs for row/col variants. 0 if particular layout is unsupported.
15326   unsigned IID_col;
15327   unsigned IID_row;
15328 };
15329
15330 #define MMA_INTR(geom_op_type, layout) \
15331   Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
15332 #define MMA_LDST(n, geom_op_type)                                              \
15333   { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
15334
15335 static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(unsigned BuiltinID) {
15336   switch (BuiltinID) {
15337   // FP MMA loads
15338   case NVPTX::BI__hmma_m16n16k16_ld_a:
15339     return MMA_LDST(8, m16n16k16_load_a_f16);
15340   case NVPTX::BI__hmma_m16n16k16_ld_b:
15341     return MMA_LDST(8, m16n16k16_load_b_f16);
15342   case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
15343     return MMA_LDST(4, m16n16k16_load_c_f16);
15344   case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
15345     return MMA_LDST(8, m16n16k16_load_c_f32);
15346   case NVPTX::BI__hmma_m32n8k16_ld_a:
15347     return MMA_LDST(8, m32n8k16_load_a_f16);
15348   case NVPTX::BI__hmma_m32n8k16_ld_b:
15349     return MMA_LDST(8, m32n8k16_load_b_f16);
15350   case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
15351     return MMA_LDST(4, m32n8k16_load_c_f16);
15352   case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
15353     return MMA_LDST(8, m32n8k16_load_c_f32);
15354   case NVPTX::BI__hmma_m8n32k16_ld_a:
15355     return MMA_LDST(8, m8n32k16_load_a_f16);
15356   case NVPTX::BI__hmma_m8n32k16_ld_b:
15357     return MMA_LDST(8, m8n32k16_load_b_f16);
15358   case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
15359     return MMA_LDST(4, m8n32k16_load_c_f16);
15360   case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
15361     return MMA_LDST(8, m8n32k16_load_c_f32);
15362
15363   // Integer MMA loads
15364   case NVPTX::BI__imma_m16n16k16_ld_a_s8:
15365     return MMA_LDST(2, m16n16k16_load_a_s8);
15366   case NVPTX::BI__imma_m16n16k16_ld_a_u8:
15367     return MMA_LDST(2, m16n16k16_load_a_u8);
15368   case NVPTX::BI__imma_m16n16k16_ld_b_s8:
15369     return MMA_LDST(2, m16n16k16_load_b_s8);
15370   case NVPTX::BI__imma_m16n16k16_ld_b_u8:
15371     return MMA_LDST(2, m16n16k16_load_b_u8);
15372   case NVPTX::BI__imma_m16n16k16_ld_c:
15373     return MMA_LDST(8, m16n16k16_load_c_s32);
15374   case NVPTX::BI__imma_m32n8k16_ld_a_s8:
15375     return MMA_LDST(4, m32n8k16_load_a_s8);
15376   case NVPTX::BI__imma_m32n8k16_ld_a_u8:
15377     return MMA_LDST(4, m32n8k16_load_a_u8);
15378   case NVPTX::BI__imma_m32n8k16_ld_b_s8:
15379     return MMA_LDST(1, m32n8k16_load_b_s8);
15380   case NVPTX::BI__imma_m32n8k16_ld_b_u8:
15381     return MMA_LDST(1, m32n8k16_load_b_u8);
15382   case NVPTX::BI__imma_m32n8k16_ld_c:
15383     return MMA_LDST(8, m32n8k16_load_c_s32);
15384   case NVPTX::BI__imma_m8n32k16_ld_a_s8:
15385     return MMA_LDST(1, m8n32k16_load_a_s8);
15386   case NVPTX::BI__imma_m8n32k16_ld_a_u8:
15387     return MMA_LDST(1, m8n32k16_load_a_u8);
15388   case NVPTX::BI__imma_m8n32k16_ld_b_s8:
15389     return MMA_LDST(4, m8n32k16_load_b_s8);
15390   case NVPTX::BI__imma_m8n32k16_ld_b_u8:
15391     return MMA_LDST(4, m8n32k16_load_b_u8);
15392   case NVPTX::BI__imma_m8n32k16_ld_c:
15393     return MMA_LDST(8, m8n32k16_load_c_s32);
15394
15395   // Sub-integer MMA loads.
15396   // Only row/col layout is supported by A/B fragments.
15397   case NVPTX::BI__imma_m8n8k32_ld_a_s4:
15398     return {1, 0, MMA_INTR(m8n8k32_load_a_s4, row)};
15399   case NVPTX::BI__imma_m8n8k32_ld_a_u4:
15400     return {1, 0, MMA_INTR(m8n8k32_load_a_u4, row)};
15401   case NVPTX::BI__imma_m8n8k32_ld_b_s4:
15402     return {1, MMA_INTR(m8n8k32_load_b_s4, col), 0};
15403   case NVPTX::BI__imma_m8n8k32_ld_b_u4:
15404     return {1, MMA_INTR(m8n8k32_load_b_u4, col), 0};
15405   case NVPTX::BI__imma_m8n8k32_ld_c:
15406     return MMA_LDST(2, m8n8k32_load_c_s32);
15407   case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
15408     return {1, 0, MMA_INTR(m8n8k128_load_a_b1, row)};
15409   case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
15410     return {1, MMA_INTR(m8n8k128_load_b_b1, col), 0};
15411   case NVPTX::BI__bmma_m8n8k128_ld_c:
15412     return MMA_LDST(2, m8n8k128_load_c_s32);
15413
15414   // NOTE: We need to follow inconsitent naming scheme used by NVCC.  Unlike
15415   // PTX and LLVM IR where stores always use fragment D, NVCC builtins always
15416   // use fragment C for both loads and stores.
15417   // FP MMA stores.
15418   case NVPTX::BI__hmma_m16n16k16_st_c_f16:
15419     return MMA_LDST(4, m16n16k16_store_d_f16);
15420   case NVPTX::BI__hmma_m16n16k16_st_c_f32:
15421     return MMA_LDST(8, m16n16k16_store_d_f32);
15422   case NVPTX::BI__hmma_m32n8k16_st_c_f16:
15423     return MMA_LDST(4, m32n8k16_store_d_f16);
15424   case NVPTX::BI__hmma_m32n8k16_st_c_f32:
15425     return MMA_LDST(8, m32n8k16_store_d_f32);
15426   case NVPTX::BI__hmma_m8n32k16_st_c_f16:
15427     return MMA_LDST(4, m8n32k16_store_d_f16);
15428   case NVPTX::BI__hmma_m8n32k16_st_c_f32:
15429     return MMA_LDST(8, m8n32k16_store_d_f32);
15430
15431   // Integer and sub-integer MMA stores.
15432   // Another naming quirk. Unlike other MMA builtins that use PTX types in the
15433   // name, integer loads/stores use LLVM's i32.
15434   case NVPTX::BI__imma_m16n16k16_st_c_i32:
15435     return MMA_LDST(8, m16n16k16_store_d_s32);
15436   case NVPTX::BI__imma_m32n8k16_st_c_i32:
15437     return MMA_LDST(8, m32n8k16_store_d_s32);
15438   case NVPTX::BI__imma_m8n32k16_st_c_i32:
15439     return MMA_LDST(8, m8n32k16_store_d_s32);
15440   case NVPTX::BI__imma_m8n8k32_st_c_i32:
15441     return MMA_LDST(2, m8n8k32_store_d_s32);
15442   case NVPTX::BI__bmma_m8n8k128_st_c_i32:
15443     return MMA_LDST(2, m8n8k128_store_d_s32);
15444
15445   default:
15446     llvm_unreachable("Unknown MMA builtin");
15447   }
15448 }
15449 #undef MMA_LDST
15450 #undef MMA_INTR
15451
15452
15453 struct NVPTXMmaInfo {
15454   unsigned NumEltsA;
15455   unsigned NumEltsB;
15456   unsigned NumEltsC;
15457   unsigned NumEltsD;
15458   std::array<unsigned, 8> Variants;
15459
15460   unsigned getMMAIntrinsic(int Layout, bool Satf) {
15461     unsigned Index = Layout * 2 + Satf;
15462     if (Index >= Variants.size())
15463       return 0;
15464     return Variants[Index];
15465   }
15466 };
15467
15468   // Returns an intrinsic that matches Layout and Satf for valid combinations of
15469   // Layout and Satf, 0 otherwise.
15470 static NVPTXMmaInfo getNVPTXMmaInfo(unsigned BuiltinID) {
15471   // clang-format off
15472 #define MMA_VARIANTS(geom, type) {{                                 \
15473       Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type,             \
15474       Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
15475       Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type,             \
15476       Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
15477       Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type,             \
15478       Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
15479       Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type,             \
15480       Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite  \
15481     }}
15482 // Sub-integer MMA only supports row.col layout.
15483 #define MMA_VARIANTS_I4(geom, type) {{ \
15484       0, \
15485       0, \
15486       Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type,             \
15487       Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
15488       0, \
15489       0, \
15490       0, \
15491       0  \
15492     }}
15493 // b1 MMA does not support .satfinite.
15494 #define MMA_VARIANTS_B1(geom, type) {{ \
15495       0, \
15496       0, \
15497       Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type,             \
15498       0, \
15499       0, \
15500       0, \
15501       0, \
15502       0  \
15503     }}
15504     // clang-format on
15505     switch (BuiltinID) {
15506     // FP MMA
15507     // Note that 'type' argument of MMA_VARIANT uses D_C notation, while
15508     // NumEltsN of return value are ordered as A,B,C,D.
15509     case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
15510       return {8, 8, 4, 4, MMA_VARIANTS(m16n16k16, f16_f16)};
15511     case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
15512       return {8, 8, 4, 8, MMA_VARIANTS(m16n16k16, f32_f16)};
15513     case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
15514       return {8, 8, 8, 4, MMA_VARIANTS(m16n16k16, f16_f32)};
15515     case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
15516       return {8, 8, 8, 8, MMA_VARIANTS(m16n16k16, f32_f32)};
15517     case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
15518       return {8, 8, 4, 4, MMA_VARIANTS(m32n8k16, f16_f16)};
15519     case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
15520       return {8, 8, 4, 8, MMA_VARIANTS(m32n8k16, f32_f16)};
15521     case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
15522       return {8, 8, 8, 4, MMA_VARIANTS(m32n8k16, f16_f32)};
15523     case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
15524       return {8, 8, 8, 8, MMA_VARIANTS(m32n8k16, f32_f32)};
15525     case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
15526       return {8, 8, 4, 4, MMA_VARIANTS(m8n32k16, f16_f16)};
15527     case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
15528       return {8, 8, 4, 8, MMA_VARIANTS(m8n32k16, f32_f16)};
15529     case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
15530       return {8, 8, 8, 4, MMA_VARIANTS(m8n32k16, f16_f32)};
15531     case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
15532       return {8, 8, 8, 8, MMA_VARIANTS(m8n32k16, f32_f32)};
15533
15534     // Integer MMA
15535     case NVPTX::BI__imma_m16n16k16_mma_s8:
15536       return {2, 2, 8, 8, MMA_VARIANTS(m16n16k16, s8)};
15537     case NVPTX::BI__imma_m16n16k16_mma_u8:
15538       return {2, 2, 8, 8, MMA_VARIANTS(m16n16k16, u8)};
15539     case NVPTX::BI__imma_m32n8k16_mma_s8:
15540       return {4, 1, 8, 8, MMA_VARIANTS(m32n8k16, s8)};
15541     case NVPTX::BI__imma_m32n8k16_mma_u8:
15542       return {4, 1, 8, 8, MMA_VARIANTS(m32n8k16, u8)};
15543     case NVPTX::BI__imma_m8n32k16_mma_s8:
15544       return {1, 4, 8, 8, MMA_VARIANTS(m8n32k16, s8)};
15545     case NVPTX::BI__imma_m8n32k16_mma_u8:
15546       return {1, 4, 8, 8, MMA_VARIANTS(m8n32k16, u8)};
15547
15548     // Sub-integer MMA
15549     case NVPTX::BI__imma_m8n8k32_mma_s4:
15550       return {1, 1, 2, 2, MMA_VARIANTS_I4(m8n8k32, s4)};
15551     case NVPTX::BI__imma_m8n8k32_mma_u4:
15552       return {1, 1, 2, 2, MMA_VARIANTS_I4(m8n8k32, u4)};
15553     case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
15554       return {1, 1, 2, 2, MMA_VARIANTS_B1(m8n8k128, b1)};
15555     default:
15556       llvm_unreachable("Unexpected builtin ID.");
15557     }
15558 #undef MMA_VARIANTS
15559 #undef MMA_VARIANTS_I4
15560 #undef MMA_VARIANTS_B1
15561 }
15562
15563 } // namespace
15564
15565 Value *
15566 CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E) {
15567   auto MakeLdg = [&](unsigned IntrinsicID) {
15568     Value *Ptr = EmitScalarExpr(E->getArg(0));
15569     clang::CharUnits Align =
15570         CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
15571     return Builder.CreateCall(
15572         CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
15573                                        Ptr->getType()}),
15574         {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
15575   };
15576   auto MakeScopedAtomic = [&](unsigned IntrinsicID) {
15577     Value *Ptr = EmitScalarExpr(E->getArg(0));
15578     return Builder.CreateCall(
15579         CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
15580                                        Ptr->getType()}),
15581         {Ptr, EmitScalarExpr(E->getArg(1))});
15582   };
15583   switch (BuiltinID) {
15584   case NVPTX::BI__nvvm_atom_add_gen_i:
15585   case NVPTX::BI__nvvm_atom_add_gen_l:
15586   case NVPTX::BI__nvvm_atom_add_gen_ll:
15587     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E);
15588
15589   case NVPTX::BI__nvvm_atom_sub_gen_i:
15590   case NVPTX::BI__nvvm_atom_sub_gen_l:
15591   case NVPTX::BI__nvvm_atom_sub_gen_ll:
15592     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E);
15593
15594   case NVPTX::BI__nvvm_atom_and_gen_i:
15595   case NVPTX::BI__nvvm_atom_and_gen_l:
15596   case NVPTX::BI__nvvm_atom_and_gen_ll:
15597     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E);
15598
15599   case NVPTX::BI__nvvm_atom_or_gen_i:
15600   case NVPTX::BI__nvvm_atom_or_gen_l:
15601   case NVPTX::BI__nvvm_atom_or_gen_ll:
15602     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E);
15603
15604   case NVPTX::BI__nvvm_atom_xor_gen_i:
15605   case NVPTX::BI__nvvm_atom_xor_gen_l:
15606   case NVPTX::BI__nvvm_atom_xor_gen_ll:
15607     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E);
15608
15609   case NVPTX::BI__nvvm_atom_xchg_gen_i:
15610   case NVPTX::BI__nvvm_atom_xchg_gen_l:
15611   case NVPTX::BI__nvvm_atom_xchg_gen_ll:
15612     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E);
15613
15614   case NVPTX::BI__nvvm_atom_max_gen_i:
15615   case NVPTX::BI__nvvm_atom_max_gen_l:
15616   case NVPTX::BI__nvvm_atom_max_gen_ll:
15617     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E);
15618
15619   case NVPTX::BI__nvvm_atom_max_gen_ui:
15620   case NVPTX::BI__nvvm_atom_max_gen_ul:
15621   case NVPTX::BI__nvvm_atom_max_gen_ull:
15622     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E);
15623
15624   case NVPTX::BI__nvvm_atom_min_gen_i:
15625   case NVPTX::BI__nvvm_atom_min_gen_l:
15626   case NVPTX::BI__nvvm_atom_min_gen_ll:
15627     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E);
15628
15629   case NVPTX::BI__nvvm_atom_min_gen_ui:
15630   case NVPTX::BI__nvvm_atom_min_gen_ul:
15631   case NVPTX::BI__nvvm_atom_min_gen_ull:
15632     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E);
15633
15634   case NVPTX::BI__nvvm_atom_cas_gen_i:
15635   case NVPTX::BI__nvvm_atom_cas_gen_l:
15636   case NVPTX::BI__nvvm_atom_cas_gen_ll:
15637     // __nvvm_atom_cas_gen_* should return the old value rather than the
15638     // success flag.
15639     return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false);
15640
15641   case NVPTX::BI__nvvm_atom_add_gen_f:
15642   case NVPTX::BI__nvvm_atom_add_gen_d: {
15643     Value *Ptr = EmitScalarExpr(E->getArg(0));
15644     Value *Val = EmitScalarExpr(E->getArg(1));
15645     return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::FAdd, Ptr, Val,
15646                                    AtomicOrdering::SequentiallyConsistent);
15647   }
15648
15649   case NVPTX::BI__nvvm_atom_inc_gen_ui: {
15650     Value *Ptr = EmitScalarExpr(E->getArg(0));
15651     Value *Val = EmitScalarExpr(E->getArg(1));
15652     Function *FnALI32 =
15653         CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType());
15654     return Builder.CreateCall(FnALI32, {Ptr, Val});
15655   }
15656
15657   case NVPTX::BI__nvvm_atom_dec_gen_ui: {
15658     Value *Ptr = EmitScalarExpr(E->getArg(0));
15659     Value *Val = EmitScalarExpr(E->getArg(1));
15660     Function *FnALD32 =
15661         CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType());
15662     return Builder.CreateCall(FnALD32, {Ptr, Val});
15663   }
15664
15665   case NVPTX::BI__nvvm_ldg_c:
15666   case NVPTX::BI__nvvm_ldg_c2:
15667   case NVPTX::BI__nvvm_ldg_c4:
15668   case NVPTX::BI__nvvm_ldg_s:
15669   case NVPTX::BI__nvvm_ldg_s2:
15670   case NVPTX::BI__nvvm_ldg_s4:
15671   case NVPTX::BI__nvvm_ldg_i:
15672   case NVPTX::BI__nvvm_ldg_i2:
15673   case NVPTX::BI__nvvm_ldg_i4:
15674   case NVPTX::BI__nvvm_ldg_l:
15675   case NVPTX::BI__nvvm_ldg_ll:
15676   case NVPTX::BI__nvvm_ldg_ll2:
15677   case NVPTX::BI__nvvm_ldg_uc:
15678   case NVPTX::BI__nvvm_ldg_uc2:
15679   case NVPTX::BI__nvvm_ldg_uc4:
15680   case NVPTX::BI__nvvm_ldg_us:
15681   case NVPTX::BI__nvvm_ldg_us2:
15682   case NVPTX::BI__nvvm_ldg_us4:
15683   case NVPTX::BI__nvvm_ldg_ui:
15684   case NVPTX::BI__nvvm_ldg_ui2:
15685   case NVPTX::BI__nvvm_ldg_ui4:
15686   case NVPTX::BI__nvvm_ldg_ul:
15687   case NVPTX::BI__nvvm_ldg_ull:
15688   case NVPTX::BI__nvvm_ldg_ull2:
15689     // PTX Interoperability section 2.2: "For a vector with an even number of
15690     // elements, its alignment is set to number of elements times the alignment
15691     // of its member: n*alignof(t)."
15692     return MakeLdg(Intrinsic::nvvm_ldg_global_i);
15693   case NVPTX::BI__nvvm_ldg_f:
15694   case NVPTX::BI__nvvm_ldg_f2:
15695   case NVPTX::BI__nvvm_ldg_f4:
15696   case NVPTX::BI__nvvm_ldg_d:
15697   case NVPTX::BI__nvvm_ldg_d2:
15698     return MakeLdg(Intrinsic::nvvm_ldg_global_f);
15699
15700   case NVPTX::BI__nvvm_atom_cta_add_gen_i:
15701   case NVPTX::BI__nvvm_atom_cta_add_gen_l:
15702   case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
15703     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta);
15704   case NVPTX::BI__nvvm_atom_sys_add_gen_i:
15705   case NVPTX::BI__nvvm_atom_sys_add_gen_l:
15706   case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
15707     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys);
15708   case NVPTX::BI__nvvm_atom_cta_add_gen_f:
15709   case NVPTX::BI__nvvm_atom_cta_add_gen_d:
15710     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta);
15711   case NVPTX::BI__nvvm_atom_sys_add_gen_f:
15712   case NVPTX::BI__nvvm_atom_sys_add_gen_d:
15713     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys);
15714   case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
15715   case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
15716   case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
15717     return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta);
15718   case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
15719   case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
15720   case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
15721     return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys);
15722   case NVPTX::BI__nvvm_atom_cta_max_gen_i:
15723   case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
15724   case NVPTX::BI__nvvm_atom_cta_max_gen_l:
15725   case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
15726   case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
15727   case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
15728     return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta);
15729   case NVPTX::BI__nvvm_atom_sys_max_gen_i:
15730   case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
15731   case NVPTX::BI__nvvm_atom_sys_max_gen_l:
15732   case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
15733   case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
15734   case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
15735     return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys);
15736   case NVPTX::BI__nvvm_atom_cta_min_gen_i:
15737   case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
15738   case NVPTX::BI__nvvm_atom_cta_min_gen_l:
15739   case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
15740   case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
15741   case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
15742     return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta);
15743   case NVPTX::BI__nvvm_atom_sys_min_gen_i:
15744   case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
15745   case NVPTX::BI__nvvm_atom_sys_min_gen_l:
15746   case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
15747   case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
15748   case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
15749     return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys);
15750   case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
15751     return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta);
15752   case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
15753     return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta);
15754   case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
15755     return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys);
15756   case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
15757     return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys);
15758   case NVPTX::BI__nvvm_atom_cta_and_gen_i:
15759   case NVPTX::BI__nvvm_atom_cta_and_gen_l:
15760   case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
15761     return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta);
15762   case NVPTX::BI__nvvm_atom_sys_and_gen_i:
15763   case NVPTX::BI__nvvm_atom_sys_and_gen_l:
15764   case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
15765     return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys);
15766   case NVPTX::BI__nvvm_atom_cta_or_gen_i:
15767   case NVPTX::BI__nvvm_atom_cta_or_gen_l:
15768   case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
15769     return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta);
15770   case NVPTX::BI__nvvm_atom_sys_or_gen_i:
15771   case NVPTX::BI__nvvm_atom_sys_or_gen_l:
15772   case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
15773     return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys);
15774   case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
15775   case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
15776   case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
15777     return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta);
15778   case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
15779   case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
15780   case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
15781     return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys);
15782   case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
15783   case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
15784   case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
15785     Value *Ptr = EmitScalarExpr(E->getArg(0));
15786     return Builder.CreateCall(
15787         CGM.getIntrinsic(
15788             Intrinsic::nvvm_atomic_cas_gen_i_cta,
15789             {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
15790         {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
15791   }
15792   case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
15793   case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
15794   case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
15795     Value *Ptr = EmitScalarExpr(E->getArg(0));
15796     return Builder.CreateCall(
15797         CGM.getIntrinsic(
15798             Intrinsic::nvvm_atomic_cas_gen_i_sys,
15799             {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
15800         {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
15801   }
15802   case NVPTX::BI__nvvm_match_all_sync_i32p:
15803   case NVPTX::BI__nvvm_match_all_sync_i64p: {
15804     Value *Mask = EmitScalarExpr(E->getArg(0));
15805     Value *Val = EmitScalarExpr(E->getArg(1));
15806     Address PredOutPtr = EmitPointerWithAlignment(E->getArg(2));
15807     Value *ResultPair = Builder.CreateCall(
15808         CGM.getIntrinsic(BuiltinID == NVPTX::BI__nvvm_match_all_sync_i32p
15809                              ? Intrinsic::nvvm_match_all_sync_i32p
15810                              : Intrinsic::nvvm_match_all_sync_i64p),
15811         {Mask, Val});
15812     Value *Pred = Builder.CreateZExt(Builder.CreateExtractValue(ResultPair, 1),
15813                                      PredOutPtr.getElementType());
15814     Builder.CreateStore(Pred, PredOutPtr);
15815     return Builder.CreateExtractValue(ResultPair, 0);
15816   }
15817
15818   // FP MMA loads
15819   case NVPTX::BI__hmma_m16n16k16_ld_a:
15820   case NVPTX::BI__hmma_m16n16k16_ld_b:
15821   case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
15822   case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
15823   case NVPTX::BI__hmma_m32n8k16_ld_a:
15824   case NVPTX::BI__hmma_m32n8k16_ld_b:
15825   case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
15826   case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
15827   case NVPTX::BI__hmma_m8n32k16_ld_a:
15828   case NVPTX::BI__hmma_m8n32k16_ld_b:
15829   case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
15830   case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
15831   // Integer MMA loads.
15832   case NVPTX::BI__imma_m16n16k16_ld_a_s8:
15833   case NVPTX::BI__imma_m16n16k16_ld_a_u8:
15834   case NVPTX::BI__imma_m16n16k16_ld_b_s8:
15835   case NVPTX::BI__imma_m16n16k16_ld_b_u8:
15836   case NVPTX::BI__imma_m16n16k16_ld_c:
15837   case NVPTX::BI__imma_m32n8k16_ld_a_s8:
15838   case NVPTX::BI__imma_m32n8k16_ld_a_u8:
15839   case NVPTX::BI__imma_m32n8k16_ld_b_s8:
15840   case NVPTX::BI__imma_m32n8k16_ld_b_u8:
15841   case NVPTX::BI__imma_m32n8k16_ld_c:
15842   case NVPTX::BI__imma_m8n32k16_ld_a_s8:
15843   case NVPTX::BI__imma_m8n32k16_ld_a_u8:
15844   case NVPTX::BI__imma_m8n32k16_ld_b_s8:
15845   case NVPTX::BI__imma_m8n32k16_ld_b_u8:
15846   case NVPTX::BI__imma_m8n32k16_ld_c:
15847   // Sub-integer MMA loads.
15848   case NVPTX::BI__imma_m8n8k32_ld_a_s4:
15849   case NVPTX::BI__imma_m8n8k32_ld_a_u4:
15850   case NVPTX::BI__imma_m8n8k32_ld_b_s4:
15851   case NVPTX::BI__imma_m8n8k32_ld_b_u4:
15852   case NVPTX::BI__imma_m8n8k32_ld_c:
15853   case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
15854   case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
15855   case NVPTX::BI__bmma_m8n8k128_ld_c:
15856   {
15857     Address Dst = EmitPointerWithAlignment(E->getArg(0));
15858     Value *Src = EmitScalarExpr(E->getArg(1));
15859     Value *Ldm = EmitScalarExpr(E->getArg(2));
15860     llvm::APSInt isColMajorArg;
15861     if (!E->getArg(3)->isIntegerConstantExpr(isColMajorArg, getContext()))
15862       return nullptr;
15863     bool isColMajor = isColMajorArg.getSExtValue();
15864     NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
15865     unsigned IID = isColMajor ? II.IID_col : II.IID_row;
15866     if (IID == 0)
15867       return nullptr;
15868
15869     Value *Result =
15870         Builder.CreateCall(CGM.getIntrinsic(IID, Src->getType()), {Src, Ldm});
15871
15872     // Save returned values.
15873     assert(II.NumResults);
15874     if (II.NumResults == 1) {
15875       Builder.CreateAlignedStore(Result, Dst.getPointer(),
15876                                  CharUnits::fromQuantity(4));
15877     } else {
15878       for (unsigned i = 0; i < II.NumResults; ++i) {
15879         Builder.CreateAlignedStore(
15880             Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
15881                                   Dst.getElementType()),
15882             Builder.CreateGEP(Dst.getPointer(),
15883                               llvm::ConstantInt::get(IntTy, i)),
15884             CharUnits::fromQuantity(4));
15885       }
15886     }
15887     return Result;
15888   }
15889
15890   case NVPTX::BI__hmma_m16n16k16_st_c_f16:
15891   case NVPTX::BI__hmma_m16n16k16_st_c_f32:
15892   case NVPTX::BI__hmma_m32n8k16_st_c_f16:
15893   case NVPTX::BI__hmma_m32n8k16_st_c_f32:
15894   case NVPTX::BI__hmma_m8n32k16_st_c_f16:
15895   case NVPTX::BI__hmma_m8n32k16_st_c_f32:
15896   case NVPTX::BI__imma_m16n16k16_st_c_i32:
15897   case NVPTX::BI__imma_m32n8k16_st_c_i32:
15898   case NVPTX::BI__imma_m8n32k16_st_c_i32:
15899   case NVPTX::BI__imma_m8n8k32_st_c_i32:
15900   case NVPTX::BI__bmma_m8n8k128_st_c_i32: {
15901     Value *Dst = EmitScalarExpr(E->getArg(0));
15902     Address Src = EmitPointerWithAlignment(E->getArg(1));
15903     Value *Ldm = EmitScalarExpr(E->getArg(2));
15904     llvm::APSInt isColMajorArg;
15905     if (!E->getArg(3)->isIntegerConstantExpr(isColMajorArg, getContext()))
15906       return nullptr;
15907     bool isColMajor = isColMajorArg.getSExtValue();
15908     NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
15909     unsigned IID = isColMajor ? II.IID_col : II.IID_row;
15910     if (IID == 0)
15911       return nullptr;
15912     Function *Intrinsic =
15913         CGM.getIntrinsic(IID, Dst->getType());
15914     llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
15915     SmallVector<Value *, 10> Values = {Dst};
15916     for (unsigned i = 0; i < II.NumResults; ++i) {
15917       Value *V = Builder.CreateAlignedLoad(
15918           Builder.CreateGEP(Src.getPointer(), llvm::ConstantInt::get(IntTy, i)),
15919           CharUnits::fromQuantity(4));
15920       Values.push_back(Builder.CreateBitCast(V, ParamType));
15921     }
15922     Values.push_back(Ldm);
15923     Value *Result = Builder.CreateCall(Intrinsic, Values);
15924     return Result;
15925   }
15926
15927   // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
15928   // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
15929   case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
15930   case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
15931   case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
15932   case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
15933   case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
15934   case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
15935   case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
15936   case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
15937   case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
15938   case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
15939   case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
15940   case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
15941   case NVPTX::BI__imma_m16n16k16_mma_s8:
15942   case NVPTX::BI__imma_m16n16k16_mma_u8:
15943   case NVPTX::BI__imma_m32n8k16_mma_s8:
15944   case NVPTX::BI__imma_m32n8k16_mma_u8:
15945   case NVPTX::BI__imma_m8n32k16_mma_s8:
15946   case NVPTX::BI__imma_m8n32k16_mma_u8:
15947   case NVPTX::BI__imma_m8n8k32_mma_s4:
15948   case NVPTX::BI__imma_m8n8k32_mma_u4:
15949   case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1: {
15950     Address Dst = EmitPointerWithAlignment(E->getArg(0));
15951     Address SrcA = EmitPointerWithAlignment(E->getArg(1));
15952     Address SrcB = EmitPointerWithAlignment(E->getArg(2));
15953     Address SrcC = EmitPointerWithAlignment(E->getArg(3));
15954     llvm::APSInt LayoutArg;
15955     if (!E->getArg(4)->isIntegerConstantExpr(LayoutArg, getContext()))
15956       return nullptr;
15957     int Layout = LayoutArg.getSExtValue();
15958     if (Layout < 0 || Layout > 3)
15959       return nullptr;
15960     llvm::APSInt SatfArg;
15961     if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1)
15962       SatfArg = 0;  // .b1 does not have satf argument.
15963     else if (!E->getArg(5)->isIntegerConstantExpr(SatfArg, getContext()))
15964       return nullptr;
15965     bool Satf = SatfArg.getSExtValue();
15966     NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
15967     unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
15968     if (IID == 0)  // Unsupported combination of Layout/Satf.
15969       return nullptr;
15970
15971     SmallVector<Value *, 24> Values;
15972     Function *Intrinsic = CGM.getIntrinsic(IID);
15973     llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
15974     // Load A
15975     for (unsigned i = 0; i < MI.NumEltsA; ++i) {
15976       Value *V = Builder.CreateAlignedLoad(
15977           Builder.CreateGEP(SrcA.getPointer(),
15978                             llvm::ConstantInt::get(IntTy, i)),
15979           CharUnits::fromQuantity(4));
15980       Values.push_back(Builder.CreateBitCast(V, AType));
15981     }
15982     // Load B
15983     llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
15984     for (unsigned i = 0; i < MI.NumEltsB; ++i) {
15985       Value *V = Builder.CreateAlignedLoad(
15986           Builder.CreateGEP(SrcB.getPointer(),
15987                             llvm::ConstantInt::get(IntTy, i)),
15988           CharUnits::fromQuantity(4));
15989       Values.push_back(Builder.CreateBitCast(V, BType));
15990     }
15991     // Load C
15992     llvm::Type *CType =
15993         Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
15994     for (unsigned i = 0; i < MI.NumEltsC; ++i) {
15995       Value *V = Builder.CreateAlignedLoad(
15996           Builder.CreateGEP(SrcC.getPointer(),
15997                             llvm::ConstantInt::get(IntTy, i)),
15998           CharUnits::fromQuantity(4));
15999       Values.push_back(Builder.CreateBitCast(V, CType));
16000     }
16001     Value *Result = Builder.CreateCall(Intrinsic, Values);
16002     llvm::Type *DType = Dst.getElementType();
16003     for (unsigned i = 0; i < MI.NumEltsD; ++i)
16004       Builder.CreateAlignedStore(
16005           Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType),
16006           Builder.CreateGEP(Dst.getPointer(), llvm::ConstantInt::get(IntTy, i)),
16007           CharUnits::fromQuantity(4));
16008     return Result;
16009   }
16010   default:
16011     return nullptr;
16012   }
16013 }
16014
16015 namespace {
16016 struct BuiltinAlignArgs {
16017   llvm::Value *Src = nullptr;
16018   llvm::Type *SrcType = nullptr;
16019   llvm::Value *Alignment = nullptr;
16020   llvm::Value *Mask = nullptr;
16021   llvm::IntegerType *IntType = nullptr;
16022
16023   BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) {
16024     QualType AstType = E->getArg(0)->getType();
16025     if (AstType->isArrayType())
16026       Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).getPointer();
16027     else
16028       Src = CGF.EmitScalarExpr(E->getArg(0));
16029     SrcType = Src->getType();
16030     if (SrcType->isPointerTy()) {
16031       IntType = IntegerType::get(
16032           CGF.getLLVMContext(),
16033           CGF.CGM.getDataLayout().getIndexTypeSizeInBits(SrcType));
16034     } else {
16035       assert(SrcType->isIntegerTy());
16036       IntType = cast<llvm::IntegerType>(SrcType);
16037     }
16038     Alignment = CGF.EmitScalarExpr(E->getArg(1));
16039     Alignment = CGF.Builder.CreateZExtOrTrunc(Alignment, IntType, "alignment");
16040     auto *One = llvm::ConstantInt::get(IntType, 1);
16041     Mask = CGF.Builder.CreateSub(Alignment, One, "mask");
16042   }
16043 };
16044 } // namespace
16045
16046 /// Generate (x & (y-1)) == 0.
16047 RValue CodeGenFunction::EmitBuiltinIsAligned(const CallExpr *E) {
16048   BuiltinAlignArgs Args(E, *this);
16049   llvm::Value *SrcAddress = Args.Src;
16050   if (Args.SrcType->isPointerTy())
16051     SrcAddress =
16052         Builder.CreateBitOrPointerCast(Args.Src, Args.IntType, "src_addr");
16053   return RValue::get(Builder.CreateICmpEQ(
16054       Builder.CreateAnd(SrcAddress, Args.Mask, "set_bits"),
16055       llvm::Constant::getNullValue(Args.IntType), "is_aligned"));
16056 }
16057
16058 /// Generate (x & ~(y-1)) to align down or ((x+(y-1)) & ~(y-1)) to align up.
16059 /// Note: For pointer types we can avoid ptrtoint/inttoptr pairs by using the
16060 /// llvm.ptrmask instrinsic (with a GEP before in the align_up case).
16061 /// TODO: actually use ptrmask once most optimization passes know about it.
16062 RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
16063   BuiltinAlignArgs Args(E, *this);
16064   llvm::Value *SrcAddr = Args.Src;
16065   if (Args.Src->getType()->isPointerTy())
16066     SrcAddr = Builder.CreatePtrToInt(Args.Src, Args.IntType, "intptr");
16067   llvm::Value *SrcForMask = SrcAddr;
16068   if (AlignUp) {
16069     // When aligning up we have to first add the mask to ensure we go over the
16070     // next alignment value and then align down to the next valid multiple.
16071     // By adding the mask, we ensure that align_up on an already aligned
16072     // value will not change the value.
16073     SrcForMask = Builder.CreateAdd(SrcForMask, Args.Mask, "over_boundary");
16074   }
16075   // Invert the mask to only clear the lower bits.
16076   llvm::Value *InvertedMask = Builder.CreateNot(Args.Mask, "inverted_mask");
16077   llvm::Value *Result =
16078       Builder.CreateAnd(SrcForMask, InvertedMask, "aligned_result");
16079   if (Args.Src->getType()->isPointerTy()) {
16080     /// TODO: Use ptrmask instead of ptrtoint+gep once it is optimized well.
16081     // Result = Builder.CreateIntrinsic(
16082     //  Intrinsic::ptrmask, {Args.SrcType, SrcForMask->getType(), Args.IntType},
16083     //  {SrcForMask, NegatedMask}, nullptr, "aligned_result");
16084     Result->setName("aligned_intptr");
16085     llvm::Value *Difference = Builder.CreateSub(Result, SrcAddr, "diff");
16086     // The result must point to the same underlying allocation. This means we
16087     // can use an inbounds GEP to enable better optimization.
16088     Value *Base = EmitCastToVoidPtr(Args.Src);
16089     if (getLangOpts().isSignedOverflowDefined())
16090       Result = Builder.CreateGEP(Base, Difference, "aligned_result");
16091     else
16092       Result = EmitCheckedInBoundsGEP(Base, Difference,
16093                                       /*SignedIndices=*/true,
16094                                       /*isSubtraction=*/!AlignUp,
16095                                       E->getExprLoc(), "aligned_result");
16096     Result = Builder.CreatePointerCast(Result, Args.SrcType);
16097     // Emit an alignment assumption to ensure that the new alignment is
16098     // propagated to loads/stores, etc.
16099     emitAlignmentAssumption(Result, E, E->getExprLoc(), Args.Alignment);
16100   }
16101   assert(Result->getType() == Args.SrcType);
16102   return RValue::get(Result);
16103 }
16104
16105 Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
16106                                                    const CallExpr *E) {
16107   switch (BuiltinID) {
16108   case WebAssembly::BI__builtin_wasm_memory_size: {
16109     llvm::Type *ResultType = ConvertType(E->getType());
16110     Value *I = EmitScalarExpr(E->getArg(0));
16111     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_size, ResultType);
16112     return Builder.CreateCall(Callee, I);
16113   }
16114   case WebAssembly::BI__builtin_wasm_memory_grow: {
16115     llvm::Type *ResultType = ConvertType(E->getType());
16116     Value *Args[] = {
16117       EmitScalarExpr(E->getArg(0)),
16118       EmitScalarExpr(E->getArg(1))
16119     };
16120     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_grow, ResultType);
16121     return Builder.CreateCall(Callee, Args);
16122   }
16123   case WebAssembly::BI__builtin_wasm_tls_size: {
16124     llvm::Type *ResultType = ConvertType(E->getType());
16125     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_tls_size, ResultType);
16126     return Builder.CreateCall(Callee);
16127   }
16128   case WebAssembly::BI__builtin_wasm_tls_align: {
16129     llvm::Type *ResultType = ConvertType(E->getType());
16130     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_tls_align, ResultType);
16131     return Builder.CreateCall(Callee);
16132   }
16133   case WebAssembly::BI__builtin_wasm_tls_base: {
16134     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_tls_base);
16135     return Builder.CreateCall(Callee);
16136   }
16137   case WebAssembly::BI__builtin_wasm_throw: {
16138     Value *Tag = EmitScalarExpr(E->getArg(0));
16139     Value *Obj = EmitScalarExpr(E->getArg(1));
16140     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_throw);
16141     return Builder.CreateCall(Callee, {Tag, Obj});
16142   }
16143   case WebAssembly::BI__builtin_wasm_rethrow_in_catch: {
16144     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_rethrow_in_catch);
16145     return Builder.CreateCall(Callee);
16146   }
16147   case WebAssembly::BI__builtin_wasm_atomic_wait_i32: {
16148     Value *Addr = EmitScalarExpr(E->getArg(0));
16149     Value *Expected = EmitScalarExpr(E->getArg(1));
16150     Value *Timeout = EmitScalarExpr(E->getArg(2));
16151     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_atomic_wait_i32);
16152     return Builder.CreateCall(Callee, {Addr, Expected, Timeout});
16153   }
16154   case WebAssembly::BI__builtin_wasm_atomic_wait_i64: {
16155     Value *Addr = EmitScalarExpr(E->getArg(0));
16156     Value *Expected = EmitScalarExpr(E->getArg(1));
16157     Value *Timeout = EmitScalarExpr(E->getArg(2));
16158     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_atomic_wait_i64);
16159     return Builder.CreateCall(Callee, {Addr, Expected, Timeout});
16160   }
16161   case WebAssembly::BI__builtin_wasm_atomic_notify: {
16162     Value *Addr = EmitScalarExpr(E->getArg(0));
16163     Value *Count = EmitScalarExpr(E->getArg(1));
16164     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_atomic_notify);
16165     return Builder.CreateCall(Callee, {Addr, Count});
16166   }
16167   case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
16168   case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
16169   case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
16170   case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
16171     Value *Src = EmitScalarExpr(E->getArg(0));
16172     llvm::Type *ResT = ConvertType(E->getType());
16173     Function *Callee =
16174         CGM.getIntrinsic(Intrinsic::wasm_trunc_signed, {ResT, Src->getType()});
16175     return Builder.CreateCall(Callee, {Src});
16176   }
16177   case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
16178   case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
16179   case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
16180   case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
16181     Value *Src = EmitScalarExpr(E->getArg(0));
16182     llvm::Type *ResT = ConvertType(E->getType());
16183     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_trunc_unsigned,
16184                                         {ResT, Src->getType()});
16185     return Builder.CreateCall(Callee, {Src});
16186   }
16187   case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
16188   case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
16189   case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
16190   case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
16191   case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
16192     Value *Src = EmitScalarExpr(E->getArg(0));
16193     llvm::Type *ResT = ConvertType(E->getType());
16194     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_trunc_saturate_signed,
16195                                      {ResT, Src->getType()});
16196     return Builder.CreateCall(Callee, {Src});
16197   }
16198   case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
16199   case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
16200   case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
16201   case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
16202   case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
16203     Value *Src = EmitScalarExpr(E->getArg(0));
16204     llvm::Type *ResT = ConvertType(E->getType());
16205     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_trunc_saturate_unsigned,
16206                                      {ResT, Src->getType()});
16207     return Builder.CreateCall(Callee, {Src});
16208   }
16209   case WebAssembly::BI__builtin_wasm_min_f32:
16210   case WebAssembly::BI__builtin_wasm_min_f64:
16211   case WebAssembly::BI__builtin_wasm_min_f32x4:
16212   case WebAssembly::BI__builtin_wasm_min_f64x2: {
16213     Value *LHS = EmitScalarExpr(E->getArg(0));
16214     Value *RHS = EmitScalarExpr(E->getArg(1));
16215     Function *Callee = CGM.getIntrinsic(Intrinsic::minimum,
16216                                      ConvertType(E->getType()));
16217     return Builder.CreateCall(Callee, {LHS, RHS});
16218   }
16219   case WebAssembly::BI__builtin_wasm_max_f32:
16220   case WebAssembly::BI__builtin_wasm_max_f64:
16221   case WebAssembly::BI__builtin_wasm_max_f32x4:
16222   case WebAssembly::BI__builtin_wasm_max_f64x2: {
16223     Value *LHS = EmitScalarExpr(E->getArg(0));
16224     Value *RHS = EmitScalarExpr(E->getArg(1));
16225     Function *Callee = CGM.getIntrinsic(Intrinsic::maximum,
16226                                      ConvertType(E->getType()));
16227     return Builder.CreateCall(Callee, {LHS, RHS});
16228   }
16229   case WebAssembly::BI__builtin_wasm_pmin_f32x4:
16230   case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
16231     Value *LHS = EmitScalarExpr(E->getArg(0));
16232     Value *RHS = EmitScalarExpr(E->getArg(1));
16233     Function *Callee =
16234         CGM.getIntrinsic(Intrinsic::wasm_pmin, ConvertType(E->getType()));
16235     return Builder.CreateCall(Callee, {LHS, RHS});
16236   }
16237   case WebAssembly::BI__builtin_wasm_pmax_f32x4:
16238   case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
16239     Value *LHS = EmitScalarExpr(E->getArg(0));
16240     Value *RHS = EmitScalarExpr(E->getArg(1));
16241     Function *Callee =
16242         CGM.getIntrinsic(Intrinsic::wasm_pmax, ConvertType(E->getType()));
16243     return Builder.CreateCall(Callee, {LHS, RHS});
16244   }
16245   case WebAssembly::BI__builtin_wasm_ceil_f32x4:
16246   case WebAssembly::BI__builtin_wasm_floor_f32x4:
16247   case WebAssembly::BI__builtin_wasm_trunc_f32x4:
16248   case WebAssembly::BI__builtin_wasm_nearest_f32x4:
16249   case WebAssembly::BI__builtin_wasm_ceil_f64x2:
16250   case WebAssembly::BI__builtin_wasm_floor_f64x2:
16251   case WebAssembly::BI__builtin_wasm_trunc_f64x2:
16252   case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
16253     unsigned IntNo;
16254     switch (BuiltinID) {
16255     case WebAssembly::BI__builtin_wasm_ceil_f32x4:
16256     case WebAssembly::BI__builtin_wasm_ceil_f64x2:
16257       IntNo = Intrinsic::wasm_ceil;
16258       break;
16259     case WebAssembly::BI__builtin_wasm_floor_f32x4:
16260     case WebAssembly::BI__builtin_wasm_floor_f64x2:
16261       IntNo = Intrinsic::wasm_floor;
16262       break;
16263     case WebAssembly::BI__builtin_wasm_trunc_f32x4:
16264     case WebAssembly::BI__builtin_wasm_trunc_f64x2:
16265       IntNo = Intrinsic::wasm_trunc;
16266       break;
16267     case WebAssembly::BI__builtin_wasm_nearest_f32x4:
16268     case WebAssembly::BI__builtin_wasm_nearest_f64x2:
16269       IntNo = Intrinsic::wasm_nearest;
16270       break;
16271     default:
16272       llvm_unreachable("unexpected builtin ID");
16273     }
16274     Value *Value = EmitScalarExpr(E->getArg(0));
16275     Function *Callee = CGM.getIntrinsic(IntNo, ConvertType(E->getType()));
16276     return Builder.CreateCall(Callee, Value);
16277   }
16278   case WebAssembly::BI__builtin_wasm_swizzle_v8x16: {
16279     Value *Src = EmitScalarExpr(E->getArg(0));
16280     Value *Indices = EmitScalarExpr(E->getArg(1));
16281     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_swizzle);
16282     return Builder.CreateCall(Callee, {Src, Indices});
16283   }
16284   case WebAssembly::BI__builtin_wasm_extract_lane_s_i8x16:
16285   case WebAssembly::BI__builtin_wasm_extract_lane_u_i8x16:
16286   case WebAssembly::BI__builtin_wasm_extract_lane_s_i16x8:
16287   case WebAssembly::BI__builtin_wasm_extract_lane_u_i16x8:
16288   case WebAssembly::BI__builtin_wasm_extract_lane_i32x4:
16289   case WebAssembly::BI__builtin_wasm_extract_lane_i64x2:
16290   case WebAssembly::BI__builtin_wasm_extract_lane_f32x4:
16291   case WebAssembly::BI__builtin_wasm_extract_lane_f64x2: {
16292     llvm::APSInt LaneConst;
16293     if (!E->getArg(1)->isIntegerConstantExpr(LaneConst, getContext()))
16294       llvm_unreachable("Constant arg isn't actually constant?");
16295     Value *Vec = EmitScalarExpr(E->getArg(0));
16296     Value *Lane = llvm::ConstantInt::get(getLLVMContext(), LaneConst);
16297     Value *Extract = Builder.CreateExtractElement(Vec, Lane);
16298     switch (BuiltinID) {
16299     case WebAssembly::BI__builtin_wasm_extract_lane_s_i8x16:
16300     case WebAssembly::BI__builtin_wasm_extract_lane_s_i16x8:
16301       return Builder.CreateSExt(Extract, ConvertType(E->getType()));
16302     case WebAssembly::BI__builtin_wasm_extract_lane_u_i8x16:
16303     case WebAssembly::BI__builtin_wasm_extract_lane_u_i16x8:
16304       return Builder.CreateZExt(Extract, ConvertType(E->getType()));
16305     case WebAssembly::BI__builtin_wasm_extract_lane_i32x4:
16306     case WebAssembly::BI__builtin_wasm_extract_lane_i64x2:
16307     case WebAssembly::BI__builtin_wasm_extract_lane_f32x4:
16308     case WebAssembly::BI__builtin_wasm_extract_lane_f64x2:
16309       return Extract;
16310     default:
16311       llvm_unreachable("unexpected builtin ID");
16312     }
16313   }
16314   case WebAssembly::BI__builtin_wasm_replace_lane_i8x16:
16315   case WebAssembly::BI__builtin_wasm_replace_lane_i16x8:
16316   case WebAssembly::BI__builtin_wasm_replace_lane_i32x4:
16317   case WebAssembly::BI__builtin_wasm_replace_lane_i64x2:
16318   case WebAssembly::BI__builtin_wasm_replace_lane_f32x4:
16319   case WebAssembly::BI__builtin_wasm_replace_lane_f64x2: {
16320     llvm::APSInt LaneConst;
16321     if (!E->getArg(1)->isIntegerConstantExpr(LaneConst, getContext()))
16322       llvm_unreachable("Constant arg isn't actually constant?");
16323     Value *Vec = EmitScalarExpr(E->getArg(0));
16324     Value *Lane = llvm::ConstantInt::get(getLLVMContext(), LaneConst);
16325     Value *Val = EmitScalarExpr(E->getArg(2));
16326     switch (BuiltinID) {
16327     case WebAssembly::BI__builtin_wasm_replace_lane_i8x16:
16328     case WebAssembly::BI__builtin_wasm_replace_lane_i16x8: {
16329       llvm::Type *ElemType =
16330           cast<llvm::VectorType>(ConvertType(E->getType()))->getElementType();
16331       Value *Trunc = Builder.CreateTrunc(Val, ElemType);
16332       return Builder.CreateInsertElement(Vec, Trunc, Lane);
16333     }
16334     case WebAssembly::BI__builtin_wasm_replace_lane_i32x4:
16335     case WebAssembly::BI__builtin_wasm_replace_lane_i64x2:
16336     case WebAssembly::BI__builtin_wasm_replace_lane_f32x4:
16337     case WebAssembly::BI__builtin_wasm_replace_lane_f64x2:
16338       return Builder.CreateInsertElement(Vec, Val, Lane);
16339     default:
16340       llvm_unreachable("unexpected builtin ID");
16341     }
16342   }
16343   case WebAssembly::BI__builtin_wasm_add_saturate_s_i8x16:
16344   case WebAssembly::BI__builtin_wasm_add_saturate_u_i8x16:
16345   case WebAssembly::BI__builtin_wasm_add_saturate_s_i16x8:
16346   case WebAssembly::BI__builtin_wasm_add_saturate_u_i16x8:
16347   case WebAssembly::BI__builtin_wasm_sub_saturate_s_i8x16:
16348   case WebAssembly::BI__builtin_wasm_sub_saturate_u_i8x16:
16349   case WebAssembly::BI__builtin_wasm_sub_saturate_s_i16x8:
16350   case WebAssembly::BI__builtin_wasm_sub_saturate_u_i16x8: {
16351     unsigned IntNo;
16352     switch (BuiltinID) {
16353     case WebAssembly::BI__builtin_wasm_add_saturate_s_i8x16:
16354     case WebAssembly::BI__builtin_wasm_add_saturate_s_i16x8:
16355       IntNo = Intrinsic::sadd_sat;
16356       break;
16357     case WebAssembly::BI__builtin_wasm_add_saturate_u_i8x16:
16358     case WebAssembly::BI__builtin_wasm_add_saturate_u_i16x8:
16359       IntNo = Intrinsic::uadd_sat;
16360       break;
16361     case WebAssembly::BI__builtin_wasm_sub_saturate_s_i8x16:
16362     case WebAssembly::BI__builtin_wasm_sub_saturate_s_i16x8:
16363       IntNo = Intrinsic::wasm_sub_saturate_signed;
16364       break;
16365     case WebAssembly::BI__builtin_wasm_sub_saturate_u_i8x16:
16366     case WebAssembly::BI__builtin_wasm_sub_saturate_u_i16x8:
16367       IntNo = Intrinsic::wasm_sub_saturate_unsigned;
16368       break;
16369     default:
16370       llvm_unreachable("unexpected builtin ID");
16371     }
16372     Value *LHS = EmitScalarExpr(E->getArg(0));
16373     Value *RHS = EmitScalarExpr(E->getArg(1));
16374     Function *Callee = CGM.getIntrinsic(IntNo, ConvertType(E->getType()));
16375     return Builder.CreateCall(Callee, {LHS, RHS});
16376   }
16377   case WebAssembly::BI__builtin_wasm_abs_i8x16:
16378   case WebAssembly::BI__builtin_wasm_abs_i16x8:
16379   case WebAssembly::BI__builtin_wasm_abs_i32x4: {
16380     Value *Vec = EmitScalarExpr(E->getArg(0));
16381     Value *Neg = Builder.CreateNeg(Vec, "neg");
16382     Constant *Zero = llvm::Constant::getNullValue(Vec->getType());
16383     Value *ICmp = Builder.CreateICmpSLT(Vec, Zero, "abscond");
16384     return Builder.CreateSelect(ICmp, Neg, Vec, "abs");
16385   }
16386   case WebAssembly::BI__builtin_wasm_min_s_i8x16:
16387   case WebAssembly::BI__builtin_wasm_min_u_i8x16:
16388   case WebAssembly::BI__builtin_wasm_max_s_i8x16:
16389   case WebAssembly::BI__builtin_wasm_max_u_i8x16:
16390   case WebAssembly::BI__builtin_wasm_min_s_i16x8:
16391   case WebAssembly::BI__builtin_wasm_min_u_i16x8:
16392   case WebAssembly::BI__builtin_wasm_max_s_i16x8:
16393   case WebAssembly::BI__builtin_wasm_max_u_i16x8:
16394   case WebAssembly::BI__builtin_wasm_min_s_i32x4:
16395   case WebAssembly::BI__builtin_wasm_min_u_i32x4:
16396   case WebAssembly::BI__builtin_wasm_max_s_i32x4:
16397   case WebAssembly::BI__builtin_wasm_max_u_i32x4: {
16398     Value *LHS = EmitScalarExpr(E->getArg(0));
16399     Value *RHS = EmitScalarExpr(E->getArg(1));
16400     Value *ICmp;
16401     switch (BuiltinID) {
16402     case WebAssembly::BI__builtin_wasm_min_s_i8x16:
16403     case WebAssembly::BI__builtin_wasm_min_s_i16x8:
16404     case WebAssembly::BI__builtin_wasm_min_s_i32x4:
16405       ICmp = Builder.CreateICmpSLT(LHS, RHS);
16406       break;
16407     case WebAssembly::BI__builtin_wasm_min_u_i8x16:
16408     case WebAssembly::BI__builtin_wasm_min_u_i16x8:
16409     case WebAssembly::BI__builtin_wasm_min_u_i32x4:
16410       ICmp = Builder.CreateICmpULT(LHS, RHS);
16411       break;
16412     case WebAssembly::BI__builtin_wasm_max_s_i8x16:
16413     case WebAssembly::BI__builtin_wasm_max_s_i16x8:
16414     case WebAssembly::BI__builtin_wasm_max_s_i32x4:
16415       ICmp = Builder.CreateICmpSGT(LHS, RHS);
16416       break;
16417     case WebAssembly::BI__builtin_wasm_max_u_i8x16:
16418     case WebAssembly::BI__builtin_wasm_max_u_i16x8:
16419     case WebAssembly::BI__builtin_wasm_max_u_i32x4:
16420       ICmp = Builder.CreateICmpUGT(LHS, RHS);
16421       break;
16422     default:
16423       llvm_unreachable("unexpected builtin ID");
16424     }
16425     return Builder.CreateSelect(ICmp, LHS, RHS);
16426   }
16427   case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
16428   case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
16429     Value *LHS = EmitScalarExpr(E->getArg(0));
16430     Value *RHS = EmitScalarExpr(E->getArg(1));
16431     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_avgr_unsigned,
16432                                         ConvertType(E->getType()));
16433     return Builder.CreateCall(Callee, {LHS, RHS});
16434   }
16435   case WebAssembly::BI__builtin_wasm_bitselect: {
16436     Value *V1 = EmitScalarExpr(E->getArg(0));
16437     Value *V2 = EmitScalarExpr(E->getArg(1));
16438     Value *C = EmitScalarExpr(E->getArg(2));
16439     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_bitselect,
16440                                      ConvertType(E->getType()));
16441     return Builder.CreateCall(Callee, {V1, V2, C});
16442   }
16443   case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
16444     Value *LHS = EmitScalarExpr(E->getArg(0));
16445     Value *RHS = EmitScalarExpr(E->getArg(1));
16446     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_dot);
16447     return Builder.CreateCall(Callee, {LHS, RHS});
16448   }
16449   case WebAssembly::BI__builtin_wasm_any_true_i8x16:
16450   case WebAssembly::BI__builtin_wasm_any_true_i16x8:
16451   case WebAssembly::BI__builtin_wasm_any_true_i32x4:
16452   case WebAssembly::BI__builtin_wasm_any_true_i64x2:
16453   case WebAssembly::BI__builtin_wasm_all_true_i8x16:
16454   case WebAssembly::BI__builtin_wasm_all_true_i16x8:
16455   case WebAssembly::BI__builtin_wasm_all_true_i32x4:
16456   case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
16457     unsigned IntNo;
16458     switch (BuiltinID) {
16459     case WebAssembly::BI__builtin_wasm_any_true_i8x16:
16460     case WebAssembly::BI__builtin_wasm_any_true_i16x8:
16461     case WebAssembly::BI__builtin_wasm_any_true_i32x4:
16462     case WebAssembly::BI__builtin_wasm_any_true_i64x2:
16463       IntNo = Intrinsic::wasm_anytrue;
16464       break;
16465     case WebAssembly::BI__builtin_wasm_all_true_i8x16:
16466     case WebAssembly::BI__builtin_wasm_all_true_i16x8:
16467     case WebAssembly::BI__builtin_wasm_all_true_i32x4:
16468     case WebAssembly::BI__builtin_wasm_all_true_i64x2:
16469       IntNo = Intrinsic::wasm_alltrue;
16470       break;
16471     default:
16472       llvm_unreachable("unexpected builtin ID");
16473     }
16474     Value *Vec = EmitScalarExpr(E->getArg(0));
16475     Function *Callee = CGM.getIntrinsic(IntNo, Vec->getType());
16476     return Builder.CreateCall(Callee, {Vec});
16477   }
16478   case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
16479   case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
16480   case WebAssembly::BI__builtin_wasm_bitmask_i32x4: {
16481     Value *Vec = EmitScalarExpr(E->getArg(0));
16482     Function *Callee =
16483         CGM.getIntrinsic(Intrinsic::wasm_bitmask, Vec->getType());
16484     return Builder.CreateCall(Callee, {Vec});
16485   }
16486   case WebAssembly::BI__builtin_wasm_abs_f32x4:
16487   case WebAssembly::BI__builtin_wasm_abs_f64x2: {
16488     Value *Vec = EmitScalarExpr(E->getArg(0));
16489     Function *Callee = CGM.getIntrinsic(Intrinsic::fabs, Vec->getType());
16490     return Builder.CreateCall(Callee, {Vec});
16491   }
16492   case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
16493   case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
16494     Value *Vec = EmitScalarExpr(E->getArg(0));
16495     Function *Callee = CGM.getIntrinsic(Intrinsic::sqrt, Vec->getType());
16496     return Builder.CreateCall(Callee, {Vec});
16497   }
16498   case WebAssembly::BI__builtin_wasm_qfma_f32x4:
16499   case WebAssembly::BI__builtin_wasm_qfms_f32x4:
16500   case WebAssembly::BI__builtin_wasm_qfma_f64x2:
16501   case WebAssembly::BI__builtin_wasm_qfms_f64x2: {
16502     Value *A = EmitScalarExpr(E->getArg(0));
16503     Value *B = EmitScalarExpr(E->getArg(1));
16504     Value *C = EmitScalarExpr(E->getArg(2));
16505     unsigned IntNo;
16506     switch (BuiltinID) {
16507     case WebAssembly::BI__builtin_wasm_qfma_f32x4:
16508     case WebAssembly::BI__builtin_wasm_qfma_f64x2:
16509       IntNo = Intrinsic::wasm_qfma;
16510       break;
16511     case WebAssembly::BI__builtin_wasm_qfms_f32x4:
16512     case WebAssembly::BI__builtin_wasm_qfms_f64x2:
16513       IntNo = Intrinsic::wasm_qfms;
16514       break;
16515     default:
16516       llvm_unreachable("unexpected builtin ID");
16517     }
16518     Function *Callee = CGM.getIntrinsic(IntNo, A->getType());
16519     return Builder.CreateCall(Callee, {A, B, C});
16520   }
16521   case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
16522   case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
16523   case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
16524   case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
16525     Value *Low = EmitScalarExpr(E->getArg(0));
16526     Value *High = EmitScalarExpr(E->getArg(1));
16527     unsigned IntNo;
16528     switch (BuiltinID) {
16529     case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
16530     case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
16531       IntNo = Intrinsic::wasm_narrow_signed;
16532       break;
16533     case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
16534     case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
16535       IntNo = Intrinsic::wasm_narrow_unsigned;
16536       break;
16537     default:
16538       llvm_unreachable("unexpected builtin ID");
16539     }
16540     Function *Callee =
16541         CGM.getIntrinsic(IntNo, {ConvertType(E->getType()), Low->getType()});
16542     return Builder.CreateCall(Callee, {Low, High});
16543   }
16544   case WebAssembly::BI__builtin_wasm_widen_low_s_i16x8_i8x16:
16545   case WebAssembly::BI__builtin_wasm_widen_high_s_i16x8_i8x16:
16546   case WebAssembly::BI__builtin_wasm_widen_low_u_i16x8_i8x16:
16547   case WebAssembly::BI__builtin_wasm_widen_high_u_i16x8_i8x16:
16548   case WebAssembly::BI__builtin_wasm_widen_low_s_i32x4_i16x8:
16549   case WebAssembly::BI__builtin_wasm_widen_high_s_i32x4_i16x8:
16550   case WebAssembly::BI__builtin_wasm_widen_low_u_i32x4_i16x8:
16551   case WebAssembly::BI__builtin_wasm_widen_high_u_i32x4_i16x8: {
16552     Value *Vec = EmitScalarExpr(E->getArg(0));
16553     unsigned IntNo;
16554     switch (BuiltinID) {
16555     case WebAssembly::BI__builtin_wasm_widen_low_s_i16x8_i8x16:
16556     case WebAssembly::BI__builtin_wasm_widen_low_s_i32x4_i16x8:
16557       IntNo = Intrinsic::wasm_widen_low_signed;
16558       break;
16559     case WebAssembly::BI__builtin_wasm_widen_high_s_i16x8_i8x16:
16560     case WebAssembly::BI__builtin_wasm_widen_high_s_i32x4_i16x8:
16561       IntNo = Intrinsic::wasm_widen_high_signed;
16562       break;
16563     case WebAssembly::BI__builtin_wasm_widen_low_u_i16x8_i8x16:
16564     case WebAssembly::BI__builtin_wasm_widen_low_u_i32x4_i16x8:
16565       IntNo = Intrinsic::wasm_widen_low_unsigned;
16566       break;
16567     case WebAssembly::BI__builtin_wasm_widen_high_u_i16x8_i8x16:
16568     case WebAssembly::BI__builtin_wasm_widen_high_u_i32x4_i16x8:
16569       IntNo = Intrinsic::wasm_widen_high_unsigned;
16570       break;
16571     default:
16572       llvm_unreachable("unexpected builtin ID");
16573     }
16574     Function *Callee =
16575         CGM.getIntrinsic(IntNo, {ConvertType(E->getType()), Vec->getType()});
16576     return Builder.CreateCall(Callee, Vec);
16577   }
16578   case WebAssembly::BI__builtin_wasm_shuffle_v8x16: {
16579     Value *Ops[18];
16580     size_t OpIdx = 0;
16581     Ops[OpIdx++] = EmitScalarExpr(E->getArg(0));
16582     Ops[OpIdx++] = EmitScalarExpr(E->getArg(1));
16583     while (OpIdx < 18) {
16584       llvm::APSInt LaneConst;
16585       if (!E->getArg(OpIdx)->isIntegerConstantExpr(LaneConst, getContext()))
16586         llvm_unreachable("Constant arg isn't actually constant?");
16587       Ops[OpIdx++] = llvm::ConstantInt::get(getLLVMContext(), LaneConst);
16588     }
16589     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_shuffle);
16590     return Builder.CreateCall(Callee, Ops);
16591   }
16592   default:
16593     return nullptr;
16594   }
16595 }
16596
16597 static std::pair<Intrinsic::ID, unsigned>
16598 getIntrinsicForHexagonNonGCCBuiltin(unsigned BuiltinID) {
16599   struct Info {
16600     unsigned BuiltinID;
16601     Intrinsic::ID IntrinsicID;
16602     unsigned VecLen;
16603   };
16604   Info Infos[] = {
16605 #define CUSTOM_BUILTIN_MAPPING(x,s) \
16606   { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
16607     CUSTOM_BUILTIN_MAPPING(L2_loadrub_pci, 0)
16608     CUSTOM_BUILTIN_MAPPING(L2_loadrb_pci, 0)
16609     CUSTOM_BUILTIN_MAPPING(L2_loadruh_pci, 0)
16610     CUSTOM_BUILTIN_MAPPING(L2_loadrh_pci, 0)
16611     CUSTOM_BUILTIN_MAPPING(L2_loadri_pci, 0)
16612     CUSTOM_BUILTIN_MAPPING(L2_loadrd_pci, 0)
16613     CUSTOM_BUILTIN_MAPPING(L2_loadrub_pcr, 0)
16614     CUSTOM_BUILTIN_MAPPING(L2_loadrb_pcr, 0)
16615     CUSTOM_BUILTIN_MAPPING(L2_loadruh_pcr, 0)
16616     CUSTOM_BUILTIN_MAPPING(L2_loadrh_pcr, 0)
16617     CUSTOM_BUILTIN_MAPPING(L2_loadri_pcr, 0)
16618     CUSTOM_BUILTIN_MAPPING(L2_loadrd_pcr, 0)
16619     CUSTOM_BUILTIN_MAPPING(S2_storerb_pci, 0)
16620     CUSTOM_BUILTIN_MAPPING(S2_storerh_pci, 0)
16621     CUSTOM_BUILTIN_MAPPING(S2_storerf_pci, 0)
16622     CUSTOM_BUILTIN_MAPPING(S2_storeri_pci, 0)
16623     CUSTOM_BUILTIN_MAPPING(S2_storerd_pci, 0)
16624     CUSTOM_BUILTIN_MAPPING(S2_storerb_pcr, 0)
16625     CUSTOM_BUILTIN_MAPPING(S2_storerh_pcr, 0)
16626     CUSTOM_BUILTIN_MAPPING(S2_storerf_pcr, 0)
16627     CUSTOM_BUILTIN_MAPPING(S2_storeri_pcr, 0)
16628     CUSTOM_BUILTIN_MAPPING(S2_storerd_pcr, 0)
16629     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstoreq, 64)
16630     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstorenq, 64)
16631     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstorentq, 64)
16632     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstorentnq, 64)
16633     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstoreq_128B, 128)
16634     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstorenq_128B, 128)
16635     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstorentq_128B, 128)
16636     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstorentnq_128B, 128)
16637 #include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
16638 #undef CUSTOM_BUILTIN_MAPPING
16639   };
16640
16641   auto CmpInfo = [] (Info A, Info B) { return A.BuiltinID < B.BuiltinID; };
16642   static const bool SortOnce = (llvm::sort(Infos, CmpInfo), true);
16643   (void)SortOnce;
16644
16645   const Info *F = std::lower_bound(std::begin(Infos), std::end(Infos),
16646                                    Info{BuiltinID, 0, 0}, CmpInfo);
16647   if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
16648     return {Intrinsic::not_intrinsic, 0};
16649
16650   return {F->IntrinsicID, F->VecLen};
16651 }
16652
16653 Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
16654                                                const CallExpr *E) {
16655   Intrinsic::ID ID;
16656   unsigned VecLen;
16657   std::tie(ID, VecLen) = getIntrinsicForHexagonNonGCCBuiltin(BuiltinID);
16658
16659   auto MakeCircOp = [this, E](unsigned IntID, bool IsLoad) {
16660     // The base pointer is passed by address, so it needs to be loaded.
16661     Address A = EmitPointerWithAlignment(E->getArg(0));
16662     Address BP = Address(
16663         Builder.CreateBitCast(A.getPointer(), Int8PtrPtrTy), A.getAlignment());
16664     llvm::Value *Base = Builder.CreateLoad(BP);
16665     // The treatment of both loads and stores is the same: the arguments for
16666     // the builtin are the same as the arguments for the intrinsic.
16667     // Load:
16668     //   builtin(Base, Inc, Mod, Start) -> intr(Base, Inc, Mod, Start)
16669     //   builtin(Base, Mod, Start)      -> intr(Base, Mod, Start)
16670     // Store:
16671     //   builtin(Base, Inc, Mod, Val, Start) -> intr(Base, Inc, Mod, Val, Start)
16672     //   builtin(Base, Mod, Val, Start)      -> intr(Base, Mod, Val, Start)
16673     SmallVector<llvm::Value*,5> Ops = { Base };
16674     for (unsigned i = 1, e = E->getNumArgs(); i != e; ++i)
16675       Ops.push_back(EmitScalarExpr(E->getArg(i)));
16676
16677     llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
16678     // The load intrinsics generate two results (Value, NewBase), stores
16679     // generate one (NewBase). The new base address needs to be stored.
16680     llvm::Value *NewBase = IsLoad ? Builder.CreateExtractValue(Result, 1)
16681                                   : Result;
16682     llvm::Value *LV = Builder.CreateBitCast(
16683         EmitScalarExpr(E->getArg(0)), NewBase->getType()->getPointerTo());
16684     Address Dest = EmitPointerWithAlignment(E->getArg(0));
16685     llvm::Value *RetVal =
16686         Builder.CreateAlignedStore(NewBase, LV, Dest.getAlignment());
16687     if (IsLoad)
16688       RetVal = Builder.CreateExtractValue(Result, 0);
16689     return RetVal;
16690   };
16691
16692   // Handle the conversion of bit-reverse load intrinsics to bit code.
16693   // The intrinsic call after this function only reads from memory and the
16694   // write to memory is dealt by the store instruction.
16695   auto MakeBrevLd = [this, E](unsigned IntID, llvm::Type *DestTy) {
16696     // The intrinsic generates one result, which is the new value for the base
16697     // pointer. It needs to be returned. The result of the load instruction is
16698     // passed to intrinsic by address, so the value needs to be stored.
16699     llvm::Value *BaseAddress =
16700         Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int8PtrTy);
16701
16702     // Expressions like &(*pt++) will be incremented per evaluation.
16703     // EmitPointerWithAlignment and EmitScalarExpr evaluates the expression
16704     // per call.
16705     Address DestAddr = EmitPointerWithAlignment(E->getArg(1));
16706     DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), Int8PtrTy),
16707                        DestAddr.getAlignment());
16708     llvm::Value *DestAddress = DestAddr.getPointer();
16709
16710     // Operands are Base, Dest, Modifier.
16711     // The intrinsic format in LLVM IR is defined as
16712     // { ValueType, i8* } (i8*, i32).
16713     llvm::Value *Result = Builder.CreateCall(
16714         CGM.getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
16715
16716     // The value needs to be stored as the variable is passed by reference.
16717     llvm::Value *DestVal = Builder.CreateExtractValue(Result, 0);
16718
16719     // The store needs to be truncated to fit the destination type.
16720     // While i32 and i64 are natively supported on Hexagon, i8 and i16 needs
16721     // to be handled with stores of respective destination type.
16722     DestVal = Builder.CreateTrunc(DestVal, DestTy);
16723
16724     llvm::Value *DestForStore =
16725         Builder.CreateBitCast(DestAddress, DestVal->getType()->getPointerTo());
16726     Builder.CreateAlignedStore(DestVal, DestForStore, DestAddr.getAlignment());
16727     // The updated value of the base pointer is returned.
16728     return Builder.CreateExtractValue(Result, 1);
16729   };
16730
16731   auto V2Q = [this, VecLen] (llvm::Value *Vec) {
16732     Intrinsic::ID ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
16733                                      : Intrinsic::hexagon_V6_vandvrt;
16734     return Builder.CreateCall(CGM.getIntrinsic(ID),
16735                               {Vec, Builder.getInt32(-1)});
16736   };
16737   auto Q2V = [this, VecLen] (llvm::Value *Pred) {
16738     Intrinsic::ID ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
16739                                      : Intrinsic::hexagon_V6_vandqrt;
16740     return Builder.CreateCall(CGM.getIntrinsic(ID),
16741                               {Pred, Builder.getInt32(-1)});
16742   };
16743
16744   switch (BuiltinID) {
16745   // These intrinsics return a tuple {Vector, VectorPred} in LLVM IR,
16746   // and the corresponding C/C++ builtins use loads/stores to update
16747   // the predicate.
16748   case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
16749   case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
16750   case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
16751   case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
16752     // Get the type from the 0-th argument.
16753     llvm::Type *VecType = ConvertType(E->getArg(0)->getType());
16754     Address PredAddr = Builder.CreateBitCast(
16755         EmitPointerWithAlignment(E->getArg(2)), VecType->getPointerTo(0));
16756     llvm::Value *PredIn = V2Q(Builder.CreateLoad(PredAddr));
16757     llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID),
16758         {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
16759
16760     llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1);
16761     Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(),
16762         PredAddr.getAlignment());
16763     return Builder.CreateExtractValue(Result, 0);
16764   }
16765
16766   case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
16767   case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
16768   case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
16769   case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
16770   case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
16771   case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
16772   case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
16773   case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
16774   case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
16775   case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
16776   case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
16777   case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
16778     return MakeCircOp(ID, /*IsLoad=*/true);
16779   case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
16780   case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
16781   case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
16782   case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
16783   case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
16784   case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
16785   case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
16786   case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
16787   case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
16788   case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
16789     return MakeCircOp(ID, /*IsLoad=*/false);
16790   case Hexagon::BI__builtin_brev_ldub:
16791     return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr, Int8Ty);
16792   case Hexagon::BI__builtin_brev_ldb:
16793     return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr, Int8Ty);
16794   case Hexagon::BI__builtin_brev_lduh:
16795     return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr, Int16Ty);
16796   case Hexagon::BI__builtin_brev_ldh:
16797     return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr, Int16Ty);
16798   case Hexagon::BI__builtin_brev_ldw:
16799     return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr, Int32Ty);
16800   case Hexagon::BI__builtin_brev_ldd:
16801     return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr, Int64Ty);
16802
16803   default: {
16804     if (ID == Intrinsic::not_intrinsic)
16805       return nullptr;
16806
16807     auto IsVectorPredTy = [](llvm::Type *T) {
16808       return T->isVectorTy() &&
16809              cast<llvm::VectorType>(T)->getElementType()->isIntegerTy(1);
16810     };
16811
16812     llvm::Function *IntrFn = CGM.getIntrinsic(ID);
16813     llvm::FunctionType *IntrTy = IntrFn->getFunctionType();
16814     SmallVector<llvm::Value*,4> Ops;
16815     for (unsigned i = 0, e = IntrTy->getNumParams(); i != e; ++i) {
16816       llvm::Type *T = IntrTy->getParamType(i);
16817       const Expr *A = E->getArg(i);
16818       if (IsVectorPredTy(T)) {
16819         // There will be an implicit cast to a boolean vector. Strip it.
16820         if (auto *Cast = dyn_cast<ImplicitCastExpr>(A)) {
16821           if (Cast->getCastKind() == CK_BitCast)
16822             A = Cast->getSubExpr();
16823         }
16824         Ops.push_back(V2Q(EmitScalarExpr(A)));
16825       } else {
16826         Ops.push_back(EmitScalarExpr(A));
16827       }
16828     }
16829
16830     llvm::Value *Call = Builder.CreateCall(IntrFn, Ops);
16831     if (IsVectorPredTy(IntrTy->getReturnType()))
16832       Call = Q2V(Call);
16833
16834     return Call;
16835   } // default
16836   } // switch
16837
16838   return nullptr;
16839 }