1 //===- NeonEmitter.cpp - Generate arm_neon.h for use with clang -*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This tablegen backend is responsible for emitting arm_neon.h, which includes
11 // a declaration and definition of each function specified by the ARM NEON
12 // compiler interface. See ARM document DUI0348B.
14 // Each NEON instruction is implemented in terms of 1 or more functions which
15 // are suffixed with the element type of the input vectors. Functions may be
16 // implemented in terms of generic vector operations such as +, *, -, etc. or
17 // by calling a __builtin_-prefixed function which will be handled by clang's
20 // Additional validation code can be generated by this file when runHeader() is
21 // called, rather than the normal run() entry point.
23 // See also the documentation in include/clang/Basic/arm_neon.td.
25 //===----------------------------------------------------------------------===//
27 #include "llvm/ADT/DenseMap.h"
28 #include "llvm/ADT/STLExtras.h"
29 #include "llvm/ADT/SmallString.h"
30 #include "llvm/ADT/SmallVector.h"
31 #include "llvm/ADT/StringExtras.h"
32 #include "llvm/ADT/StringMap.h"
33 #include "llvm/Support/ErrorHandling.h"
34 #include "llvm/TableGen/Error.h"
35 #include "llvm/TableGen/Record.h"
36 #include "llvm/TableGen/SetTheory.h"
37 #include "llvm/TableGen/TableGenBackend.h"
47 // While globals are generally bad, this one allows us to perform assertions
48 // liberally and somehow still trace them back to the def they indirectly
50 static Record *CurrentRecord = nullptr;
51 static void assert_with_loc(bool Assertion, const std::string &Str) {
54 PrintFatalError(CurrentRecord->getLoc(), Str);
62 ClassI, // generic integer instruction, e.g., "i8" suffix
63 ClassS, // signed/unsigned/poly, e.g., "s8", "u8" or "p8" suffix
64 ClassW, // width-specific instruction, e.g., "8" suffix
65 ClassB, // bitcast arguments with enum argument to specify type
66 ClassL, // Logical instructions which are op instructions
67 // but we need to not emit any suffix for in our
69 ClassNoTest // Instructions which we do not test since they are
70 // not TRUE instructions.
73 /// NeonTypeFlags - Flags to identify the types for overloaded Neon
74 /// builtins. These must be kept in sync with the flags in
75 /// include/clang/Basic/TargetBuiltins.h.
76 namespace NeonTypeFlags {
77 enum { EltTypeMask = 0xf, UnsignedFlag = 0x10, QuadFlag = 0x20 };
99 //===----------------------------------------------------------------------===//
101 //===----------------------------------------------------------------------===//
103 /// A TypeSpec is just a simple wrapper around a string, but gets its own type
104 /// for strong typing purposes.
106 /// A TypeSpec can be used to create a type.
107 class TypeSpec : public std::string {
109 static std::vector<TypeSpec> fromTypeSpecs(StringRef Str) {
110 std::vector<TypeSpec> Ret;
112 for (char I : Str.str()) {
115 Ret.push_back(TypeSpec(Acc));
125 //===----------------------------------------------------------------------===//
127 //===----------------------------------------------------------------------===//
129 /// A Type. Not much more to say here.
134 bool Float, Signed, Immediate, Void, Poly, Constant, Pointer;
135 // ScalarForMangling and NoManglingQ are really not suited to live here as
136 // they are not related to the type. But they live in the TypeSpec (not the
137 // prototype), so this is really the only place to store them.
138 bool ScalarForMangling, NoManglingQ;
139 unsigned Bitwidth, ElementBitwidth, NumVectors;
143 : Float(false), Signed(false), Immediate(false), Void(true), Poly(false),
144 Constant(false), Pointer(false), ScalarForMangling(false),
145 NoManglingQ(false), Bitwidth(0), ElementBitwidth(0), NumVectors(0) {}
147 Type(TypeSpec TS, char CharMod)
148 : TS(TS), Float(false), Signed(false), Immediate(false), Void(false),
149 Poly(false), Constant(false), Pointer(false), ScalarForMangling(false),
150 NoManglingQ(false), Bitwidth(0), ElementBitwidth(0), NumVectors(0) {
151 applyModifier(CharMod);
154 /// Returns a type representing "void".
155 static Type getVoid() { return Type(); }
157 bool operator==(const Type &Other) const { return str() == Other.str(); }
158 bool operator!=(const Type &Other) const { return !operator==(Other); }
163 bool isScalarForMangling() const { return ScalarForMangling; }
164 bool noManglingQ() const { return NoManglingQ; }
166 bool isPointer() const { return Pointer; }
167 bool isFloating() const { return Float; }
168 bool isInteger() const { return !Float && !Poly; }
169 bool isSigned() const { return Signed; }
170 bool isImmediate() const { return Immediate; }
171 bool isScalar() const { return NumVectors == 0; }
172 bool isVector() const { return NumVectors > 0; }
173 bool isFloat() const { return Float && ElementBitwidth == 32; }
174 bool isDouble() const { return Float && ElementBitwidth == 64; }
175 bool isHalf() const { return Float && ElementBitwidth == 16; }
176 bool isPoly() const { return Poly; }
177 bool isChar() const { return ElementBitwidth == 8; }
178 bool isShort() const { return !Float && ElementBitwidth == 16; }
179 bool isInt() const { return !Float && ElementBitwidth == 32; }
180 bool isLong() const { return !Float && ElementBitwidth == 64; }
181 bool isVoid() const { return Void; }
182 unsigned getNumElements() const { return Bitwidth / ElementBitwidth; }
183 unsigned getSizeInBits() const { return Bitwidth; }
184 unsigned getElementSizeInBits() const { return ElementBitwidth; }
185 unsigned getNumVectors() const { return NumVectors; }
190 void makeUnsigned() { Signed = false; }
191 void makeSigned() { Signed = true; }
192 void makeInteger(unsigned ElemWidth, bool Sign) {
197 ElementBitwidth = ElemWidth;
199 void makeImmediate(unsigned ElemWidth) {
204 ElementBitwidth = ElemWidth;
207 Bitwidth = ElementBitwidth;
210 void makeOneVector() {
215 assert_with_loc(Bitwidth != 128, "Can't get bigger than 128!");
219 assert_with_loc(Bitwidth != 64, "Can't get smaller than 64!");
223 /// Return the C string representation of a type, which is the typename
224 /// defined in stdint.h or arm_neon.h.
225 std::string str() const;
227 /// Return the string representation of a type, which is an encoded
228 /// string for passing to the BUILTIN() macro in Builtins.def.
229 std::string builtin_str() const;
231 /// Return the value in NeonTypeFlags for this type.
232 unsigned getNeonEnum() const;
234 /// Parse a type from a stdint.h or arm_neon.h typedef name,
235 /// for example uint32x2_t or int64_t.
236 static Type fromTypedefName(StringRef Name);
239 /// Creates the type based on the typespec string in TS.
240 /// Sets "Quad" to true if the "Q" or "H" modifiers were
241 /// seen. This is needed by applyModifier as some modifiers
242 /// only take effect if the type size was changed by "Q" or "H".
243 void applyTypespec(bool &Quad);
244 /// Applies a prototype modifier to the type.
245 void applyModifier(char Mod);
248 //===----------------------------------------------------------------------===//
250 //===----------------------------------------------------------------------===//
252 /// A variable is a simple class that just has a type and a name.
258 Variable() : T(Type::getVoid()), N("") {}
259 Variable(Type T, std::string N) : T(T), N(N) {}
261 Type getType() const { return T; }
262 std::string getName() const { return "__" + N; }
265 //===----------------------------------------------------------------------===//
267 //===----------------------------------------------------------------------===//
269 /// The main grunt class. This represents an instantiation of an intrinsic with
270 /// a particular typespec and prototype.
272 friend class DagEmitter;
274 /// The Record this intrinsic was created from.
276 /// The unmangled name and prototype.
277 std::string Name, Proto;
278 /// The input and output typespecs. InTS == OutTS except when
279 /// CartesianProductOfTypes is 1 - this is the case for vreinterpret.
280 TypeSpec OutTS, InTS;
281 /// The base class kind. Most intrinsics use ClassS, which has full type
282 /// info for integers (s32/u32). Some use ClassI, which doesn't care about
283 /// signedness (i32), while some (ClassB) have no type at all, only a width
286 /// The list of DAGs for the body. May be empty, in which case we should
287 /// emit a builtin call.
289 /// The architectural #ifdef guard.
291 /// Set if the Unvailable bit is 1. This means we don't generate a body,
292 /// just an "unavailable" attribute on a declaration.
294 /// Is this intrinsic safe for big-endian? or does it need its arguments
298 /// The types of return value [0] and parameters [1..].
299 std::vector<Type> Types;
300 /// The local variables defined.
301 std::map<std::string, Variable> Variables;
302 /// NeededEarly - set if any other intrinsic depends on this intrinsic.
304 /// UseMacro - set if we should implement using a macro or unset for a
307 /// The set of intrinsics that this intrinsic uses/requires.
308 std::set<Intrinsic *> Dependencies;
309 /// The "base type", which is Type('d', OutTS). InBaseType is only
310 /// different if CartesianProductOfTypes = 1 (for vreinterpret).
311 Type BaseType, InBaseType;
312 /// The return variable.
314 /// A postfix to apply to every variable. Defaults to "".
315 std::string VariablePostfix;
317 NeonEmitter &Emitter;
318 std::stringstream OS;
321 Intrinsic(Record *R, StringRef Name, StringRef Proto, TypeSpec OutTS,
322 TypeSpec InTS, ClassKind CK, ListInit *Body, NeonEmitter &Emitter,
323 StringRef Guard, bool IsUnavailable, bool BigEndianSafe)
324 : R(R), Name(Name.str()), Proto(Proto.str()), OutTS(OutTS), InTS(InTS),
325 CK(CK), Body(Body), Guard(Guard.str()), IsUnavailable(IsUnavailable),
326 BigEndianSafe(BigEndianSafe), NeededEarly(false), UseMacro(false),
327 BaseType(OutTS, 'd'), InBaseType(InTS, 'd'), Emitter(Emitter) {
328 // If this builtin takes an immediate argument, we need to #define it rather
329 // than use a standard declaration, so that SemaChecking can range check
330 // the immediate passed by the user.
331 if (Proto.find('i') != std::string::npos)
334 // Pointer arguments need to use macros to avoid hiding aligned attributes
335 // from the pointer type.
336 if (Proto.find('p') != std::string::npos ||
337 Proto.find('c') != std::string::npos)
340 // It is not permitted to pass or return an __fp16 by value, so intrinsics
341 // taking a scalar float16_t must be implemented as macros.
342 if (OutTS.find('h') != std::string::npos &&
343 Proto.find('s') != std::string::npos)
346 // Modify the TypeSpec per-argument to get a concrete Type, and create
347 // known variables for each.
348 // Types[0] is the return value.
349 Types.emplace_back(OutTS, Proto[0]);
350 for (unsigned I = 1; I < Proto.size(); ++I)
351 Types.emplace_back(InTS, Proto[I]);
354 /// Get the Record that this intrinsic is based off.
355 Record *getRecord() const { return R; }
356 /// Get the set of Intrinsics that this intrinsic calls.
357 /// this is the set of immediate dependencies, NOT the
358 /// transitive closure.
359 const std::set<Intrinsic *> &getDependencies() const { return Dependencies; }
360 /// Get the architectural guard string (#ifdef).
361 std::string getGuard() const { return Guard; }
362 /// Get the non-mangled name.
363 std::string getName() const { return Name; }
365 /// Return true if the intrinsic takes an immediate operand.
366 bool hasImmediate() const {
367 return Proto.find('i') != std::string::npos;
369 /// Return the parameter index of the immediate operand.
370 unsigned getImmediateIdx() const {
371 assert(hasImmediate());
372 unsigned Idx = Proto.find('i');
373 assert(Idx > 0 && "Can't return an immediate!");
377 /// Return true if the intrinsic takes an splat operand.
378 bool hasSplat() const { return Proto.find('a') != std::string::npos; }
379 /// Return the parameter index of the splat operand.
380 unsigned getSplatIdx() const {
382 unsigned Idx = Proto.find('a');
383 assert(Idx > 0 && "Can't return a splat!");
387 unsigned getNumParams() const { return Proto.size() - 1; }
388 Type getReturnType() const { return Types[0]; }
389 Type getParamType(unsigned I) const { return Types[I + 1]; }
390 Type getBaseType() const { return BaseType; }
391 /// Return the raw prototype string.
392 std::string getProto() const { return Proto; }
394 /// Return true if the prototype has a scalar argument.
395 /// This does not return true for the "splat" code ('a').
396 bool protoHasScalar();
398 /// Return the index that parameter PIndex will sit at
399 /// in a generated function call. This is often just PIndex,
400 /// but may not be as things such as multiple-vector operands
401 /// and sret parameters need to be taken into accont.
402 unsigned getGeneratedParamIdx(unsigned PIndex) {
404 if (getReturnType().getNumVectors() > 1)
405 // Multiple vectors are passed as sret.
408 for (unsigned I = 0; I < PIndex; ++I)
409 Idx += std::max(1U, getParamType(I).getNumVectors());
414 bool hasBody() const { return Body && Body->getValues().size() > 0; }
416 void setNeededEarly() { NeededEarly = true; }
418 bool operator<(const Intrinsic &Other) const {
419 // Sort lexicographically on a two-tuple (Guard, Name)
420 if (Guard != Other.Guard)
421 return Guard < Other.Guard;
422 return Name < Other.Name;
425 ClassKind getClassKind(bool UseClassBIfScalar = false) {
426 if (UseClassBIfScalar && !protoHasScalar())
431 /// Return the name, mangled with type information.
432 /// If ForceClassS is true, use ClassS (u32/s32) instead
433 /// of the intrinsic's own type class.
434 std::string getMangledName(bool ForceClassS = false);
435 /// Return the type code for a builtin function call.
436 std::string getInstTypeCode(Type T, ClassKind CK);
437 /// Return the type string for a BUILTIN() macro in Builtins.def.
438 std::string getBuiltinTypeStr();
440 /// Generate the intrinsic, returning code.
441 std::string generate();
442 /// Perform type checking and populate the dependency graph, but
443 /// don't generate code yet.
447 std::string mangleName(std::string Name, ClassKind CK);
449 void initVariables();
450 std::string replaceParamsIn(std::string S);
452 void emitBodyAsBuiltinCall();
454 void generateImpl(bool ReverseArguments,
455 StringRef NamePrefix, StringRef CallPrefix);
457 void emitBody(StringRef CallPrefix);
458 void emitShadowedArgs();
459 void emitArgumentReversal();
460 void emitReturnReversal();
461 void emitReverseVariable(Variable &Dest, Variable &Src);
463 void emitClosingBrace();
464 void emitOpeningBrace();
465 void emitPrototype(StringRef NamePrefix);
469 StringRef CallPrefix;
472 DagEmitter(Intrinsic &Intr, StringRef CallPrefix) :
473 Intr(Intr), CallPrefix(CallPrefix) {
475 std::pair<Type, std::string> emitDagArg(Init *Arg, std::string ArgName);
476 std::pair<Type, std::string> emitDagSaveTemp(DagInit *DI);
477 std::pair<Type, std::string> emitDagSplat(DagInit *DI);
478 std::pair<Type, std::string> emitDagDup(DagInit *DI);
479 std::pair<Type, std::string> emitDagShuffle(DagInit *DI);
480 std::pair<Type, std::string> emitDagCast(DagInit *DI, bool IsBitCast);
481 std::pair<Type, std::string> emitDagCall(DagInit *DI);
482 std::pair<Type, std::string> emitDagNameReplace(DagInit *DI);
483 std::pair<Type, std::string> emitDagLiteral(DagInit *DI);
484 std::pair<Type, std::string> emitDagOp(DagInit *DI);
485 std::pair<Type, std::string> emitDag(DagInit *DI);
490 //===----------------------------------------------------------------------===//
492 //===----------------------------------------------------------------------===//
495 RecordKeeper &Records;
496 DenseMap<Record *, ClassKind> ClassMap;
497 std::map<std::string, std::vector<Intrinsic *>> IntrinsicMap;
498 unsigned UniqueNumber;
500 void createIntrinsic(Record *R, SmallVectorImpl<Intrinsic *> &Out);
501 void genBuiltinsDef(raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs);
502 void genOverloadTypeCheckCode(raw_ostream &OS,
503 SmallVectorImpl<Intrinsic *> &Defs);
504 void genIntrinsicRangeCheckCode(raw_ostream &OS,
505 SmallVectorImpl<Intrinsic *> &Defs);
508 /// Called by Intrinsic - this attempts to get an intrinsic that takes
509 /// the given types as arguments.
510 Intrinsic *getIntrinsic(StringRef Name, ArrayRef<Type> Types);
512 /// Called by Intrinsic - returns a globally-unique number.
513 unsigned getUniqueNumber() { return UniqueNumber++; }
515 NeonEmitter(RecordKeeper &R) : Records(R), UniqueNumber(0) {
516 Record *SI = R.getClass("SInst");
517 Record *II = R.getClass("IInst");
518 Record *WI = R.getClass("WInst");
519 Record *SOpI = R.getClass("SOpInst");
520 Record *IOpI = R.getClass("IOpInst");
521 Record *WOpI = R.getClass("WOpInst");
522 Record *LOpI = R.getClass("LOpInst");
523 Record *NoTestOpI = R.getClass("NoTestOpInst");
525 ClassMap[SI] = ClassS;
526 ClassMap[II] = ClassI;
527 ClassMap[WI] = ClassW;
528 ClassMap[SOpI] = ClassS;
529 ClassMap[IOpI] = ClassI;
530 ClassMap[WOpI] = ClassW;
531 ClassMap[LOpI] = ClassL;
532 ClassMap[NoTestOpI] = ClassNoTest;
535 // run - Emit arm_neon.h.inc
536 void run(raw_ostream &o);
538 // runHeader - Emit all the __builtin prototypes used in arm_neon.h
539 void runHeader(raw_ostream &o);
541 // runTests - Emit tests for all the Neon intrinsics.
542 void runTests(raw_ostream &o);
545 } // end anonymous namespace
547 //===----------------------------------------------------------------------===//
548 // Type implementation
549 //===----------------------------------------------------------------------===//
551 std::string Type::str() const {
556 if (!Signed && isInteger())
566 S += utostr(ElementBitwidth);
568 S += "x" + utostr(getNumElements());
570 S += "x" + utostr(NumVectors);
581 std::string Type::builtin_str() const {
587 // All pointers are void pointers.
589 else if (isInteger())
590 switch (ElementBitwidth) {
591 case 8: S += "c"; break;
592 case 16: S += "s"; break;
593 case 32: S += "i"; break;
594 case 64: S += "Wi"; break;
595 case 128: S += "LLLi"; break;
596 default: llvm_unreachable("Unhandled case!");
599 switch (ElementBitwidth) {
600 case 16: S += "h"; break;
601 case 32: S += "f"; break;
602 case 64: S += "d"; break;
603 default: llvm_unreachable("Unhandled case!");
606 if (isChar() && !Pointer)
607 // Make chars explicitly signed.
609 else if (isInteger() && !Pointer && !Signed)
612 // Constant indices are "int", but have the "constant expression" modifier.
614 assert(isInteger() && isSigned());
619 if (Constant) S += "C";
620 if (Pointer) S += "*";
625 for (unsigned I = 0; I < NumVectors; ++I)
626 Ret += "V" + utostr(getNumElements()) + S;
631 unsigned Type::getNeonEnum() const {
633 switch (ElementBitwidth) {
634 case 8: Addend = 0; break;
635 case 16: Addend = 1; break;
636 case 32: Addend = 2; break;
637 case 64: Addend = 3; break;
638 case 128: Addend = 4; break;
639 default: llvm_unreachable("Unhandled element bitwidth!");
642 unsigned Base = (unsigned)NeonTypeFlags::Int8 + Addend;
644 // Adjustment needed because Poly32 doesn't exist.
647 Base = (unsigned)NeonTypeFlags::Poly8 + Addend;
650 assert(Addend != 0 && "Float8 doesn't exist!");
651 Base = (unsigned)NeonTypeFlags::Float16 + (Addend - 1);
655 Base |= (unsigned)NeonTypeFlags::QuadFlag;
656 if (isInteger() && !Signed)
657 Base |= (unsigned)NeonTypeFlags::UnsignedFlag;
662 Type Type::fromTypedefName(StringRef Name) {
668 if (Name.front() == 'u') {
670 Name = Name.drop_front();
675 if (Name.startswith("float")) {
677 Name = Name.drop_front(5);
678 } else if (Name.startswith("poly")) {
680 Name = Name.drop_front(4);
682 assert(Name.startswith("int"));
683 Name = Name.drop_front(3);
687 for (I = 0; I < Name.size(); ++I) {
688 if (!isdigit(Name[I]))
691 Name.substr(0, I).getAsInteger(10, T.ElementBitwidth);
692 Name = Name.drop_front(I);
694 T.Bitwidth = T.ElementBitwidth;
697 if (Name.front() == 'x') {
698 Name = Name.drop_front();
700 for (I = 0; I < Name.size(); ++I) {
701 if (!isdigit(Name[I]))
705 Name.substr(0, I).getAsInteger(10, NumLanes);
706 Name = Name.drop_front(I);
707 T.Bitwidth = T.ElementBitwidth * NumLanes;
712 if (Name.front() == 'x') {
713 Name = Name.drop_front();
715 for (I = 0; I < Name.size(); ++I) {
716 if (!isdigit(Name[I]))
719 Name.substr(0, I).getAsInteger(10, T.NumVectors);
720 Name = Name.drop_front(I);
723 assert(Name.startswith("_t") && "Malformed typedef!");
727 void Type::applyTypespec(bool &Quad) {
729 ScalarForMangling = false;
731 Poly = Float = false;
732 ElementBitwidth = ~0U;
739 ScalarForMangling = true;
761 ElementBitwidth = 16;
767 ElementBitwidth = 32;
773 ElementBitwidth = 64;
776 ElementBitwidth = 128;
777 // Poly doesn't have a 128x1 type.
782 llvm_unreachable("Unhandled type code!");
785 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
787 Bitwidth = Quad ? 128 : 64;
790 void Type::applyModifier(char Mod) {
791 bool AppliedQuad = false;
792 applyTypespec(AppliedQuad);
809 Bitwidth = ElementBitwidth;
816 Bitwidth = ElementBitwidth;
825 assert(!Poly && "'u' can't be used with poly types!");
829 Bitwidth = ElementBitwidth = 64;
834 Bitwidth = ElementBitwidth = 32;
839 // Special case - if we're half-precision, a floating
840 // point argument needs to be 128-bits (double size).
844 ElementBitwidth = 32;
848 ElementBitwidth = 64;
859 ElementBitwidth *= 2;
863 ElementBitwidth *= 2;
868 ElementBitwidth = Bitwidth = 32;
876 ElementBitwidth = Bitwidth = 64;
882 ElementBitwidth /= 2;
883 Bitwidth = ElementBitwidth;
887 ElementBitwidth *= 2;
888 Bitwidth = ElementBitwidth;
893 Bitwidth = ElementBitwidth;
904 Bitwidth = ElementBitwidth;
908 ElementBitwidth /= 2;
911 ElementBitwidth /= 2;
915 ElementBitwidth /= 2;
919 ElementBitwidth /= 2;
949 llvm_unreachable("Unhandled character!");
953 //===----------------------------------------------------------------------===//
954 // Intrinsic implementation
955 //===----------------------------------------------------------------------===//
957 std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) {
958 char typeCode = '\0';
959 bool printNumber = true;
966 else if (T.isInteger())
967 typeCode = T.isSigned() ? 's' : 'u';
987 if (typeCode != '\0')
988 S.push_back(typeCode);
990 S += utostr(T.getElementSizeInBits());
995 std::string Intrinsic::getBuiltinTypeStr() {
996 ClassKind LocalCK = getClassKind(true);
999 Type RetT = getReturnType();
1000 if ((LocalCK == ClassI || LocalCK == ClassW) && RetT.isScalar() &&
1002 RetT.makeInteger(RetT.getElementSizeInBits(), false);
1004 // Since the return value must be one type, return a vector type of the
1005 // appropriate width which we will bitcast. An exception is made for
1006 // returning structs of 2, 3, or 4 vectors which are returned in a sret-like
1007 // fashion, storing them to a pointer arg.
1008 if (RetT.getNumVectors() > 1) {
1009 S += "vv*"; // void result with void* first argument
1012 RetT.makeInteger(RetT.getElementSizeInBits(), false);
1013 if (!RetT.isScalar() && !RetT.isSigned())
1016 bool ForcedVectorFloatingType = Proto[0] == 'F' || Proto[0] == 'f';
1017 if (LocalCK == ClassB && !RetT.isScalar() && !ForcedVectorFloatingType)
1018 // Cast to vector of 8-bit elements.
1019 RetT.makeInteger(8, true);
1021 S += RetT.builtin_str();
1024 for (unsigned I = 0; I < getNumParams(); ++I) {
1025 Type T = getParamType(I);
1027 T.makeInteger(T.getElementSizeInBits(), false);
1029 bool ForcedFloatingType = Proto[I + 1] == 'F' || Proto[I + 1] == 'f';
1030 if (LocalCK == ClassB && !T.isScalar() && !ForcedFloatingType)
1031 T.makeInteger(8, true);
1032 // Halves always get converted to 8-bit elements.
1033 if (T.isHalf() && T.isVector() && !T.isScalarForMangling())
1034 T.makeInteger(8, true);
1036 if (LocalCK == ClassI)
1039 if (hasImmediate() && getImmediateIdx() == I)
1040 T.makeImmediate(32);
1042 S += T.builtin_str();
1045 // Extra constant integer to hold type class enum for this function, e.g. s8
1046 if (LocalCK == ClassB)
1052 std::string Intrinsic::getMangledName(bool ForceClassS) {
1053 // Check if the prototype has a scalar operand with the type of the vector
1054 // elements. If not, bitcasting the args will take care of arg checking.
1055 // The actual signedness etc. will be taken care of with special enums.
1056 ClassKind LocalCK = CK;
1057 if (!protoHasScalar())
1060 return mangleName(Name, ForceClassS ? ClassS : LocalCK);
1063 std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) {
1064 std::string typeCode = getInstTypeCode(BaseType, LocalCK);
1065 std::string S = Name;
1067 if (Name == "vcvt_f32_f16" || Name == "vcvt_f32_f64" ||
1068 Name == "vcvt_f64_f32")
1071 if (typeCode.size() > 0) {
1072 // If the name ends with _xN (N = 2,3,4), insert the typeCode before _xN.
1073 if (Name.size() >= 3 && isdigit(Name.back()) &&
1074 Name[Name.length() - 2] == 'x' && Name[Name.length() - 3] == '_')
1075 S.insert(S.length() - 3, "_" + typeCode);
1077 S += "_" + typeCode;
1080 if (BaseType != InBaseType) {
1081 // A reinterpret - out the input base type at the end.
1082 S += "_" + getInstTypeCode(InBaseType, LocalCK);
1085 if (LocalCK == ClassB)
1088 // Insert a 'q' before the first '_' character so that it ends up before
1089 // _lane or _n on vector-scalar operations.
1090 if (BaseType.getSizeInBits() == 128 && !BaseType.noManglingQ()) {
1091 size_t Pos = S.find('_');
1096 if (BaseType.isScalarForMangling()) {
1097 switch (BaseType.getElementSizeInBits()) {
1098 case 8: Suffix = 'b'; break;
1099 case 16: Suffix = 'h'; break;
1100 case 32: Suffix = 's'; break;
1101 case 64: Suffix = 'd'; break;
1102 default: llvm_unreachable("Bad suffix!");
1105 if (Suffix != '\0') {
1106 size_t Pos = S.find('_');
1107 S.insert(Pos, &Suffix, 1);
1113 std::string Intrinsic::replaceParamsIn(std::string S) {
1114 while (S.find('$') != std::string::npos) {
1115 size_t Pos = S.find('$');
1116 size_t End = Pos + 1;
1117 while (isalpha(S[End]))
1120 std::string VarName = S.substr(Pos + 1, End - Pos - 1);
1121 assert_with_loc(Variables.find(VarName) != Variables.end(),
1122 "Variable not defined!");
1123 S.replace(Pos, End - Pos, Variables.find(VarName)->second.getName());
1129 void Intrinsic::initVariables() {
1132 // Modify the TypeSpec per-argument to get a concrete Type, and create
1133 // known variables for each.
1134 for (unsigned I = 1; I < Proto.size(); ++I) {
1135 char NameC = '0' + (I - 1);
1136 std::string Name = "p";
1137 Name.push_back(NameC);
1139 Variables[Name] = Variable(Types[I], Name + VariablePostfix);
1141 RetVar = Variable(Types[0], "ret" + VariablePostfix);
1144 void Intrinsic::emitPrototype(StringRef NamePrefix) {
1148 OS << "__ai " << Types[0].str() << " ";
1150 OS << NamePrefix.str() << mangleName(Name, ClassS) << "(";
1152 for (unsigned I = 0; I < getNumParams(); ++I) {
1156 char NameC = '0' + I;
1157 std::string Name = "p";
1158 Name.push_back(NameC);
1159 assert(Variables.find(Name) != Variables.end());
1160 Variable &V = Variables[Name];
1163 OS << V.getType().str() << " ";
1170 void Intrinsic::emitOpeningBrace() {
1172 OS << " __extension__ ({";
1178 void Intrinsic::emitClosingBrace() {
1185 void Intrinsic::emitNewLine() {
1192 void Intrinsic::emitReverseVariable(Variable &Dest, Variable &Src) {
1193 if (Dest.getType().getNumVectors() > 1) {
1196 for (unsigned K = 0; K < Dest.getType().getNumVectors(); ++K) {
1197 OS << " " << Dest.getName() << ".val[" << utostr(K) << "] = "
1198 << "__builtin_shufflevector("
1199 << Src.getName() << ".val[" << utostr(K) << "], "
1200 << Src.getName() << ".val[" << utostr(K) << "]";
1201 for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
1202 OS << ", " << utostr(J);
1207 OS << " " << Dest.getName()
1208 << " = __builtin_shufflevector(" << Src.getName() << ", " << Src.getName();
1209 for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
1210 OS << ", " << utostr(J);
1216 void Intrinsic::emitArgumentReversal() {
1220 // Reverse all vector arguments.
1221 for (unsigned I = 0; I < getNumParams(); ++I) {
1222 std::string Name = "p" + utostr(I);
1223 std::string NewName = "rev" + utostr(I);
1225 Variable &V = Variables[Name];
1226 Variable NewV(V.getType(), NewName + VariablePostfix);
1228 if (!NewV.getType().isVector() || NewV.getType().getNumElements() == 1)
1231 OS << " " << NewV.getType().str() << " " << NewV.getName() << ";";
1232 emitReverseVariable(NewV, V);
1237 void Intrinsic::emitReturnReversal() {
1240 if (!getReturnType().isVector() || getReturnType().isVoid() ||
1241 getReturnType().getNumElements() == 1)
1243 emitReverseVariable(RetVar, RetVar);
1247 void Intrinsic::emitShadowedArgs() {
1248 // Macro arguments are not type-checked like inline function arguments,
1249 // so assign them to local temporaries to get the right type checking.
1253 for (unsigned I = 0; I < getNumParams(); ++I) {
1254 // Do not create a temporary for an immediate argument.
1255 // That would defeat the whole point of using a macro!
1256 if (hasImmediate() && Proto[I+1] == 'i')
1258 // Do not create a temporary for pointer arguments. The input
1259 // pointer may have an alignment hint.
1260 if (getParamType(I).isPointer())
1263 std::string Name = "p" + utostr(I);
1265 assert(Variables.find(Name) != Variables.end());
1266 Variable &V = Variables[Name];
1268 std::string NewName = "s" + utostr(I);
1269 Variable V2(V.getType(), NewName + VariablePostfix);
1271 OS << " " << V2.getType().str() << " " << V2.getName() << " = "
1272 << V.getName() << ";";
1279 // We don't check 'a' in this function, because for builtin function the
1280 // argument matching to 'a' uses a vector type splatted from a scalar type.
1281 bool Intrinsic::protoHasScalar() {
1282 return (Proto.find('s') != std::string::npos ||
1283 Proto.find('z') != std::string::npos ||
1284 Proto.find('r') != std::string::npos ||
1285 Proto.find('b') != std::string::npos ||
1286 Proto.find('$') != std::string::npos ||
1287 Proto.find('y') != std::string::npos ||
1288 Proto.find('o') != std::string::npos);
1291 void Intrinsic::emitBodyAsBuiltinCall() {
1294 // If this builtin returns a struct 2, 3, or 4 vectors, pass it as an implicit
1295 // sret-like argument.
1296 bool SRet = getReturnType().getNumVectors() >= 2;
1300 // Call the non-splat builtin: chop off the "_n" suffix from the name.
1301 assert(N.endswith("_n"));
1305 ClassKind LocalCK = CK;
1306 if (!protoHasScalar())
1309 if (!getReturnType().isVoid() && !SRet)
1310 S += "(" + RetVar.getType().str() + ") ";
1312 S += "__builtin_neon_" + mangleName(N, LocalCK) + "(";
1315 S += "&" + RetVar.getName() + ", ";
1317 for (unsigned I = 0; I < getNumParams(); ++I) {
1318 Variable &V = Variables["p" + utostr(I)];
1319 Type T = V.getType();
1321 // Handle multiple-vector values specially, emitting each subvector as an
1322 // argument to the builtin.
1323 if (T.getNumVectors() > 1) {
1324 // Check if an explicit cast is needed.
1326 if (T.isChar() || T.isPoly() || !T.isSigned()) {
1329 T2.makeInteger(8, /*Signed=*/true);
1330 Cast = "(" + T2.str() + ")";
1333 for (unsigned J = 0; J < T.getNumVectors(); ++J)
1334 S += Cast + V.getName() + ".val[" + utostr(J) + "], ";
1339 Type CastToType = T;
1340 if (hasSplat() && I == getSplatIdx()) {
1341 Arg = "(" + BaseType.str() + ") {";
1342 for (unsigned J = 0; J < BaseType.getNumElements(); ++J) {
1349 CastToType = BaseType;
1354 // Check if an explicit cast is needed.
1355 if (CastToType.isVector()) {
1356 CastToType.makeInteger(8, true);
1357 Arg = "(" + CastToType.str() + ")" + Arg;
1363 // Extra constant integer to hold type class enum for this function, e.g. s8
1364 if (getClassKind(true) == ClassB) {
1365 Type ThisTy = getReturnType();
1366 if (Proto[0] == 'v' || Proto[0] == 'f' || Proto[0] == 'F')
1367 ThisTy = getParamType(0);
1368 if (ThisTy.isPointer())
1369 ThisTy = getParamType(1);
1371 S += utostr(ThisTy.getNeonEnum());
1373 // Remove extraneous ", ".
1379 std::string RetExpr;
1380 if (!SRet && !RetVar.getType().isVoid())
1381 RetExpr = RetVar.getName() + " = ";
1383 OS << " " << RetExpr << S;
1387 void Intrinsic::emitBody(StringRef CallPrefix) {
1388 std::vector<std::string> Lines;
1390 assert(RetVar.getType() == Types[0]);
1391 // Create a return variable, if we're not void.
1392 if (!RetVar.getType().isVoid()) {
1393 OS << " " << RetVar.getType().str() << " " << RetVar.getName() << ";";
1397 if (!Body || Body->getValues().size() == 0) {
1398 // Nothing specific to output - must output a builtin.
1399 emitBodyAsBuiltinCall();
1403 // We have a list of "things to output". The last should be returned.
1404 for (auto *I : Body->getValues()) {
1405 if (StringInit *SI = dyn_cast<StringInit>(I)) {
1406 Lines.push_back(replaceParamsIn(SI->getAsString()));
1407 } else if (DagInit *DI = dyn_cast<DagInit>(I)) {
1408 DagEmitter DE(*this, CallPrefix);
1409 Lines.push_back(DE.emitDag(DI).second + ";");
1413 assert(!Lines.empty() && "Empty def?");
1414 if (!RetVar.getType().isVoid())
1415 Lines.back().insert(0, RetVar.getName() + " = ");
1417 for (auto &L : Lines) {
1423 void Intrinsic::emitReturn() {
1424 if (RetVar.getType().isVoid())
1427 OS << " " << RetVar.getName() << ";";
1429 OS << " return " << RetVar.getName() << ";";
1433 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDag(DagInit *DI) {
1434 // At this point we should only be seeing a def.
1435 DefInit *DefI = cast<DefInit>(DI->getOperator());
1436 std::string Op = DefI->getAsString();
1438 if (Op == "cast" || Op == "bitcast")
1439 return emitDagCast(DI, Op == "bitcast");
1440 if (Op == "shuffle")
1441 return emitDagShuffle(DI);
1443 return emitDagDup(DI);
1445 return emitDagSplat(DI);
1446 if (Op == "save_temp")
1447 return emitDagSaveTemp(DI);
1449 return emitDagOp(DI);
1451 return emitDagCall(DI);
1452 if (Op == "name_replace")
1453 return emitDagNameReplace(DI);
1454 if (Op == "literal")
1455 return emitDagLiteral(DI);
1456 assert_with_loc(false, "Unknown operation!");
1457 return std::make_pair(Type::getVoid(), "");
1460 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagOp(DagInit *DI) {
1461 std::string Op = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1462 if (DI->getNumArgs() == 2) {
1464 std::pair<Type, std::string> R =
1465 emitDagArg(DI->getArg(1), DI->getArgName(1));
1466 return std::make_pair(R.first, Op + R.second);
1468 assert(DI->getNumArgs() == 3 && "Can only handle unary and binary ops!");
1469 std::pair<Type, std::string> R1 =
1470 emitDagArg(DI->getArg(1), DI->getArgName(1));
1471 std::pair<Type, std::string> R2 =
1472 emitDagArg(DI->getArg(2), DI->getArgName(2));
1473 assert_with_loc(R1.first == R2.first, "Argument type mismatch!");
1474 return std::make_pair(R1.first, R1.second + " " + Op + " " + R2.second);
1478 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCall(DagInit *DI) {
1479 std::vector<Type> Types;
1480 std::vector<std::string> Values;
1481 for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
1482 std::pair<Type, std::string> R =
1483 emitDagArg(DI->getArg(I + 1), DI->getArgName(I + 1));
1484 Types.push_back(R.first);
1485 Values.push_back(R.second);
1488 // Look up the called intrinsic.
1490 if (StringInit *SI = dyn_cast<StringInit>(DI->getArg(0)))
1491 N = SI->getAsUnquotedString();
1493 N = emitDagArg(DI->getArg(0), "").second;
1494 Intrinsic *Callee = Intr.Emitter.getIntrinsic(N, Types);
1495 assert(Callee && "getIntrinsic should not return us nullptr!");
1497 // Make sure the callee is known as an early def.
1498 Callee->setNeededEarly();
1499 Intr.Dependencies.insert(Callee);
1501 // Now create the call itself.
1502 std::string S = CallPrefix.str() + Callee->getMangledName(true) + "(";
1503 for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
1510 return std::make_pair(Callee->getReturnType(), S);
1513 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCast(DagInit *DI,
1515 // (cast MOD* VAL) -> cast VAL to type given by MOD.
1516 std::pair<Type, std::string> R = emitDagArg(
1517 DI->getArg(DI->getNumArgs() - 1), DI->getArgName(DI->getNumArgs() - 1));
1518 Type castToType = R.first;
1519 for (unsigned ArgIdx = 0; ArgIdx < DI->getNumArgs() - 1; ++ArgIdx) {
1521 // MOD can take several forms:
1522 // 1. $X - take the type of parameter / variable X.
1523 // 2. The value "R" - take the type of the return type.
1525 // 4. The value "U" or "S" to switch the signedness.
1526 // 5. The value "H" or "D" to half or double the bitwidth.
1527 // 6. The value "8" to convert to 8-bit (signed) integer lanes.
1528 if (DI->getArgName(ArgIdx).size()) {
1529 assert_with_loc(Intr.Variables.find(DI->getArgName(ArgIdx)) !=
1530 Intr.Variables.end(),
1531 "Variable not found");
1532 castToType = Intr.Variables[DI->getArgName(ArgIdx)].getType();
1534 StringInit *SI = dyn_cast<StringInit>(DI->getArg(ArgIdx));
1535 assert_with_loc(SI, "Expected string type or $Name for cast type");
1537 if (SI->getAsUnquotedString() == "R") {
1538 castToType = Intr.getReturnType();
1539 } else if (SI->getAsUnquotedString() == "U") {
1540 castToType.makeUnsigned();
1541 } else if (SI->getAsUnquotedString() == "S") {
1542 castToType.makeSigned();
1543 } else if (SI->getAsUnquotedString() == "H") {
1544 castToType.halveLanes();
1545 } else if (SI->getAsUnquotedString() == "D") {
1546 castToType.doubleLanes();
1547 } else if (SI->getAsUnquotedString() == "8") {
1548 castToType.makeInteger(8, true);
1550 castToType = Type::fromTypedefName(SI->getAsUnquotedString());
1551 assert_with_loc(!castToType.isVoid(), "Unknown typedef");
1558 // Emit a reinterpret cast. The second operand must be an lvalue, so create
1560 std::string N = "reint";
1562 while (Intr.Variables.find(N) != Intr.Variables.end())
1563 N = "reint" + utostr(++I);
1564 Intr.Variables[N] = Variable(R.first, N + Intr.VariablePostfix);
1566 Intr.OS << R.first.str() << " " << Intr.Variables[N].getName() << " = "
1570 S = "*(" + castToType.str() + " *) &" + Intr.Variables[N].getName() + "";
1572 // Emit a normal (static) cast.
1573 S = "(" + castToType.str() + ")(" + R.second + ")";
1576 return std::make_pair(castToType, S);
1579 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagShuffle(DagInit *DI){
1580 // See the documentation in arm_neon.td for a description of these operators.
1581 class LowHalf : public SetTheory::Operator {
1583 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1584 ArrayRef<SMLoc> Loc) override {
1585 SetTheory::RecSet Elts2;
1586 ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc);
1587 Elts.insert(Elts2.begin(), Elts2.begin() + (Elts2.size() / 2));
1590 class HighHalf : public SetTheory::Operator {
1592 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1593 ArrayRef<SMLoc> Loc) override {
1594 SetTheory::RecSet Elts2;
1595 ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc);
1596 Elts.insert(Elts2.begin() + (Elts2.size() / 2), Elts2.end());
1599 class Rev : public SetTheory::Operator {
1600 unsigned ElementSize;
1603 Rev(unsigned ElementSize) : ElementSize(ElementSize) {}
1604 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1605 ArrayRef<SMLoc> Loc) override {
1606 SetTheory::RecSet Elts2;
1607 ST.evaluate(Expr->arg_begin() + 1, Expr->arg_end(), Elts2, Loc);
1609 int64_t VectorSize = cast<IntInit>(Expr->getArg(0))->getValue();
1610 VectorSize /= ElementSize;
1612 std::vector<Record *> Revved;
1613 for (unsigned VI = 0; VI < Elts2.size(); VI += VectorSize) {
1614 for (int LI = VectorSize - 1; LI >= 0; --LI) {
1615 Revved.push_back(Elts2[VI + LI]);
1619 Elts.insert(Revved.begin(), Revved.end());
1622 class MaskExpander : public SetTheory::Expander {
1626 MaskExpander(unsigned N) : N(N) {}
1627 void expand(SetTheory &ST, Record *R, SetTheory::RecSet &Elts) override {
1628 unsigned Addend = 0;
1629 if (R->getName() == "mask0")
1631 else if (R->getName() == "mask1")
1635 for (unsigned I = 0; I < N; ++I)
1636 Elts.insert(R->getRecords().getDef("sv" + utostr(I + Addend)));
1640 // (shuffle arg1, arg2, sequence)
1641 std::pair<Type, std::string> Arg1 =
1642 emitDagArg(DI->getArg(0), DI->getArgName(0));
1643 std::pair<Type, std::string> Arg2 =
1644 emitDagArg(DI->getArg(1), DI->getArgName(1));
1645 assert_with_loc(Arg1.first == Arg2.first,
1646 "Different types in arguments to shuffle!");
1649 SetTheory::RecSet Elts;
1650 ST.addOperator("lowhalf", llvm::make_unique<LowHalf>());
1651 ST.addOperator("highhalf", llvm::make_unique<HighHalf>());
1652 ST.addOperator("rev",
1653 llvm::make_unique<Rev>(Arg1.first.getElementSizeInBits()));
1654 ST.addExpander("MaskExpand",
1655 llvm::make_unique<MaskExpander>(Arg1.first.getNumElements()));
1656 ST.evaluate(DI->getArg(2), Elts, None);
1658 std::string S = "__builtin_shufflevector(" + Arg1.second + ", " + Arg2.second;
1659 for (auto &E : Elts) {
1660 StringRef Name = E->getName();
1661 assert_with_loc(Name.startswith("sv"),
1662 "Incorrect element kind in shuffle mask!");
1663 S += ", " + Name.drop_front(2).str();
1667 // Recalculate the return type - the shuffle may have halved or doubled it.
1669 if (Elts.size() > T.getNumElements()) {
1671 Elts.size() == T.getNumElements() * 2,
1672 "Can only double or half the number of elements in a shuffle!");
1674 } else if (Elts.size() < T.getNumElements()) {
1676 Elts.size() == T.getNumElements() / 2,
1677 "Can only double or half the number of elements in a shuffle!");
1681 return std::make_pair(T, S);
1684 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDup(DagInit *DI) {
1685 assert_with_loc(DI->getNumArgs() == 1, "dup() expects one argument");
1686 std::pair<Type, std::string> A = emitDagArg(DI->getArg(0), DI->getArgName(0));
1687 assert_with_loc(A.first.isScalar(), "dup() expects a scalar argument");
1689 Type T = Intr.getBaseType();
1690 assert_with_loc(T.isVector(), "dup() used but default type is scalar!");
1691 std::string S = "(" + T.str() + ") {";
1692 for (unsigned I = 0; I < T.getNumElements(); ++I) {
1699 return std::make_pair(T, S);
1702 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSplat(DagInit *DI) {
1703 assert_with_loc(DI->getNumArgs() == 2, "splat() expects two arguments");
1704 std::pair<Type, std::string> A = emitDagArg(DI->getArg(0), DI->getArgName(0));
1705 std::pair<Type, std::string> B = emitDagArg(DI->getArg(1), DI->getArgName(1));
1707 assert_with_loc(B.first.isScalar(),
1708 "splat() requires a scalar int as the second argument");
1710 std::string S = "__builtin_shufflevector(" + A.second + ", " + A.second;
1711 for (unsigned I = 0; I < Intr.getBaseType().getNumElements(); ++I) {
1712 S += ", " + B.second;
1716 return std::make_pair(Intr.getBaseType(), S);
1719 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSaveTemp(DagInit *DI) {
1720 assert_with_loc(DI->getNumArgs() == 2, "save_temp() expects two arguments");
1721 std::pair<Type, std::string> A = emitDagArg(DI->getArg(1), DI->getArgName(1));
1723 assert_with_loc(!A.first.isVoid(),
1724 "Argument to save_temp() must have non-void type!");
1726 std::string N = DI->getArgName(0);
1727 assert_with_loc(N.size(), "save_temp() expects a name as the first argument");
1729 assert_with_loc(Intr.Variables.find(N) == Intr.Variables.end(),
1730 "Variable already defined!");
1731 Intr.Variables[N] = Variable(A.first, N + Intr.VariablePostfix);
1734 A.first.str() + " " + Intr.Variables[N].getName() + " = " + A.second;
1736 return std::make_pair(Type::getVoid(), S);
1739 std::pair<Type, std::string>
1740 Intrinsic::DagEmitter::emitDagNameReplace(DagInit *DI) {
1741 std::string S = Intr.Name;
1743 assert_with_loc(DI->getNumArgs() == 2, "name_replace requires 2 arguments!");
1744 std::string ToReplace = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1745 std::string ReplaceWith = cast<StringInit>(DI->getArg(1))->getAsUnquotedString();
1747 size_t Idx = S.find(ToReplace);
1749 assert_with_loc(Idx != std::string::npos, "name should contain '" + ToReplace + "'!");
1750 S.replace(Idx, ToReplace.size(), ReplaceWith);
1752 return std::make_pair(Type::getVoid(), S);
1755 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagLiteral(DagInit *DI){
1756 std::string Ty = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1757 std::string Value = cast<StringInit>(DI->getArg(1))->getAsUnquotedString();
1758 return std::make_pair(Type::fromTypedefName(Ty), Value);
1761 std::pair<Type, std::string>
1762 Intrinsic::DagEmitter::emitDagArg(Init *Arg, std::string ArgName) {
1763 if (ArgName.size()) {
1764 assert_with_loc(!Arg->isComplete(),
1765 "Arguments must either be DAGs or names, not both!");
1766 assert_with_loc(Intr.Variables.find(ArgName) != Intr.Variables.end(),
1767 "Variable not defined!");
1768 Variable &V = Intr.Variables[ArgName];
1769 return std::make_pair(V.getType(), V.getName());
1772 assert(Arg && "Neither ArgName nor Arg?!");
1773 DagInit *DI = dyn_cast<DagInit>(Arg);
1774 assert_with_loc(DI, "Arguments must either be DAGs or names!");
1779 std::string Intrinsic::generate() {
1780 // Little endian intrinsics are simple and don't require any argument
1782 OS << "#ifdef __LITTLE_ENDIAN__\n";
1784 generateImpl(false, "", "");
1788 // Big endian intrinsics are more complex. The user intended these
1789 // intrinsics to operate on a vector "as-if" loaded by (V)LDR,
1790 // but we load as-if (V)LD1. So we should swap all arguments and
1791 // swap the return value too.
1793 // If we call sub-intrinsics, we should call a version that does
1794 // not re-swap the arguments!
1795 generateImpl(true, "", "__noswap_");
1797 // If we're needed early, create a non-swapping variant for
1800 generateImpl(false, "__noswap_", "__noswap_");
1807 void Intrinsic::generateImpl(bool ReverseArguments,
1808 StringRef NamePrefix, StringRef CallPrefix) {
1811 // If we call a macro, our local variables may be corrupted due to
1812 // lack of proper lexical scoping. So, add a globally unique postfix
1813 // to every variable.
1815 // indexBody() should have set up the Dependencies set by now.
1816 for (auto *I : Dependencies)
1818 VariablePostfix = "_" + utostr(Emitter.getUniqueNumber());
1824 emitPrototype(NamePrefix);
1826 if (IsUnavailable) {
1827 OS << " __attribute__((unavailable));";
1831 if (ReverseArguments)
1832 emitArgumentReversal();
1833 emitBody(CallPrefix);
1834 if (ReverseArguments)
1835 emitReturnReversal();
1841 CurrentRecord = nullptr;
1844 void Intrinsic::indexBody() {
1851 CurrentRecord = nullptr;
1854 //===----------------------------------------------------------------------===//
1855 // NeonEmitter implementation
1856 //===----------------------------------------------------------------------===//
1858 Intrinsic *NeonEmitter::getIntrinsic(StringRef Name, ArrayRef<Type> Types) {
1859 // First, look up the name in the intrinsic map.
1860 assert_with_loc(IntrinsicMap.find(Name.str()) != IntrinsicMap.end(),
1861 ("Intrinsic '" + Name + "' not found!").str());
1862 std::vector<Intrinsic *> &V = IntrinsicMap[Name.str()];
1863 std::vector<Intrinsic *> GoodVec;
1865 // Create a string to print if we end up failing.
1866 std::string ErrMsg = "looking up intrinsic '" + Name.str() + "(";
1867 for (unsigned I = 0; I < Types.size(); ++I) {
1870 ErrMsg += Types[I].str();
1873 ErrMsg += "Available overloads:\n";
1875 // Now, look through each intrinsic implementation and see if the types are
1878 ErrMsg += " - " + I->getReturnType().str() + " " + I->getMangledName();
1880 for (unsigned A = 0; A < I->getNumParams(); ++A) {
1883 ErrMsg += I->getParamType(A).str();
1887 if (I->getNumParams() != Types.size())
1891 for (unsigned Arg = 0; Arg < Types.size(); ++Arg) {
1892 if (I->getParamType(Arg) != Types[Arg]) {
1898 GoodVec.push_back(I);
1901 assert_with_loc(GoodVec.size() > 0,
1902 "No compatible intrinsic found - " + ErrMsg);
1903 assert_with_loc(GoodVec.size() == 1, "Multiple overloads found - " + ErrMsg);
1905 return GoodVec.front();
1908 void NeonEmitter::createIntrinsic(Record *R,
1909 SmallVectorImpl<Intrinsic *> &Out) {
1910 std::string Name = R->getValueAsString("Name");
1911 std::string Proto = R->getValueAsString("Prototype");
1912 std::string Types = R->getValueAsString("Types");
1913 Record *OperationRec = R->getValueAsDef("Operation");
1914 bool CartesianProductOfTypes = R->getValueAsBit("CartesianProductOfTypes");
1915 bool BigEndianSafe = R->getValueAsBit("BigEndianSafe");
1916 std::string Guard = R->getValueAsString("ArchGuard");
1917 bool IsUnavailable = OperationRec->getValueAsBit("Unavailable");
1919 // Set the global current record. This allows assert_with_loc to produce
1920 // decent location information even when highly nested.
1923 ListInit *Body = OperationRec->getValueAsListInit("Ops");
1925 std::vector<TypeSpec> TypeSpecs = TypeSpec::fromTypeSpecs(Types);
1927 ClassKind CK = ClassNone;
1928 if (R->getSuperClasses().size() >= 2)
1929 CK = ClassMap[R->getSuperClasses()[1]];
1931 std::vector<std::pair<TypeSpec, TypeSpec>> NewTypeSpecs;
1932 for (auto TS : TypeSpecs) {
1933 if (CartesianProductOfTypes) {
1934 Type DefaultT(TS, 'd');
1935 for (auto SrcTS : TypeSpecs) {
1936 Type DefaultSrcT(SrcTS, 'd');
1938 DefaultSrcT.getSizeInBits() != DefaultT.getSizeInBits())
1940 NewTypeSpecs.push_back(std::make_pair(TS, SrcTS));
1943 NewTypeSpecs.push_back(std::make_pair(TS, TS));
1947 std::sort(NewTypeSpecs.begin(), NewTypeSpecs.end());
1948 NewTypeSpecs.erase(std::unique(NewTypeSpecs.begin(), NewTypeSpecs.end()),
1949 NewTypeSpecs.end());
1951 for (auto &I : NewTypeSpecs) {
1952 Intrinsic *IT = new Intrinsic(R, Name, Proto, I.first, I.second, CK, Body,
1953 *this, Guard, IsUnavailable, BigEndianSafe);
1955 IntrinsicMap[Name].push_back(IT);
1959 CurrentRecord = nullptr;
1962 /// genBuiltinsDef: Generate the BuiltinsARM.def and BuiltinsAArch64.def
1963 /// declaration of builtins, checking for unique builtin declarations.
1964 void NeonEmitter::genBuiltinsDef(raw_ostream &OS,
1965 SmallVectorImpl<Intrinsic *> &Defs) {
1966 OS << "#ifdef GET_NEON_BUILTINS\n";
1968 // We only want to emit a builtin once, and we want to emit them in
1969 // alphabetical order, so use a std::set.
1970 std::set<std::string> Builtins;
1972 for (auto *Def : Defs) {
1975 // Functions with 'a' (the splat code) in the type prototype should not get
1976 // their own builtin as they use the non-splat variant.
1977 if (Def->hasSplat())
1980 std::string S = "BUILTIN(__builtin_neon_" + Def->getMangledName() + ", \"";
1982 S += Def->getBuiltinTypeStr();
1988 for (auto &S : Builtins)
1993 /// Generate the ARM and AArch64 overloaded type checking code for
1994 /// SemaChecking.cpp, checking for unique builtin declarations.
1995 void NeonEmitter::genOverloadTypeCheckCode(raw_ostream &OS,
1996 SmallVectorImpl<Intrinsic *> &Defs) {
1997 OS << "#ifdef GET_NEON_OVERLOAD_CHECK\n";
1999 // We record each overload check line before emitting because subsequent Inst
2000 // definitions may extend the number of permitted types (i.e. augment the
2001 // Mask). Use std::map to avoid sorting the table by hash number.
2002 struct OverloadInfo {
2006 OverloadInfo() : Mask(0ULL), PtrArgNum(0), HasConstPtr(false) {}
2008 std::map<std::string, OverloadInfo> OverloadMap;
2010 for (auto *Def : Defs) {
2011 // If the def has a body (that is, it has Operation DAGs), it won't call
2012 // __builtin_neon_* so we don't need to generate a definition for it.
2015 // Functions with 'a' (the splat code) in the type prototype should not get
2016 // their own builtin as they use the non-splat variant.
2017 if (Def->hasSplat())
2019 // Functions which have a scalar argument cannot be overloaded, no need to
2020 // check them if we are emitting the type checking code.
2021 if (Def->protoHasScalar())
2024 uint64_t Mask = 0ULL;
2025 Type Ty = Def->getReturnType();
2026 if (Def->getProto()[0] == 'v' || Def->getProto()[0] == 'f' ||
2027 Def->getProto()[0] == 'F')
2028 Ty = Def->getParamType(0);
2030 Ty = Def->getParamType(1);
2032 Mask |= 1ULL << Ty.getNeonEnum();
2034 // Check if the function has a pointer or const pointer argument.
2035 std::string Proto = Def->getProto();
2037 bool HasConstPtr = false;
2038 for (unsigned I = 0; I < Def->getNumParams(); ++I) {
2039 char ArgType = Proto[I + 1];
2040 if (ArgType == 'c') {
2045 if (ArgType == 'p') {
2050 // For sret builtins, adjust the pointer argument index.
2051 if (PtrArgNum >= 0 && Def->getReturnType().getNumVectors() > 1)
2054 std::string Name = Def->getName();
2055 // Omit type checking for the pointer arguments of vld1_lane, vld1_dup,
2056 // and vst1_lane intrinsics. Using a pointer to the vector element
2057 // type with one of those operations causes codegen to select an aligned
2058 // load/store instruction. If you want an unaligned operation,
2059 // the pointer argument needs to have less alignment than element type,
2060 // so just accept any pointer type.
2061 if (Name == "vld1_lane" || Name == "vld1_dup" || Name == "vst1_lane") {
2063 HasConstPtr = false;
2067 std::string Name = Def->getMangledName();
2068 OverloadMap.insert(std::make_pair(Name, OverloadInfo()));
2069 OverloadInfo &OI = OverloadMap[Name];
2071 OI.PtrArgNum |= PtrArgNum;
2072 OI.HasConstPtr = HasConstPtr;
2076 for (auto &I : OverloadMap) {
2077 OverloadInfo &OI = I.second;
2079 OS << "case NEON::BI__builtin_neon_" << I.first << ": ";
2080 OS << "mask = 0x" << utohexstr(OI.Mask) << "ULL";
2081 if (OI.PtrArgNum >= 0)
2082 OS << "; PtrArgNum = " << OI.PtrArgNum;
2084 OS << "; HasConstPtr = true";
2091 NeonEmitter::genIntrinsicRangeCheckCode(raw_ostream &OS,
2092 SmallVectorImpl<Intrinsic *> &Defs) {
2093 OS << "#ifdef GET_NEON_IMMEDIATE_CHECK\n";
2095 std::set<std::string> Emitted;
2097 for (auto *Def : Defs) {
2100 // Functions with 'a' (the splat code) in the type prototype should not get
2101 // their own builtin as they use the non-splat variant.
2102 if (Def->hasSplat())
2104 // Functions which do not have an immediate do not need to have range
2105 // checking code emitted.
2106 if (!Def->hasImmediate())
2108 if (Emitted.find(Def->getMangledName()) != Emitted.end())
2111 std::string LowerBound, UpperBound;
2113 Record *R = Def->getRecord();
2114 if (R->getValueAsBit("isVCVT_N")) {
2115 // VCVT between floating- and fixed-point values takes an immediate
2116 // in the range [1, 32) for f32 or [1, 64) for f64.
2118 if (Def->getBaseType().getElementSizeInBits() == 32)
2122 } else if (R->getValueAsBit("isScalarShift")) {
2123 // Right shifts have an 'r' in the name, left shifts do not. Convert
2124 // instructions have the same bounds and right shifts.
2125 if (Def->getName().find('r') != std::string::npos ||
2126 Def->getName().find("cvt") != std::string::npos)
2129 UpperBound = utostr(Def->getReturnType().getElementSizeInBits() - 1);
2130 } else if (R->getValueAsBit("isShift")) {
2131 // Builtins which are overloaded by type will need to have their upper
2132 // bound computed at Sema time based on the type constant.
2134 // Right shifts have an 'r' in the name, left shifts do not.
2135 if (Def->getName().find('r') != std::string::npos)
2137 UpperBound = "RFT(TV, true)";
2138 } else if (Def->getClassKind(true) == ClassB) {
2139 // ClassB intrinsics have a type (and hence lane number) that is only
2140 // known at runtime.
2141 if (R->getValueAsBit("isLaneQ"))
2142 UpperBound = "RFT(TV, false, true)";
2144 UpperBound = "RFT(TV, false, false)";
2146 // The immediate generally refers to a lane in the preceding argument.
2147 assert(Def->getImmediateIdx() > 0);
2148 Type T = Def->getParamType(Def->getImmediateIdx() - 1);
2149 UpperBound = utostr(T.getNumElements() - 1);
2152 // Calculate the index of the immediate that should be range checked.
2153 unsigned Idx = Def->getNumParams();
2154 if (Def->hasImmediate())
2155 Idx = Def->getGeneratedParamIdx(Def->getImmediateIdx());
2157 OS << "case NEON::BI__builtin_neon_" << Def->getMangledName() << ": "
2158 << "i = " << Idx << ";";
2159 if (LowerBound.size())
2160 OS << " l = " << LowerBound << ";";
2161 if (UpperBound.size())
2162 OS << " u = " << UpperBound << ";";
2165 Emitted.insert(Def->getMangledName());
2171 /// runHeader - Emit a file with sections defining:
2172 /// 1. the NEON section of BuiltinsARM.def and BuiltinsAArch64.def.
2173 /// 2. the SemaChecking code for the type overload checking.
2174 /// 3. the SemaChecking code for validation of intrinsic immediate arguments.
2175 void NeonEmitter::runHeader(raw_ostream &OS) {
2176 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2178 SmallVector<Intrinsic *, 128> Defs;
2180 createIntrinsic(R, Defs);
2182 // Generate shared BuiltinsXXX.def
2183 genBuiltinsDef(OS, Defs);
2185 // Generate ARM overloaded type checking code for SemaChecking.cpp
2186 genOverloadTypeCheckCode(OS, Defs);
2188 // Generate ARM range checking code for shift/lane immediates.
2189 genIntrinsicRangeCheckCode(OS, Defs);
2192 /// run - Read the records in arm_neon.td and output arm_neon.h. arm_neon.h
2193 /// is comprised of type definitions and function declarations.
2194 void NeonEmitter::run(raw_ostream &OS) {
2195 OS << "/*===---- arm_neon.h - ARM Neon intrinsics "
2196 "------------------------------"
2199 " * Permission is hereby granted, free of charge, to any person "
2202 " * of this software and associated documentation files (the "
2205 " * in the Software without restriction, including without limitation "
2208 " * to use, copy, modify, merge, publish, distribute, sublicense, "
2210 " * copies of the Software, and to permit persons to whom the Software "
2212 " * furnished to do so, subject to the following conditions:\n"
2214 " * The above copyright notice and this permission notice shall be "
2216 " * all copies or substantial portions of the Software.\n"
2218 " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, "
2220 " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF "
2221 "MERCHANTABILITY,\n"
2222 " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT "
2224 " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR "
2226 " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, "
2228 " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER "
2230 " * THE SOFTWARE.\n"
2232 " *===-----------------------------------------------------------------"
2237 OS << "#ifndef __ARM_NEON_H\n";
2238 OS << "#define __ARM_NEON_H\n\n";
2240 OS << "#if !defined(__ARM_NEON)\n";
2241 OS << "#error \"NEON support not enabled\"\n";
2244 OS << "#include <stdint.h>\n\n";
2246 // Emit NEON-specific scalar typedefs.
2247 OS << "typedef float float32_t;\n";
2248 OS << "typedef __fp16 float16_t;\n";
2250 OS << "#ifdef __aarch64__\n";
2251 OS << "typedef double float64_t;\n";
2254 // For now, signedness of polynomial types depends on target
2255 OS << "#ifdef __aarch64__\n";
2256 OS << "typedef uint8_t poly8_t;\n";
2257 OS << "typedef uint16_t poly16_t;\n";
2258 OS << "typedef uint64_t poly64_t;\n";
2259 OS << "typedef __uint128_t poly128_t;\n";
2261 OS << "typedef int8_t poly8_t;\n";
2262 OS << "typedef int16_t poly16_t;\n";
2265 // Emit Neon vector typedefs.
2266 std::string TypedefTypes(
2267 "cQcsQsiQilQlUcQUcUsQUsUiQUiUlQUlhQhfQfdQdPcQPcPsQPsPlQPl");
2268 std::vector<TypeSpec> TDTypeVec = TypeSpec::fromTypeSpecs(TypedefTypes);
2270 // Emit vector typedefs.
2271 bool InIfdef = false;
2272 for (auto &TS : TDTypeVec) {
2275 if (T.isDouble() || (T.isPoly() && T.isLong()))
2278 if (InIfdef && !IsA64) {
2282 if (!InIfdef && IsA64) {
2283 OS << "#ifdef __aarch64__\n";
2288 OS << "typedef __attribute__((neon_polyvector_type(";
2290 OS << "typedef __attribute__((neon_vector_type(";
2294 OS << utostr(T.getNumElements()) << "))) ";
2296 OS << " " << T.str() << ";\n";
2302 // Emit struct typedefs.
2304 for (unsigned NumMembers = 2; NumMembers <= 4; ++NumMembers) {
2305 for (auto &TS : TDTypeVec) {
2308 if (T.isDouble() || (T.isPoly() && T.isLong()))
2311 if (InIfdef && !IsA64) {
2315 if (!InIfdef && IsA64) {
2316 OS << "#ifdef __aarch64__\n";
2320 char M = '2' + (NumMembers - 2);
2322 OS << "typedef struct " << VT.str() << " {\n";
2323 OS << " " << T.str() << " val";
2324 OS << "[" << utostr(NumMembers) << "]";
2326 OS << VT.str() << ";\n";
2334 OS << "#define __ai static inline __attribute__((__always_inline__, "
2335 "__nodebug__))\n\n";
2337 SmallVector<Intrinsic *, 128> Defs;
2338 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2340 createIntrinsic(R, Defs);
2342 for (auto *I : Defs)
2346 Defs.begin(), Defs.end(),
2347 [](const Intrinsic *A, const Intrinsic *B) { return *A < *B; });
2349 // Only emit a def when its requirements have been met.
2350 // FIXME: This loop could be made faster, but it's fast enough for now.
2351 bool MadeProgress = true;
2352 std::string InGuard = "";
2353 while (!Defs.empty() && MadeProgress) {
2354 MadeProgress = false;
2356 for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2357 I != Defs.end(); /*No step*/) {
2358 bool DependenciesSatisfied = true;
2359 for (auto *II : (*I)->getDependencies()) {
2360 if (std::find(Defs.begin(), Defs.end(), II) != Defs.end())
2361 DependenciesSatisfied = false;
2363 if (!DependenciesSatisfied) {
2364 // Try the next one.
2369 // Emit #endif/#if pair if needed.
2370 if ((*I)->getGuard() != InGuard) {
2371 if (!InGuard.empty())
2373 InGuard = (*I)->getGuard();
2374 if (!InGuard.empty())
2375 OS << "#if " << InGuard << "\n";
2378 // Actually generate the intrinsic code.
2379 OS << (*I)->generate();
2381 MadeProgress = true;
2385 assert(Defs.empty() && "Some requirements were not satisfied!");
2386 if (!InGuard.empty())
2390 OS << "#undef __ai\n\n";
2391 OS << "#endif /* __ARM_NEON_H */\n";
2395 void EmitNeon(RecordKeeper &Records, raw_ostream &OS) {
2396 NeonEmitter(Records).run(OS);
2398 void EmitNeonSema(RecordKeeper &Records, raw_ostream &OS) {
2399 NeonEmitter(Records).runHeader(OS);
2401 void EmitNeonTest(RecordKeeper &Records, raw_ostream &OS) {
2402 llvm_unreachable("Neon test generation no longer implemented!");
2404 } // End namespace clang