contrib/llvm/lib/Target/X86/X86InstrFMA3Info.cpp

   1 //===-- X86InstrFMA3Info.cpp - X86 FMA3 Instruction Information -----------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file contains the implementation of the classes providing information
  11 // about existing X86 FMA3 opcodes, classifying and grouping them.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "X86InstrFMA3Info.h"
  16 #include "X86InstrInfo.h"
  17 #include "llvm/Support/ManagedStatic.h"
  18 #include "llvm/Support/Threading.h"
  19 #include <cassert>
  20 #include <cstdint>
  21
  22 using namespace llvm;
  23
  24 /// This flag is used in the method llvm::call_once() used below to make the
  25 /// initialization of the map 'OpcodeToGroup' thread safe.
  26 static llvm::once_flag InitGroupsOnceFlag;
  27
  28 static ManagedStatic<X86InstrFMA3Info> X86InstrFMA3InfoObj;
  29 X86InstrFMA3Info *X86InstrFMA3Info::getX86InstrFMA3Info() {
  30   return &*X86InstrFMA3InfoObj;
  31 }
  32
  33 void X86InstrFMA3Info::initRMGroup(const uint16_t *RegOpcodes,
  34                                    const uint16_t *MemOpcodes, unsigned Attr) {
  35   // Create a new instance of this class that would hold a group of FMA opcodes.
  36   X86InstrFMA3Group *G = new X86InstrFMA3Group(RegOpcodes, MemOpcodes, Attr);
  37
  38   // Add the references from indvidual opcodes to the group holding them.
  39   assert((!OpcodeToGroup[RegOpcodes[0]] && !OpcodeToGroup[RegOpcodes[1]] &&
  40           !OpcodeToGroup[RegOpcodes[2]] && !OpcodeToGroup[MemOpcodes[0]] &&
  41           !OpcodeToGroup[MemOpcodes[1]] && !OpcodeToGroup[MemOpcodes[2]]) &&
  42          "Duplication or rewrite of elements in OpcodeToGroup.");
  43   OpcodeToGroup[RegOpcodes[0]] = G;
  44   OpcodeToGroup[RegOpcodes[1]] = G;
  45   OpcodeToGroup[RegOpcodes[2]] = G;
  46   OpcodeToGroup[MemOpcodes[0]] = G;
  47   OpcodeToGroup[MemOpcodes[1]] = G;
  48   OpcodeToGroup[MemOpcodes[2]] = G;
  49 }
  50
  51 void X86InstrFMA3Info::initRGroup(const uint16_t *RegOpcodes, unsigned Attr) {
  52   // Create a new instance of this class that would hold a group of FMA opcodes.
  53   X86InstrFMA3Group *G = new X86InstrFMA3Group(RegOpcodes, nullptr, Attr);
  54
  55   // Add the references from indvidual opcodes to the group holding them.
  56   assert((!OpcodeToGroup[RegOpcodes[0]] && !OpcodeToGroup[RegOpcodes[1]] &&
  57           !OpcodeToGroup[RegOpcodes[2]]) &&
  58          "Duplication or rewrite of elements in OpcodeToGroup.");
  59   OpcodeToGroup[RegOpcodes[0]] = G;
  60   OpcodeToGroup[RegOpcodes[1]] = G;
  61   OpcodeToGroup[RegOpcodes[2]] = G;
  62 }
  63
  64 void X86InstrFMA3Info::initMGroup(const uint16_t *MemOpcodes, unsigned Attr) {
  65   // Create a new instance of this class that would hold a group of FMA opcodes.
  66   X86InstrFMA3Group *G = new X86InstrFMA3Group(nullptr, MemOpcodes, Attr);
  67
  68   // Add the references from indvidual opcodes to the group holding them.
  69   assert((!OpcodeToGroup[MemOpcodes[0]] && !OpcodeToGroup[MemOpcodes[1]] &&
  70           !OpcodeToGroup[MemOpcodes[2]]) &&
  71          "Duplication or rewrite of elements in OpcodeToGroup.");
  72   OpcodeToGroup[MemOpcodes[0]] = G;
  73   OpcodeToGroup[MemOpcodes[1]] = G;
  74   OpcodeToGroup[MemOpcodes[2]] = G;
  75 }
  76
  77 #define FMA3RM(R132, R213, R231, M132, M213, M231)                             \
  78   static const uint16_t Reg##R132[3] = {X86::R132, X86::R213, X86::R231};      \
  79   static const uint16_t Mem##R132[3] = {X86::M132, X86::M213, X86::M231};      \
  80   initRMGroup(Reg##R132, Mem##R132);
  81
  82 #define FMA3RMA(R132, R213, R231, M132, M213, M231, Attrs)                     \
  83   static const uint16_t Reg##R132[3] = {X86::R132, X86::R213, X86::R231};      \
  84   static const uint16_t Mem##R132[3] = {X86::M132, X86::M213, X86::M231};      \
  85   initRMGroup(Reg##R132, Mem##R132, (Attrs));
  86
  87 #define FMA3R(R132, R213, R231)                                                \
  88   static const uint16_t Reg##R132[3] = {X86::R132, X86::R213, X86::R231};      \
  89   initRGroup(Reg##R132);
  90
  91 #define FMA3RA(R132, R213, R231, Attrs)                                        \
  92   static const uint16_t Reg##R132[3] = {X86::R132, X86::R213, X86::R231};      \
  93   initRGroup(Reg##R132, (Attrs));
  94
  95 #define FMA3M(M132, M213, M231)                                                \
  96   static const uint16_t Mem##M132[3] = {X86::M132, X86::M213, X86::M231};      \
  97   initMGroup(Mem##M132);
  98
  99 #define FMA3MA(M132, M213, M231, Attrs)                                        \
 100   static const uint16_t Mem##M132[3] = {X86::M132, X86::M213, X86::M231};      \
 101   initMGroup(Mem##M132, (Attrs));
 102
 103 #define FMA3_AVX2_VECTOR_GROUP(Name)                                           \
 104   FMA3RM(Name##132PSr, Name##213PSr, Name##231PSr,                             \
 105          Name##132PSm, Name##213PSm, Name##231PSm);                            \
 106   FMA3RM(Name##132PDr, Name##213PDr, Name##231PDr,                             \
 107          Name##132PDm, Name##213PDm, Name##231PDm);                            \
 108   FMA3RM(Name##132PSYr, Name##213PSYr, Name##231PSYr,                          \
 109          Name##132PSYm, Name##213PSYm, Name##231PSYm);                         \
 110   FMA3RM(Name##132PDYr, Name##213PDYr, Name##231PDYr,                          \
 111          Name##132PDYm, Name##213PDYm, Name##231PDYm);
 112
 113 #define FMA3_AVX2_SCALAR_GROUP(Name)                                           \
 114   FMA3RM(Name##132SSr, Name##213SSr, Name##231SSr,                             \
 115          Name##132SSm, Name##213SSm, Name##231SSm);                            \
 116   FMA3RM(Name##132SDr, Name##213SDr, Name##231SDr,                             \
 117          Name##132SDm, Name##213SDm, Name##231SDm);                            \
 118   FMA3RMA(Name##132SSr_Int, Name##213SSr_Int, Name##231SSr_Int,                \
 119           Name##132SSm_Int, Name##213SSm_Int, Name##231SSm_Int,                \
 120           X86InstrFMA3Group::X86FMA3Intrinsic);                                \
 121   FMA3RMA(Name##132SDr_Int, Name##213SDr_Int, Name##231SDr_Int,                \
 122           Name##132SDm_Int, Name##213SDm_Int, Name##231SDm_Int,                \
 123           X86InstrFMA3Group::X86FMA3Intrinsic);
 124
 125 #define FMA3_AVX2_FULL_GROUP(Name)                                             \
 126   FMA3_AVX2_VECTOR_GROUP(Name);                                                \
 127   FMA3_AVX2_SCALAR_GROUP(Name);
 128
 129 #define FMA3_AVX512_VECTOR_GROUP(Name)                                         \
 130   FMA3RM(Name##132PSZ128r, Name##213PSZ128r, Name##231PSZ128r,                 \
 131          Name##132PSZ128m, Name##213PSZ128m, Name##231PSZ128m);                \
 132   FMA3RM(Name##132PDZ128r, Name##213PDZ128r, Name##231PDZ128r,                 \
 133          Name##132PDZ128m, Name##213PDZ128m, Name##231PDZ128m);                \
 134   FMA3RM(Name##132PSZ256r, Name##213PSZ256r, Name##231PSZ256r,                 \
 135          Name##132PSZ256m, Name##213PSZ256m, Name##231PSZ256m);                \
 136   FMA3RM(Name##132PDZ256r, Name##213PDZ256r, Name##231PDZ256r,                 \
 137          Name##132PDZ256m, Name##213PDZ256m, Name##231PDZ256m);                \
 138   FMA3RM(Name##132PSZr,    Name##213PSZr,    Name##231PSZr,                    \
 139          Name##132PSZm,    Name##213PSZm,    Name##231PSZm);                   \
 140   FMA3RM(Name##132PDZr,    Name##213PDZr,    Name##231PDZr,                    \
 141          Name##132PDZm,    Name##213PDZm,    Name##231PDZm);                   \
 142   FMA3RMA(Name##132PSZ128rk, Name##213PSZ128rk, Name##231PSZ128rk,             \
 143           Name##132PSZ128mk, Name##213PSZ128mk, Name##231PSZ128mk,             \
 144           X86InstrFMA3Group::X86FMA3KMergeMasked);                             \
 145   FMA3RMA(Name##132PDZ128rk, Name##213PDZ128rk, Name##231PDZ128rk,             \
 146           Name##132PDZ128mk, Name##213PDZ128mk, Name##231PDZ128mk,             \
 147           X86InstrFMA3Group::X86FMA3KMergeMasked);                             \
 148   FMA3RMA(Name##132PSZ256rk, Name##213PSZ256rk, Name##231PSZ256rk,             \
 149           Name##132PSZ256mk, Name##213PSZ256mk, Name##231PSZ256mk,             \
 150           X86InstrFMA3Group::X86FMA3KMergeMasked);                             \
 151   FMA3RMA(Name##132PDZ256rk, Name##213PDZ256rk, Name##231PDZ256rk,             \
 152           Name##132PDZ256mk, Name##213PDZ256mk, Name##231PDZ256mk,             \
 153           X86InstrFMA3Group::X86FMA3KMergeMasked);                             \
 154   FMA3RMA(Name##132PSZrk,    Name##213PSZrk,    Name##231PSZrk,                \
 155           Name##132PSZmk,    Name##213PSZmk,    Name##231PSZmk,                \
 156           X86InstrFMA3Group::X86FMA3KMergeMasked);                             \
 157   FMA3RMA(Name##132PDZrk,    Name##213PDZrk,    Name##231PDZrk,                \
 158           Name##132PDZmk,    Name##213PDZmk,    Name##231PDZmk,                \
 159           X86InstrFMA3Group::X86FMA3KMergeMasked);                             \
 160   FMA3RMA(Name##132PSZ128rkz, Name##213PSZ128rkz, Name##231PSZ128rkz,          \
 161           Name##132PSZ128mkz, Name##213PSZ128mkz, Name##231PSZ128mkz,          \
 162           X86InstrFMA3Group::X86FMA3KZeroMasked);                              \
 163   FMA3RMA(Name##132PDZ128rkz, Name##213PDZ128rkz, Name##231PDZ128rkz,          \
 164           Name##132PDZ128mkz, Name##213PDZ128mkz, Name##231PDZ128mkz,          \
 165           X86InstrFMA3Group::X86FMA3KZeroMasked);                              \
 166   FMA3RMA(Name##132PSZ256rkz, Name##213PSZ256rkz, Name##231PSZ256rkz,          \
 167           Name##132PSZ256mkz, Name##213PSZ256mkz, Name##231PSZ256mkz,          \
 168           X86InstrFMA3Group::X86FMA3KZeroMasked);                              \
 169   FMA3RMA(Name##132PDZ256rkz, Name##213PDZ256rkz, Name##231PDZ256rkz,          \
 170           Name##132PDZ256mkz, Name##213PDZ256mkz, Name##231PDZ256mkz,          \
 171           X86InstrFMA3Group::X86FMA3KZeroMasked);                              \
 172   FMA3RMA(Name##132PSZrkz,    Name##213PSZrkz,    Name##231PSZrkz,             \
 173           Name##132PSZmkz,    Name##213PSZmkz,    Name##231PSZmkz,             \
 174           X86InstrFMA3Group::X86FMA3KZeroMasked);                              \
 175   FMA3RMA(Name##132PDZrkz,    Name##213PDZrkz,    Name##231PDZrkz,             \
 176           Name##132PDZmkz,    Name##213PDZmkz,    Name##231PDZmkz,             \
 177           X86InstrFMA3Group::X86FMA3KZeroMasked);                              \
 178   FMA3R(Name##132PSZrb, Name##213PSZrb, Name##231PSZrb);                       \
 179   FMA3R(Name##132PDZrb, Name##213PDZrb, Name##231PDZrb);                       \
 180   FMA3RA(Name##132PSZrbk, Name##213PSZrbk, Name##231PSZrbk,                    \
 181          X86InstrFMA3Group::X86FMA3KMergeMasked);                              \
 182   FMA3RA(Name##132PDZrbk, Name##213PDZrbk, Name##231PDZrbk,                    \
 183          X86InstrFMA3Group::X86FMA3KMergeMasked);                              \
 184   FMA3RA(Name##132PSZrbkz, Name##213PSZrbkz, Name##231PSZrbkz,                 \
 185          X86InstrFMA3Group::X86FMA3KZeroMasked);                               \
 186   FMA3RA(Name##132PDZrbkz, Name##213PDZrbkz, Name##231PDZrbkz,                 \
 187          X86InstrFMA3Group::X86FMA3KZeroMasked);                               \
 188   FMA3M(Name##132PSZ128mb, Name##213PSZ128mb, Name##231PSZ128mb);              \
 189   FMA3M(Name##132PDZ128mb, Name##213PDZ128mb, Name##231PDZ128mb);              \
 190   FMA3M(Name##132PSZ256mb, Name##213PSZ256mb, Name##231PSZ256mb);              \
 191   FMA3M(Name##132PDZ256mb, Name##213PDZ256mb, Name##231PDZ256mb);              \
 192   FMA3M(Name##132PSZmb, Name##213PSZmb, Name##231PSZmb);                       \
 193   FMA3M(Name##132PDZmb, Name##213PDZmb, Name##231PDZmb);                       \
 194   FMA3MA(Name##132PSZ128mbk, Name##213PSZ128mbk, Name##231PSZ128mbk,           \
 195          X86InstrFMA3Group::X86FMA3KMergeMasked);                              \
 196   FMA3MA(Name##132PDZ128mbk, Name##213PDZ128mbk, Name##231PDZ128mbk,           \
 197          X86InstrFMA3Group::X86FMA3KMergeMasked);                              \
 198   FMA3MA(Name##132PSZ256mbk, Name##213PSZ256mbk, Name##231PSZ256mbk,           \
 199          X86InstrFMA3Group::X86FMA3KMergeMasked);                              \
 200   FMA3MA(Name##132PDZ256mbk, Name##213PDZ256mbk, Name##231PDZ256mbk,           \
 201          X86InstrFMA3Group::X86FMA3KMergeMasked);                              \
 202   FMA3MA(Name##132PSZmbk,    Name##213PSZmbk,    Name##231PSZmbk,              \
 203          X86InstrFMA3Group::X86FMA3KMergeMasked);                              \
 204   FMA3MA(Name##132PDZmbk,    Name##213PDZmbk,    Name##231PDZmbk,              \
 205          X86InstrFMA3Group::X86FMA3KMergeMasked);                              \
 206   FMA3MA(Name##132PSZ128mbkz, Name##213PSZ128mbkz, Name##231PSZ128mbkz,        \
 207          X86InstrFMA3Group::X86FMA3KZeroMasked);                               \
 208   FMA3MA(Name##132PDZ128mbkz, Name##213PDZ128mbkz, Name##231PDZ128mbkz,        \
 209          X86InstrFMA3Group::X86FMA3KZeroMasked);                               \
 210   FMA3MA(Name##132PSZ256mbkz, Name##213PSZ256mbkz, Name##231PSZ256mbkz,        \
 211          X86InstrFMA3Group::X86FMA3KZeroMasked);                               \
 212   FMA3MA(Name##132PDZ256mbkz, Name##213PDZ256mbkz, Name##231PDZ256mbkz,        \
 213          X86InstrFMA3Group::X86FMA3KZeroMasked);                               \
 214   FMA3MA(Name##132PSZmbkz, Name##213PSZmbkz, Name##231PSZmbkz,                 \
 215          X86InstrFMA3Group::X86FMA3KZeroMasked);                               \
 216   FMA3MA(Name##132PDZmbkz, Name##213PDZmbkz, Name##231PDZmbkz,                 \
 217          X86InstrFMA3Group::X86FMA3KZeroMasked);
 218
 219 #define FMA3_AVX512_SCALAR_GROUP(Name)                                         \
 220   FMA3RM(Name##132SSZr,      Name##213SSZr,     Name##231SSZr,                 \
 221          Name##132SSZm,      Name##213SSZm,     Name##231SSZm);                \
 222   FMA3RM(Name##132SDZr,      Name##213SDZr,     Name##231SDZr,                 \
 223          Name##132SDZm,      Name##213SDZm,     Name##231SDZm);                \
 224   FMA3RMA(Name##132SSZr_Int, Name##213SSZr_Int, Name##231SSZr_Int,             \
 225           Name##132SSZm_Int, Name##213SSZm_Int, Name##231SSZm_Int,             \
 226           X86InstrFMA3Group::X86FMA3Intrinsic);                                \
 227   FMA3RMA(Name##132SDZr_Int, Name##213SDZr_Int, Name##231SDZr_Int,             \
 228           Name##132SDZm_Int, Name##213SDZm_Int, Name##231SDZm_Int,             \
 229           X86InstrFMA3Group::X86FMA3Intrinsic);                                \
 230   FMA3RMA(Name##132SSZr_Intk, Name##213SSZr_Intk, Name##231SSZr_Intk,          \
 231           Name##132SSZm_Intk, Name##213SSZm_Intk, Name##231SSZm_Intk,          \
 232           X86InstrFMA3Group::X86FMA3Intrinsic |                                \
 233               X86InstrFMA3Group::X86FMA3KMergeMasked);                         \
 234   FMA3RMA(Name##132SDZr_Intk, Name##213SDZr_Intk, Name##231SDZr_Intk,          \
 235           Name##132SDZm_Intk, Name##213SDZm_Intk, Name##231SDZm_Intk,          \
 236           X86InstrFMA3Group::X86FMA3Intrinsic |                                \
 237               X86InstrFMA3Group::X86FMA3KMergeMasked);                         \
 238   FMA3RMA(Name##132SSZr_Intkz, Name##213SSZr_Intkz, Name##231SSZr_Intkz,       \
 239           Name##132SSZm_Intkz, Name##213SSZm_Intkz, Name##231SSZm_Intkz,       \
 240           X86InstrFMA3Group::X86FMA3Intrinsic |                                \
 241               X86InstrFMA3Group::X86FMA3KZeroMasked);                          \
 242   FMA3RMA(Name##132SDZr_Intkz, Name##213SDZr_Intkz, Name##231SDZr_Intkz,       \
 243           Name##132SDZm_Intkz, Name##213SDZm_Intkz, Name##231SDZm_Intkz,       \
 244           X86InstrFMA3Group::X86FMA3Intrinsic |                                \
 245               X86InstrFMA3Group::X86FMA3KZeroMasked);                          \
 246   FMA3RA(Name##132SSZrb_Int, Name##213SSZrb_Int, Name##231SSZrb_Int,           \
 247          X86InstrFMA3Group::X86FMA3Intrinsic);                                 \
 248   FMA3RA(Name##132SDZrb_Int, Name##213SDZrb_Int, Name##231SDZrb_Int,           \
 249          X86InstrFMA3Group::X86FMA3Intrinsic);                                 \
 250   FMA3RA(Name##132SSZrb_Intk, Name##213SSZrb_Intk, Name##231SSZrb_Intk,        \
 251          X86InstrFMA3Group::X86FMA3Intrinsic |                                 \
 252              X86InstrFMA3Group::X86FMA3KMergeMasked);                          \
 253   FMA3RA(Name##132SDZrb_Intk, Name##213SDZrb_Intk, Name##231SDZrb_Intk,        \
 254          X86InstrFMA3Group::X86FMA3Intrinsic |                                 \
 255              X86InstrFMA3Group::X86FMA3KMergeMasked);                          \
 256   FMA3RA(Name##132SSZrb_Intkz, Name##213SSZrb_Intkz, Name##231SSZrb_Intkz,     \
 257          X86InstrFMA3Group::X86FMA3Intrinsic |                                 \
 258              X86InstrFMA3Group::X86FMA3KZeroMasked);                           \
 259   FMA3RA(Name##132SDZrb_Intkz, Name##213SDZrb_Intkz, Name##231SDZrb_Intkz,     \
 260          X86InstrFMA3Group::X86FMA3Intrinsic |                                 \
 261              X86InstrFMA3Group::X86FMA3KZeroMasked);
 262
 263 #define FMA3_AVX512_FULL_GROUP(Name)                                           \
 264   FMA3_AVX512_VECTOR_GROUP(Name);                                              \
 265   FMA3_AVX512_SCALAR_GROUP(Name);
 266
 267 void X86InstrFMA3Info::initGroupsOnceImpl() {
 268   FMA3_AVX2_FULL_GROUP(VFMADD);
 269   FMA3_AVX2_FULL_GROUP(VFMSUB);
 270   FMA3_AVX2_FULL_GROUP(VFNMADD);
 271   FMA3_AVX2_FULL_GROUP(VFNMSUB);
 272
 273   FMA3_AVX2_VECTOR_GROUP(VFMADDSUB);
 274   FMA3_AVX2_VECTOR_GROUP(VFMSUBADD);
 275
 276   FMA3_AVX512_FULL_GROUP(VFMADD);
 277   FMA3_AVX512_FULL_GROUP(VFMSUB);
 278   FMA3_AVX512_FULL_GROUP(VFNMADD);
 279   FMA3_AVX512_FULL_GROUP(VFNMSUB);
 280
 281   FMA3_AVX512_VECTOR_GROUP(VFMADDSUB);
 282   FMA3_AVX512_VECTOR_GROUP(VFMSUBADD);
 283 }
 284
 285 void X86InstrFMA3Info::initGroupsOnce() {
 286   llvm::call_once(InitGroupsOnceFlag,
 287                   []() { getX86InstrFMA3Info()->initGroupsOnceImpl(); });
 288 }