1 //===-- SystemZTargetTransformInfo.cpp - SystemZ-specific TTI -------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements a TargetTransformInfo analysis pass specific to the
11 // SystemZ target machine. It uses the target's detailed information to provide
12 // more precise answers to certain TTI queries, while letting the target
13 // independent and default TTI implementations handle the rest.
15 //===----------------------------------------------------------------------===//
17 #include "SystemZTargetTransformInfo.h"
18 #include "llvm/Analysis/TargetTransformInfo.h"
19 #include "llvm/CodeGen/BasicTTIImpl.h"
20 #include "llvm/IR/IntrinsicInst.h"
21 #include "llvm/Support/Debug.h"
22 #include "llvm/Target/CostTable.h"
23 #include "llvm/Target/TargetLowering.h"
26 #define DEBUG_TYPE "systemztti"
28 //===----------------------------------------------------------------------===//
30 // SystemZ cost model.
32 //===----------------------------------------------------------------------===//
34 int SystemZTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty) {
35 assert(Ty->isIntegerTy());
37 unsigned BitSize = Ty->getPrimitiveSizeInBits();
38 // There is no cost model for constants with a bit size of 0. Return TCC_Free
39 // here, so that constant hoisting will ignore this constant.
42 // No cost model for operations on integers larger than 64 bit implemented yet.
49 if (Imm.getBitWidth() <= 64) {
50 // Constants loaded via lgfi.
51 if (isInt<32>(Imm.getSExtValue()))
52 return TTI::TCC_Basic;
53 // Constants loaded via llilf.
54 if (isUInt<32>(Imm.getZExtValue()))
55 return TTI::TCC_Basic;
56 // Constants loaded via llihf:
57 if ((Imm.getZExtValue() & 0xffffffff) == 0)
58 return TTI::TCC_Basic;
60 return 2 * TTI::TCC_Basic;
63 return 4 * TTI::TCC_Basic;
66 int SystemZTTIImpl::getIntImmCost(unsigned Opcode, unsigned Idx,
67 const APInt &Imm, Type *Ty) {
68 assert(Ty->isIntegerTy());
70 unsigned BitSize = Ty->getPrimitiveSizeInBits();
71 // There is no cost model for constants with a bit size of 0. Return TCC_Free
72 // here, so that constant hoisting will ignore this constant.
75 // No cost model for operations on integers larger than 64 bit implemented yet.
82 case Instruction::GetElementPtr:
83 // Always hoist the base address of a GetElementPtr. This prevents the
84 // creation of new constants for every base constant that gets constant
85 // folded with the offset.
87 return 2 * TTI::TCC_Basic;
89 case Instruction::Store:
90 if (Idx == 0 && Imm.getBitWidth() <= 64) {
91 // Any 8-bit immediate store can by implemented via mvi.
94 // 16-bit immediate values can be stored via mvhhi/mvhi/mvghi.
95 if (isInt<16>(Imm.getSExtValue()))
99 case Instruction::ICmp:
100 if (Idx == 1 && Imm.getBitWidth() <= 64) {
101 // Comparisons against signed 32-bit immediates implemented via cgfi.
102 if (isInt<32>(Imm.getSExtValue()))
103 return TTI::TCC_Free;
104 // Comparisons against unsigned 32-bit immediates implemented via clgfi.
105 if (isUInt<32>(Imm.getZExtValue()))
106 return TTI::TCC_Free;
109 case Instruction::Add:
110 case Instruction::Sub:
111 if (Idx == 1 && Imm.getBitWidth() <= 64) {
112 // We use algfi/slgfi to add/subtract 32-bit unsigned immediates.
113 if (isUInt<32>(Imm.getZExtValue()))
114 return TTI::TCC_Free;
115 // Or their negation, by swapping addition vs. subtraction.
116 if (isUInt<32>(-Imm.getSExtValue()))
117 return TTI::TCC_Free;
120 case Instruction::Mul:
121 if (Idx == 1 && Imm.getBitWidth() <= 64) {
122 // We use msgfi to multiply by 32-bit signed immediates.
123 if (isInt<32>(Imm.getSExtValue()))
124 return TTI::TCC_Free;
127 case Instruction::Or:
128 case Instruction::Xor:
129 if (Idx == 1 && Imm.getBitWidth() <= 64) {
130 // Masks supported by oilf/xilf.
131 if (isUInt<32>(Imm.getZExtValue()))
132 return TTI::TCC_Free;
133 // Masks supported by oihf/xihf.
134 if ((Imm.getZExtValue() & 0xffffffff) == 0)
135 return TTI::TCC_Free;
138 case Instruction::And:
139 if (Idx == 1 && Imm.getBitWidth() <= 64) {
140 // Any 32-bit AND operation can by implemented via nilf.
142 return TTI::TCC_Free;
143 // 64-bit masks supported by nilf.
144 if (isUInt<32>(~Imm.getZExtValue()))
145 return TTI::TCC_Free;
146 // 64-bit masks supported by nilh.
147 if ((Imm.getZExtValue() & 0xffffffff) == 0xffffffff)
148 return TTI::TCC_Free;
149 // Some 64-bit AND operations can be implemented via risbg.
150 const SystemZInstrInfo *TII = ST->getInstrInfo();
152 if (TII->isRxSBGMask(Imm.getZExtValue(), BitSize, Start, End))
153 return TTI::TCC_Free;
156 case Instruction::Shl:
157 case Instruction::LShr:
158 case Instruction::AShr:
159 // Always return TCC_Free for the shift value of a shift instruction.
161 return TTI::TCC_Free;
163 case Instruction::UDiv:
164 case Instruction::SDiv:
165 case Instruction::URem:
166 case Instruction::SRem:
167 case Instruction::Trunc:
168 case Instruction::ZExt:
169 case Instruction::SExt:
170 case Instruction::IntToPtr:
171 case Instruction::PtrToInt:
172 case Instruction::BitCast:
173 case Instruction::PHI:
174 case Instruction::Call:
175 case Instruction::Select:
176 case Instruction::Ret:
177 case Instruction::Load:
181 return SystemZTTIImpl::getIntImmCost(Imm, Ty);
184 int SystemZTTIImpl::getIntImmCost(Intrinsic::ID IID, unsigned Idx,
185 const APInt &Imm, Type *Ty) {
186 assert(Ty->isIntegerTy());
188 unsigned BitSize = Ty->getPrimitiveSizeInBits();
189 // There is no cost model for constants with a bit size of 0. Return TCC_Free
190 // here, so that constant hoisting will ignore this constant.
192 return TTI::TCC_Free;
193 // No cost model for operations on integers larger than 64 bit implemented yet.
195 return TTI::TCC_Free;
199 return TTI::TCC_Free;
200 case Intrinsic::sadd_with_overflow:
201 case Intrinsic::uadd_with_overflow:
202 case Intrinsic::ssub_with_overflow:
203 case Intrinsic::usub_with_overflow:
204 // These get expanded to include a normal addition/subtraction.
205 if (Idx == 1 && Imm.getBitWidth() <= 64) {
206 if (isUInt<32>(Imm.getZExtValue()))
207 return TTI::TCC_Free;
208 if (isUInt<32>(-Imm.getSExtValue()))
209 return TTI::TCC_Free;
212 case Intrinsic::smul_with_overflow:
213 case Intrinsic::umul_with_overflow:
214 // These get expanded to include a normal multiplication.
215 if (Idx == 1 && Imm.getBitWidth() <= 64) {
216 if (isInt<32>(Imm.getSExtValue()))
217 return TTI::TCC_Free;
220 case Intrinsic::experimental_stackmap:
221 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
222 return TTI::TCC_Free;
224 case Intrinsic::experimental_patchpoint_void:
225 case Intrinsic::experimental_patchpoint_i64:
226 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
227 return TTI::TCC_Free;
230 return SystemZTTIImpl::getIntImmCost(Imm, Ty);
233 TargetTransformInfo::PopcntSupportKind
234 SystemZTTIImpl::getPopcntSupport(unsigned TyWidth) {
235 assert(isPowerOf2_32(TyWidth) && "Type width must be power of 2");
236 if (ST->hasPopulationCount() && TyWidth <= 64)
237 return TTI::PSK_FastHardware;
238 return TTI::PSK_Software;
241 void SystemZTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
242 TTI::UnrollingPreferences &UP) {
243 // Find out if L contains a call, what the machine instruction count
244 // estimate is, and how many stores there are.
245 bool HasCall = false;
246 unsigned NumStores = 0;
247 for (auto &BB : L->blocks())
248 for (auto &I : *BB) {
249 if (isa<CallInst>(&I) || isa<InvokeInst>(&I)) {
250 ImmutableCallSite CS(&I);
251 if (const Function *F = CS.getCalledFunction()) {
252 if (isLoweredToCall(F))
254 if (F->getIntrinsicID() == Intrinsic::memcpy ||
255 F->getIntrinsicID() == Intrinsic::memset)
257 } else { // indirect call.
261 if (isa<StoreInst>(&I)) {
262 Type *MemAccessTy = I.getOperand(0)->getType();
263 NumStores += getMemoryOpCost(Instruction::Store, MemAccessTy, 0, 0);
267 // The z13 processor will run out of store tags if too many stores
268 // are fed into it too quickly. Therefore make sure there are not
269 // too many stores in the resulting unrolled loop.
270 unsigned const Max = (NumStores ? (12 / NumStores) : UINT_MAX);
273 // Only allow full unrolling if loop has any calls.
274 UP.FullUnrollMaxCount = Max;
280 if (UP.MaxCount <= 1)
283 // Allow partial and runtime trip count unrolling.
284 UP.Partial = UP.Runtime = true;
286 UP.PartialThreshold = 75;
287 UP.DefaultUnrollRuntimeCount = 4;
289 // Allow expensive instructions in the pre-header of the loop.
290 UP.AllowExpensiveTripCount = true;
295 unsigned SystemZTTIImpl::getNumberOfRegisters(bool Vector) {
297 // Discount the stack pointer. Also leave out %r0, since it can't
298 // be used in an address.
305 unsigned SystemZTTIImpl::getRegisterBitWidth(bool Vector) const {
313 int SystemZTTIImpl::getArithmeticInstrCost(
314 unsigned Opcode, Type *Ty,
315 TTI::OperandValueKind Op1Info, TTI::OperandValueKind Op2Info,
316 TTI::OperandValueProperties Opd1PropInfo,
317 TTI::OperandValueProperties Opd2PropInfo,
318 ArrayRef<const Value *> Args) {
320 // TODO: return a good value for BB-VECTORIZER that includes the
321 // immediate loads, which we do not want to count for the loop
322 // vectorizer, since they are hopefully hoisted out of the loop. This
323 // would require a new parameter 'InLoop', but not sure if constant
324 // args are common enough to motivate this.
326 unsigned ScalarBits = Ty->getScalarSizeInBits();
328 // Div with a constant which is a power of 2 will be converted by
329 // DAGCombiner to use shifts. With vector shift-element instructions, a
330 // vector sdiv costs about as much as a scalar one.
331 const unsigned SDivCostEstimate = 4;
332 bool SDivPow2 = false;
333 bool UDivPow2 = false;
334 if ((Opcode == Instruction::SDiv || Opcode == Instruction::UDiv) &&
336 const ConstantInt *CI = nullptr;
337 if (const Constant *C = dyn_cast<Constant>(Args[1])) {
338 if (C->getType()->isVectorTy())
339 CI = dyn_cast_or_null<const ConstantInt>(C->getSplatValue());
341 CI = dyn_cast<const ConstantInt>(C);
344 (CI->getValue().isPowerOf2() || (-CI->getValue()).isPowerOf2())) {
345 if (Opcode == Instruction::SDiv)
352 if (Ty->isVectorTy()) {
353 assert (ST->hasVector() && "getArithmeticInstrCost() called with vector type.");
354 unsigned VF = Ty->getVectorNumElements();
355 unsigned NumVectors = getNumberOfParts(Ty);
357 // These vector operations are custom handled, but are still supported
358 // with one instruction per vector, regardless of element size.
359 if (Opcode == Instruction::Shl || Opcode == Instruction::LShr ||
360 Opcode == Instruction::AShr || UDivPow2) {
365 return (NumVectors * SDivCostEstimate);
367 // These FP operations are supported with a single vector instruction for
368 // double (base implementation assumes float generally costs 2). For
369 // FP128, the scalar cost is 1, and there is no overhead since the values
370 // are already in scalar registers.
371 if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
372 Opcode == Instruction::FMul || Opcode == Instruction::FDiv) {
373 switch (ScalarBits) {
375 // Return the cost of multiple scalar invocation plus the cost of
376 // inserting and extracting the values.
377 unsigned ScalarCost = getArithmeticInstrCost(Opcode, Ty->getScalarType());
378 unsigned Cost = (VF * ScalarCost) + getScalarizationOverhead(Ty, Args);
379 // FIXME: VF 2 for these FP operations are currently just as
380 // expensive as for VF 4.
393 // There is no native support for FRem.
394 if (Opcode == Instruction::FRem) {
395 unsigned Cost = (VF * LIBCALL_COST) + getScalarizationOverhead(Ty, Args);
396 // FIXME: VF 2 for float is currently just as expensive as for VF 4.
397 if (VF == 2 && ScalarBits == 32)
403 // These FP operations are supported with a dedicated instruction for
404 // float, double and fp128 (base implementation assumes float generally
406 if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
407 Opcode == Instruction::FMul || Opcode == Instruction::FDiv)
410 // There is no native support for FRem.
411 if (Opcode == Instruction::FRem)
414 if (Opcode == Instruction::LShr || Opcode == Instruction::AShr)
415 return (ScalarBits >= 32 ? 1 : 2 /*ext*/);
417 // Or requires one instruction, although it has custom handling for i64.
418 if (Opcode == Instruction::Or)
421 if (Opcode == Instruction::Xor && ScalarBits == 1)
422 // 2 * ipm sequences ; xor ; shift ; compare
428 return SDivCostEstimate;
430 // An extra extension for narrow types is needed.
431 if ((Opcode == Instruction::SDiv || Opcode == Instruction::SRem))
432 // sext of op(s) for narrow types
433 return (ScalarBits < 32 ? 4 : (ScalarBits == 32 ? 2 : 1));
435 if (Opcode == Instruction::UDiv || Opcode == Instruction::URem)
436 // Clearing of low 64 bit reg + sext of op(s) for narrow types + dl[g]r
437 return (ScalarBits < 32 ? 4 : 2);
440 // Fallback to the default implementation.
441 return BaseT::getArithmeticInstrCost(Opcode, Ty, Op1Info, Op2Info,
442 Opd1PropInfo, Opd2PropInfo, Args);
446 int SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
448 assert (Tp->isVectorTy());
449 assert (ST->hasVector() && "getShuffleCost() called.");
450 unsigned NumVectors = getNumberOfParts(Tp);
452 // TODO: Since fp32 is expanded, the shuffle cost should always be 0.
454 // FP128 values are always in scalar registers, so there is no work
455 // involved with a shuffle, except for broadcast. In that case register
456 // moves are done with a single instruction per element.
457 if (Tp->getScalarType()->isFP128Ty())
458 return (Kind == TargetTransformInfo::SK_Broadcast ? NumVectors - 1 : 0);
461 case TargetTransformInfo::SK_ExtractSubvector:
462 // ExtractSubvector Index indicates start offset.
464 // Extracting a subvector from first index is a noop.
465 return (Index == 0 ? 0 : NumVectors);
467 case TargetTransformInfo::SK_Broadcast:
468 // Loop vectorizer calls here to figure out the extra cost of
469 // broadcasting a loaded value to all elements of a vector. Since vlrep
470 // loads and replicates with a single instruction, adjust the returned
472 return NumVectors - 1;
476 // SystemZ supports single instruction permutation / replication.
480 return BaseT::getShuffleCost(Kind, Tp, Index, SubTp);
483 // Return the log2 difference of the element sizes of the two vector types.
484 static unsigned getElSizeLog2Diff(Type *Ty0, Type *Ty1) {
485 unsigned Bits0 = Ty0->getScalarSizeInBits();
486 unsigned Bits1 = Ty1->getScalarSizeInBits();
489 return (Log2_32(Bits1) - Log2_32(Bits0));
491 return (Log2_32(Bits0) - Log2_32(Bits1));
494 // Return the number of instructions needed to truncate SrcTy to DstTy.
495 unsigned SystemZTTIImpl::
496 getVectorTruncCost(Type *SrcTy, Type *DstTy) {
497 assert (SrcTy->isVectorTy() && DstTy->isVectorTy());
498 assert (SrcTy->getPrimitiveSizeInBits() > DstTy->getPrimitiveSizeInBits() &&
499 "Packing must reduce size of vector type.");
500 assert (SrcTy->getVectorNumElements() == DstTy->getVectorNumElements() &&
501 "Packing should not change number of elements.");
503 // TODO: Since fp32 is expanded, the extract cost should always be 0.
505 unsigned NumParts = getNumberOfParts(SrcTy);
507 // Up to 2 vector registers can be truncated efficiently with pack or
508 // permute. The latter requires an immediate mask to be loaded, which
509 // typically gets hoisted out of a loop. TODO: return a good value for
510 // BB-VECTORIZER that includes the immediate loads, which we do not want
511 // to count for the loop vectorizer.
515 unsigned Log2Diff = getElSizeLog2Diff(SrcTy, DstTy);
516 unsigned VF = SrcTy->getVectorNumElements();
517 for (unsigned P = 0; P < Log2Diff; ++P) {
523 // Currently, a general mix of permutes and pack instructions is output by
524 // isel, which follow the cost computation above except for this case which
525 // is one instruction less:
526 if (VF == 8 && SrcTy->getScalarSizeInBits() == 64 &&
527 DstTy->getScalarSizeInBits() == 8)
533 // Return the cost of converting a vector bitmask produced by a compare
534 // (SrcTy), to the type of the select or extend instruction (DstTy).
535 unsigned SystemZTTIImpl::
536 getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy) {
537 assert (SrcTy->isVectorTy() && DstTy->isVectorTy() &&
538 "Should only be called with vector types.");
540 unsigned PackCost = 0;
541 unsigned SrcScalarBits = SrcTy->getScalarSizeInBits();
542 unsigned DstScalarBits = DstTy->getScalarSizeInBits();
543 unsigned Log2Diff = getElSizeLog2Diff(SrcTy, DstTy);
544 if (SrcScalarBits > DstScalarBits)
545 // The bitmask will be truncated.
546 PackCost = getVectorTruncCost(SrcTy, DstTy);
547 else if (SrcScalarBits < DstScalarBits) {
548 unsigned DstNumParts = getNumberOfParts(DstTy);
549 // Each vector select needs its part of the bitmask unpacked.
550 PackCost = Log2Diff * DstNumParts;
551 // Extra cost for moving part of mask before unpacking.
552 PackCost += DstNumParts - 1;
558 // Return the type of the compared operands. This is needed to compute the
559 // cost for a Select / ZExt or SExt instruction.
560 static Type *getCmpOpsType(const Instruction *I, unsigned VF = 1) {
561 Type *OpTy = nullptr;
562 if (CmpInst *CI = dyn_cast<CmpInst>(I->getOperand(0)))
563 OpTy = CI->getOperand(0)->getType();
564 else if (Instruction *LogicI = dyn_cast<Instruction>(I->getOperand(0)))
565 if (LogicI->getNumOperands() == 2)
566 if (CmpInst *CI0 = dyn_cast<CmpInst>(LogicI->getOperand(0)))
567 if (isa<CmpInst>(LogicI->getOperand(1)))
568 OpTy = CI0->getOperand(0)->getType();
570 if (OpTy != nullptr) {
572 assert (!OpTy->isVectorTy() && "Expected scalar type");
575 // Return the potentially vectorized type based on 'I' and 'VF'. 'I' may
576 // be either scalar or already vectorized with a same or lesser VF.
577 Type *ElTy = OpTy->getScalarType();
578 return VectorType::get(ElTy, VF);
584 int SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
585 const Instruction *I) {
586 unsigned DstScalarBits = Dst->getScalarSizeInBits();
587 unsigned SrcScalarBits = Src->getScalarSizeInBits();
589 if (Src->isVectorTy()) {
590 assert (ST->hasVector() && "getCastInstrCost() called with vector type.");
591 assert (Dst->isVectorTy());
592 unsigned VF = Src->getVectorNumElements();
593 unsigned NumDstVectors = getNumberOfParts(Dst);
594 unsigned NumSrcVectors = getNumberOfParts(Src);
596 if (Opcode == Instruction::Trunc) {
597 if (Src->getScalarSizeInBits() == Dst->getScalarSizeInBits())
598 return 0; // Check for NOOP conversions.
599 return getVectorTruncCost(Src, Dst);
602 if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt) {
603 if (SrcScalarBits >= 8) {
604 // ZExt/SExt will be handled with one unpack per doubling of width.
605 unsigned NumUnpacks = getElSizeLog2Diff(Src, Dst);
607 // For types that spans multiple vector registers, some additional
608 // instructions are used to setup the unpacking.
609 unsigned NumSrcVectorOps =
610 (NumUnpacks > 1 ? (NumDstVectors - NumSrcVectors)
611 : (NumDstVectors / 2));
613 return (NumUnpacks * NumDstVectors) + NumSrcVectorOps;
615 else if (SrcScalarBits == 1) {
616 // This should be extension of a compare i1 result.
617 // If we know what the widths of the compared operands, get the
618 // cost of converting it to Dst. Otherwise assume same widths.
620 Type *CmpOpTy = ((I != nullptr) ? getCmpOpsType(I, VF) : nullptr);
621 if (CmpOpTy != nullptr)
622 Cost = getVectorBitmaskConversionCost(CmpOpTy, Dst);
623 if (Opcode == Instruction::ZExt)
624 // One 'vn' per dst vector with an immediate mask.
625 Cost += NumDstVectors;
630 if (Opcode == Instruction::SIToFP || Opcode == Instruction::UIToFP ||
631 Opcode == Instruction::FPToSI || Opcode == Instruction::FPToUI) {
632 // TODO: Fix base implementation which could simplify things a bit here
633 // (seems to miss on differentiating on scalar/vector types).
635 // Only 64 bit vector conversions are natively supported.
636 if (SrcScalarBits == 64 && DstScalarBits == 64)
637 return NumDstVectors;
639 // Return the cost of multiple scalar invocation plus the cost of
640 // inserting and extracting the values. Base implementation does not
641 // realize float->int gets scalarized.
642 unsigned ScalarCost = getCastInstrCost(Opcode, Dst->getScalarType(),
643 Src->getScalarType());
644 unsigned TotCost = VF * ScalarCost;
645 bool NeedsInserts = true, NeedsExtracts = true;
646 // FP128 registers do not get inserted or extracted.
647 if (DstScalarBits == 128 &&
648 (Opcode == Instruction::SIToFP || Opcode == Instruction::UIToFP))
649 NeedsInserts = false;
650 if (SrcScalarBits == 128 &&
651 (Opcode == Instruction::FPToSI || Opcode == Instruction::FPToUI))
652 NeedsExtracts = false;
654 TotCost += getScalarizationOverhead(Dst, NeedsInserts, NeedsExtracts);
656 // FIXME: VF 2 for float<->i32 is currently just as expensive as for VF 4.
657 if (VF == 2 && SrcScalarBits == 32 && DstScalarBits == 32)
663 if (Opcode == Instruction::FPTrunc) {
664 if (SrcScalarBits == 128) // fp128 -> double/float + inserts of elements.
665 return VF /*ldxbr/lexbr*/ + getScalarizationOverhead(Dst, true, false);
666 else // double -> float
667 return VF / 2 /*vledb*/ + std::max(1U, VF / 4 /*vperm*/);
670 if (Opcode == Instruction::FPExt) {
671 if (SrcScalarBits == 32 && DstScalarBits == 64) {
672 // float -> double is very rare and currently unoptimized. Instead of
673 // using vldeb, which can do two at a time, all conversions are
677 // -> fp128. VF * lxdb/lxeb + extraction of elements.
678 return VF + getScalarizationOverhead(Src, false, true);
682 assert (!Dst->isVectorTy());
684 if (Opcode == Instruction::SIToFP || Opcode == Instruction::UIToFP)
685 return (SrcScalarBits >= 32 ? 1 : 2 /*i8/i16 extend*/);
687 if ((Opcode == Instruction::ZExt || Opcode == Instruction::SExt) &&
688 Src->isIntegerTy(1)) {
689 // This should be extension of a compare i1 result, which is done with
690 // ipm and a varying sequence of instructions.
692 if (Opcode == Instruction::SExt)
693 Cost = (DstScalarBits < 64 ? 3 : 4);
694 if (Opcode == Instruction::ZExt)
696 Type *CmpOpTy = ((I != nullptr) ? getCmpOpsType(I) : nullptr);
697 if (CmpOpTy != nullptr && CmpOpTy->isFloatingPointTy())
698 // If operands of an fp-type was compared, this costs +1.
705 return BaseT::getCastInstrCost(Opcode, Dst, Src, I);
708 int SystemZTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
709 const Instruction *I) {
710 if (ValTy->isVectorTy()) {
711 assert (ST->hasVector() && "getCmpSelInstrCost() called with vector type.");
712 unsigned VF = ValTy->getVectorNumElements();
714 // Called with a compare instruction.
715 if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) {
716 unsigned PredicateExtraCost = 0;
718 // Some predicates cost one or two extra instructions.
719 switch (dyn_cast<CmpInst>(I)->getPredicate()) {
720 case CmpInst::Predicate::ICMP_NE:
721 case CmpInst::Predicate::ICMP_UGE:
722 case CmpInst::Predicate::ICMP_ULE:
723 case CmpInst::Predicate::ICMP_SGE:
724 case CmpInst::Predicate::ICMP_SLE:
725 PredicateExtraCost = 1;
727 case CmpInst::Predicate::FCMP_ONE:
728 case CmpInst::Predicate::FCMP_ORD:
729 case CmpInst::Predicate::FCMP_UEQ:
730 case CmpInst::Predicate::FCMP_UNO:
731 PredicateExtraCost = 2;
738 // Float is handled with 2*vmr[lh]f + 2*vldeb + vfchdb for each pair of
739 // floats. FIXME: <2 x float> generates same code as <4 x float>.
740 unsigned CmpCostPerVector = (ValTy->getScalarType()->isFloatTy() ? 10 : 1);
741 unsigned NumVecs_cmp = getNumberOfParts(ValTy);
743 unsigned Cost = (NumVecs_cmp * (CmpCostPerVector + PredicateExtraCost));
746 else { // Called with a select instruction.
747 assert (Opcode == Instruction::Select);
749 // We can figure out the extra cost of packing / unpacking if the
750 // instruction was passed and the compare instruction is found.
751 unsigned PackCost = 0;
752 Type *CmpOpTy = ((I != nullptr) ? getCmpOpsType(I, VF) : nullptr);
753 if (CmpOpTy != nullptr)
755 getVectorBitmaskConversionCost(CmpOpTy, ValTy);
757 return getNumberOfParts(ValTy) /*vsel*/ + PackCost;
762 case Instruction::ICmp: {
764 if (ValTy->isIntegerTy() && ValTy->getScalarSizeInBits() <= 16)
765 Cost += 2; // extend both operands
768 case Instruction::Select:
769 if (ValTy->isFloatingPointTy())
770 return 4; // No load on condition for FP, so this costs a conditional jump.
771 return 1; // Load On Condition.
775 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, nullptr);
779 getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
780 // vlvgp will insert two grs into a vector register, so only count half the
781 // number of instructions.
782 if (Opcode == Instruction::InsertElement &&
783 Val->getScalarType()->isIntegerTy(64))
784 return ((Index % 2 == 0) ? 1 : 0);
786 if (Opcode == Instruction::ExtractElement) {
787 int Cost = ((Val->getScalarSizeInBits() == 1) ? 2 /*+test-under-mask*/ : 1);
789 // Give a slight penalty for moving out of vector pipeline to FXU unit.
790 if (Index == 0 && Val->getScalarType()->isIntegerTy())
796 return BaseT::getVectorInstrCost(Opcode, Val, Index);
799 int SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
800 unsigned Alignment, unsigned AddressSpace,
801 const Instruction *I) {
802 assert(!Src->isVoidTy() && "Invalid type");
804 if (!Src->isVectorTy() && Opcode == Instruction::Load &&
805 I != nullptr && I->hasOneUse()) {
806 const Instruction *UserI = cast<Instruction>(*I->user_begin());
807 unsigned Bits = Src->getScalarSizeInBits();
808 bool FoldsLoad = false;
809 switch (UserI->getOpcode()) {
810 case Instruction::ICmp:
811 case Instruction::Add:
812 case Instruction::Sub:
813 case Instruction::Mul:
814 case Instruction::SDiv:
815 case Instruction::UDiv:
816 case Instruction::And:
817 case Instruction::Or:
818 case Instruction::Xor:
819 // This also makes sense for float operations, but disabled for now due
821 // case Instruction::FCmp:
822 // case Instruction::FAdd:
823 // case Instruction::FSub:
824 // case Instruction::FMul:
825 // case Instruction::FDiv:
826 FoldsLoad = (Bits == 32 || Bits == 64);
831 assert (UserI->getNumOperands() == 2 &&
832 "Expected to only handle binops.");
834 // UserI can't fold two loads, so in that case return 0 cost only
836 for (unsigned i = 0; i < 2; ++i) {
837 if (UserI->getOperand(i) == I)
839 if (LoadInst *LI = dyn_cast<LoadInst>(UserI->getOperand(i))) {
849 unsigned NumOps = getNumberOfParts(Src);
851 if (Src->getScalarSizeInBits() == 128)
852 // 128 bit scalars are held in a pair of two 64 bit registers.
858 int SystemZTTIImpl::getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
860 ArrayRef<unsigned> Indices,
862 unsigned AddressSpace) {
863 assert(isa<VectorType>(VecTy) &&
864 "Expect a vector type for interleaved memory op");
866 unsigned WideBits = (VecTy->isPtrOrPtrVectorTy() ?
867 (64U * VecTy->getVectorNumElements()) : VecTy->getPrimitiveSizeInBits());
868 assert (WideBits > 0 && "Could not compute size of vector");
870 ((WideBits % 128U) ? ((WideBits / 128U) + 1) : (WideBits / 128U));
872 // How many source vectors are handled to produce a vectorized operand?
873 int NumElsPerVector = (VecTy->getVectorNumElements() / NumWideParts);
875 ((NumWideParts > NumElsPerVector) ? NumElsPerVector : NumWideParts);
877 // A Load group may have gaps.
878 unsigned NumOperands =
879 ((Opcode == Instruction::Load) ? Indices.size() : Factor);
881 // Each needed permute takes two vectors as input.
884 int NumPermutes = NumSrcParts * NumOperands;
886 // Cost of load/store operations and the permutations needed.
887 return NumWideParts + NumPermutes;