1 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file defines an instruction selector for the SystemZ target.
11 //===----------------------------------------------------------------------===//
13 #include "SystemZTargetMachine.h"
14 #include "SystemZISelLowering.h"
15 #include "llvm/Analysis/AliasAnalysis.h"
16 #include "llvm/CodeGen/SelectionDAGISel.h"
17 #include "llvm/Support/Debug.h"
18 #include "llvm/Support/KnownBits.h"
19 #include "llvm/Support/raw_ostream.h"
23 #define DEBUG_TYPE "systemz-isel"
26 // Used to build addressing modes.
27 struct SystemZAddressingMode {
28 // The shape of the address.
33 // base+displacement+index for load and store operands
36 // base+displacement+index for load address operands
39 // base+displacement+index+ADJDYNALLOC
44 // The type of displacement. The enum names here correspond directly
45 // to the definitions in SystemZOperand.td. We could split them into
46 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
56 // The parts of the address. The address is equivalent to:
58 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
62 bool IncludesDynAlloc;
64 SystemZAddressingMode(AddrForm form, DispRange dr)
65 : Form(form), DR(dr), Base(), Disp(0), Index(),
66 IncludesDynAlloc(false) {}
68 // True if the address can have an index register.
69 bool hasIndexField() { return Form != FormBD; }
71 // True if the address can (and must) include ADJDYNALLOC.
72 bool isDynAlloc() { return Form == FormBDXDynAlloc; }
74 void dump(const llvm::SelectionDAG *DAG) {
75 errs() << "SystemZAddressingMode " << this << '\n';
79 Base.getNode()->dump(DAG);
83 if (hasIndexField()) {
86 Index.getNode()->dump(DAG);
91 errs() << " Disp " << Disp;
93 errs() << " + ADJDYNALLOC";
98 // Return a mask with Count low bits set.
99 static uint64_t allOnes(unsigned int Count) {
103 return (uint64_t(1) << Count) - 1;
106 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
107 // given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
108 // Rotate (I5). The combined operand value is effectively:
110 // (or (rotl Input, Rotate), ~Mask)
114 // (and (rotl Input, Rotate), Mask)
116 // otherwise. The output value has BitSize bits, although Input may be
117 // narrower (in which case the upper bits are don't care), or wider (in which
118 // case the result will be truncated as part of the operation).
119 struct RxSBGOperands {
120 RxSBGOperands(unsigned Op, SDValue N)
121 : Opcode(Op), BitSize(N.getValueSizeInBits()),
122 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
134 class SystemZDAGToDAGISel : public SelectionDAGISel {
135 const SystemZSubtarget *Subtarget;
137 // Used by SystemZOperands.td to create integer constants.
138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
139 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
142 const SystemZTargetMachine &getTargetMachine() const {
143 return static_cast<const SystemZTargetMachine &>(TM);
146 const SystemZInstrInfo *getInstrInfo() const {
147 return Subtarget->getInstrInfo();
150 // Try to fold more of the base or index of AM into AM, where IsBase
151 // selects between the base and index.
152 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
154 // Try to describe N in AM, returning true on success.
155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
157 // Extract individual target operands from matched address AM.
158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159 SDValue &Base, SDValue &Disp) const;
160 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
161 SDValue &Base, SDValue &Disp, SDValue &Index) const;
163 // Try to match Addr as a FormBD address with displacement type DR.
164 // Return true on success, storing the base and displacement in
165 // Base and Disp respectively.
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167 SDValue &Base, SDValue &Disp) const;
169 // Try to match Addr as a FormBDX address with displacement type DR.
170 // Return true on success and if the result had no index. Store the
171 // base and displacement in Base and Disp respectively.
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173 SDValue &Base, SDValue &Disp) const;
175 // Try to match Addr as a FormBDX* address of form Form with
176 // displacement type DR. Return true on success, storing the base,
177 // displacement and index in Base, Disp and Index respectively.
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
179 SystemZAddressingMode::DispRange DR, SDValue Addr,
180 SDValue &Base, SDValue &Disp, SDValue &Index) const;
182 // PC-relative address matching routines used by SystemZOperands.td.
183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
184 if (SystemZISD::isPCREL(Addr.getOpcode())) {
185 Target = Addr.getOperand(0);
191 // BD matching routines used by SystemZOperands.td.
192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
193 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
199 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
205 // MVI matching routines used by SystemZOperands.td.
206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
213 // BDX matching routines used by SystemZOperands.td.
214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215 SDValue &Index) const {
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
218 Addr, Base, Disp, Index);
220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221 SDValue &Index) const {
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
224 Addr, Base, Disp, Index);
226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227 SDValue &Index) const {
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
230 Addr, Base, Disp, Index);
232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233 SDValue &Index) const {
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
236 Addr, Base, Disp, Index);
238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239 SDValue &Index) const {
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
242 Addr, Base, Disp, Index);
244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245 SDValue &Index) const {
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
248 Addr, Base, Disp, Index);
250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251 SDValue &Index) const {
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
254 Addr, Base, Disp, Index);
256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257 SDValue &Index) const {
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
260 Addr, Base, Disp, Index);
263 // Try to match Addr as an address with a base, 12-bit displacement
264 // and index, where the index is element Elem of a vector.
265 // Return true on success, storing the base, displacement and vector
266 // in Base, Disp and Index respectively.
267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268 SDValue &Disp, SDValue &Index) const;
270 // Check whether (or Op (and X InsertMask)) is effectively an insertion
271 // of X into bits InsertMask of some Y != Op. Return true if so and
273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
275 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
276 // Return true on success.
277 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
279 // Try to fold some of RxSBG.Input into other fields of RxSBG.
280 // Return true on success.
281 bool expandRxSBG(RxSBGOperands &RxSBG) const;
283 // Return an undefined value of type VT.
284 SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
286 // Convert N to VT, if it isn't already.
287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
289 // Try to implement AND or shift node N using RISBG with the zero flag set.
290 // Return the selected node on success, otherwise return null.
291 bool tryRISBGZero(SDNode *N);
293 // Try to use RISBG or Opcode to implement OR or XOR node N.
294 // Return the selected node on success, otherwise return null.
295 bool tryRxSBG(SDNode *N, unsigned Opcode);
297 // If Op0 is null, then Node is a constant that can be loaded using:
299 // (Opcode UpperVal LowerVal)
301 // If Op0 is nonnull, then Node can be implemented using:
303 // (Opcode (Opcode Op0 UpperVal) LowerVal)
304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
305 uint64_t UpperVal, uint64_t LowerVal);
307 void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
310 // Try to use gather instruction Opcode to implement vector insertion N.
311 bool tryGather(SDNode *N, unsigned Opcode);
313 // Try to use scatter instruction Opcode to implement store Store.
314 bool tryScatter(StoreSDNode *Store, unsigned Opcode);
316 // Change a chain of {load; op; store} of the same value into a simple op
317 // through memory of that value, if the uses of the modified value and its
318 // address are suitable.
319 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
321 // Return true if Load and Store are loads and stores of the same size
322 // and are guaranteed not to overlap. Such operations can be implemented
323 // using block (SS-format) instructions.
325 // Partial overlap would lead to incorrect code, since the block operations
326 // are logically bytewise, even though they have a fast path for the
327 // non-overlapping case. We also need to avoid full overlap (i.e. two
328 // addresses that might be equal at run time) because although that case
329 // would be handled correctly, it might be implemented by millicode.
330 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
332 // N is a (store (load Y), X) pattern. Return true if it can use an MVC
334 bool storeLoadCanUseMVC(SDNode *N) const;
336 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
337 // if A[1 - I] == X and if N can use a block operation like NC from A[I]
339 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
341 // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
342 SDValue expandSelectBoolean(SDNode *Node);
345 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
346 : SelectionDAGISel(TM, OptLevel) {}
348 bool runOnMachineFunction(MachineFunction &MF) override {
349 const Function &F = MF.getFunction();
350 if (F.getFnAttribute("fentry-call").getValueAsString() != "true") {
351 if (F.hasFnAttribute("mnop-mcount"))
352 report_fatal_error("mnop-mcount only supported with fentry-call");
353 if (F.hasFnAttribute("mrecord-mcount"))
354 report_fatal_error("mrecord-mcount only supported with fentry-call");
357 Subtarget = &MF.getSubtarget<SystemZSubtarget>();
358 return SelectionDAGISel::runOnMachineFunction(MF);
361 // Override MachineFunctionPass.
362 StringRef getPassName() const override {
363 return "SystemZ DAG->DAG Pattern Instruction Selection";
366 // Override SelectionDAGISel.
367 void Select(SDNode *Node) override;
368 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
369 std::vector<SDValue> &OutOps) override;
370 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
371 void PreprocessISelDAG() override;
373 // Include the pieces autogenerated from the target description.
374 #include "SystemZGenDAGISel.inc"
376 } // end anonymous namespace
378 FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM,
379 CodeGenOpt::Level OptLevel) {
380 return new SystemZDAGToDAGISel(TM, OptLevel);
383 // Return true if Val should be selected as a displacement for an address
384 // with range DR. Here we're interested in the range of both the instruction
385 // described by DR and of any pairing instruction.
386 static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
388 case SystemZAddressingMode::Disp12Only:
389 return isUInt<12>(Val);
391 case SystemZAddressingMode::Disp12Pair:
392 case SystemZAddressingMode::Disp20Only:
393 case SystemZAddressingMode::Disp20Pair:
394 return isInt<20>(Val);
396 case SystemZAddressingMode::Disp20Only128:
397 return isInt<20>(Val) && isInt<20>(Val + 8);
399 llvm_unreachable("Unhandled displacement range");
402 // Change the base or index in AM to Value, where IsBase selects
403 // between the base and index.
404 static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
412 // The base or index of AM is equivalent to Value + ADJDYNALLOC,
413 // where IsBase selects between the base and index. Try to fold the
414 // ADJDYNALLOC into AM.
415 static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
417 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
418 changeComponent(AM, IsBase, Value);
419 AM.IncludesDynAlloc = true;
425 // The base of AM is equivalent to Base + Index. Try to use Index as
426 // the index register.
427 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
429 if (AM.hasIndexField() && !AM.Index.getNode()) {
437 // The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
438 // between the base and index. Try to fold Op1 into AM's displacement.
439 static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
440 SDValue Op0, uint64_t Op1) {
441 // First try adjusting the displacement.
442 int64_t TestDisp = AM.Disp + Op1;
443 if (selectDisp(AM.DR, TestDisp)) {
444 changeComponent(AM, IsBase, Op0);
449 // We could consider forcing the displacement into a register and
450 // using it as an index, but it would need to be carefully tuned.
454 bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
456 SDValue N = IsBase ? AM.Base : AM.Index;
457 unsigned Opcode = N.getOpcode();
458 if (Opcode == ISD::TRUNCATE) {
460 Opcode = N.getOpcode();
462 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
463 SDValue Op0 = N.getOperand(0);
464 SDValue Op1 = N.getOperand(1);
466 unsigned Op0Code = Op0->getOpcode();
467 unsigned Op1Code = Op1->getOpcode();
469 if (Op0Code == SystemZISD::ADJDYNALLOC)
470 return expandAdjDynAlloc(AM, IsBase, Op1);
471 if (Op1Code == SystemZISD::ADJDYNALLOC)
472 return expandAdjDynAlloc(AM, IsBase, Op0);
474 if (Op0Code == ISD::Constant)
475 return expandDisp(AM, IsBase, Op1,
476 cast<ConstantSDNode>(Op0)->getSExtValue());
477 if (Op1Code == ISD::Constant)
478 return expandDisp(AM, IsBase, Op0,
479 cast<ConstantSDNode>(Op1)->getSExtValue());
481 if (IsBase && expandIndex(AM, Op0, Op1))
484 if (Opcode == SystemZISD::PCREL_OFFSET) {
485 SDValue Full = N.getOperand(0);
486 SDValue Base = N.getOperand(1);
487 SDValue Anchor = Base.getOperand(0);
488 uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
489 cast<GlobalAddressSDNode>(Anchor)->getOffset());
490 return expandDisp(AM, IsBase, Base, Offset);
495 // Return true if an instruction with displacement range DR should be
496 // used for displacement value Val. selectDisp(DR, Val) must already hold.
497 static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
498 assert(selectDisp(DR, Val) && "Invalid displacement");
500 case SystemZAddressingMode::Disp12Only:
501 case SystemZAddressingMode::Disp20Only:
502 case SystemZAddressingMode::Disp20Only128:
505 case SystemZAddressingMode::Disp12Pair:
506 // Use the other instruction if the displacement is too large.
507 return isUInt<12>(Val);
509 case SystemZAddressingMode::Disp20Pair:
510 // Use the other instruction if the displacement is small enough.
511 return !isUInt<12>(Val);
513 llvm_unreachable("Unhandled displacement range");
516 // Return true if Base + Disp + Index should be performed by LA(Y).
517 static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
518 // Don't use LA(Y) for constants.
522 // Always use LA(Y) for frame addresses, since we know that the destination
523 // register is almost always (perhaps always) going to be different from
524 // the frame register.
525 if (Base->getOpcode() == ISD::FrameIndex)
529 // Always use LA(Y) if there is a base, displacement and index.
533 // Always use LA if the displacement is small enough. It should always
534 // be no worse than AGHI (and better if it avoids a move).
535 if (isUInt<12>(Disp))
538 // For similar reasons, always use LAY if the constant is too big for AGHI.
539 // LAY should be no worse than AGFI.
540 if (!isInt<16>(Disp))
543 // Don't use LA for plain registers.
547 // Don't use LA for plain addition if the index operand is only used
548 // once. It should be a natural two-operand addition in that case.
549 if (Index->hasOneUse())
552 // Prefer addition if the second operation is sign-extended, in the
553 // hope of using AGF.
554 unsigned IndexOpcode = Index->getOpcode();
555 if (IndexOpcode == ISD::SIGN_EXTEND ||
556 IndexOpcode == ISD::SIGN_EXTEND_INREG)
560 // Don't use LA for two-operand addition if either operand is only
561 // used once. The addition instructions are better in that case.
562 if (Base->hasOneUse())
568 // Return true if Addr is suitable for AM, updating AM if so.
569 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
570 SystemZAddressingMode &AM) const {
571 // Start out assuming that the address will need to be loaded separately,
572 // then try to extend it as much as we can.
575 // First try treating the address as a constant.
576 if (Addr.getOpcode() == ISD::Constant &&
577 expandDisp(AM, true, SDValue(),
578 cast<ConstantSDNode>(Addr)->getSExtValue()))
580 // Also see if it's a bare ADJDYNALLOC.
581 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
582 expandAdjDynAlloc(AM, true, SDValue()))
585 // Otherwise try expanding each component.
586 while (expandAddress(AM, true) ||
587 (AM.Index.getNode() && expandAddress(AM, false)))
590 // Reject cases where it isn't profitable to use LA(Y).
591 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
592 !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
595 // Reject cases where the other instruction in a pair should be used.
596 if (!isValidDisp(AM.DR, AM.Disp))
599 // Make sure that ADJDYNALLOC is included where necessary.
600 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
603 LLVM_DEBUG(AM.dump(CurDAG));
607 // Insert a node into the DAG at least before Pos. This will reposition
608 // the node as needed, and will assign it a node ID that is <= Pos's ID.
609 // Note that this does *not* preserve the uniqueness of node IDs!
610 // The selection DAG must no longer depend on their uniqueness when this
612 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
613 if (N->getNodeId() == -1 ||
614 (SelectionDAGISel::getUninvalidatedNodeId(N.getNode()) >
615 SelectionDAGISel::getUninvalidatedNodeId(Pos))) {
616 DAG->RepositionNode(Pos->getIterator(), N.getNode());
617 // Mark Node as invalid for pruning as after this it may be a successor to a
618 // selected node but otherwise be in the same position of Pos.
619 // Conservatively mark it with the same -abs(Id) to assure node id
620 // invariant is preserved.
621 N->setNodeId(Pos->getNodeId());
622 SelectionDAGISel::InvalidateNodeId(N.getNode());
626 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
627 EVT VT, SDValue &Base,
628 SDValue &Disp) const {
631 // Register 0 means "no base". This is mostly useful for shifts.
632 Base = CurDAG->getRegister(0, VT);
633 else if (Base.getOpcode() == ISD::FrameIndex) {
634 // Lower a FrameIndex to a TargetFrameIndex.
635 int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
636 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
637 } else if (Base.getValueType() != VT) {
638 // Truncate values from i64 to i32, for shifts.
639 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
640 "Unexpected truncation");
642 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
643 insertDAGNode(CurDAG, Base.getNode(), Trunc);
647 // Lower the displacement to a TargetConstant.
648 Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
651 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
652 EVT VT, SDValue &Base,
654 SDValue &Index) const {
655 getAddressOperands(AM, VT, Base, Disp);
658 if (!Index.getNode())
659 // Register 0 means "no index".
660 Index = CurDAG->getRegister(0, VT);
663 bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
664 SDValue Addr, SDValue &Base,
665 SDValue &Disp) const {
666 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
667 if (!selectAddress(Addr, AM))
670 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
674 bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
675 SDValue Addr, SDValue &Base,
676 SDValue &Disp) const {
677 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
678 if (!selectAddress(Addr, AM) || AM.Index.getNode())
681 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
685 bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
686 SystemZAddressingMode::DispRange DR,
687 SDValue Addr, SDValue &Base,
688 SDValue &Disp, SDValue &Index) const {
689 SystemZAddressingMode AM(Form, DR);
690 if (!selectAddress(Addr, AM))
693 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
697 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
700 SDValue &Index) const {
702 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
703 Regs[0].getNode() && Regs[1].getNode()) {
704 for (unsigned int I = 0; I < 2; ++I) {
707 // We can't tell here whether the index vector has the right type
708 // for the access; the caller needs to do that instead.
709 if (Index.getOpcode() == ISD::ZERO_EXTEND)
710 Index = Index.getOperand(0);
711 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
712 Index.getOperand(1) == Elem) {
713 Index = Index.getOperand(0);
721 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
722 uint64_t InsertMask) const {
723 // We're only interested in cases where the insertion is into some operand
724 // of Op, rather than into Op itself. The only useful case is an AND.
725 if (Op.getOpcode() != ISD::AND)
728 // We need a constant mask.
729 auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
733 // It's not an insertion of Op.getOperand(0) if the two masks overlap.
734 uint64_t AndMask = MaskNode->getZExtValue();
735 if (InsertMask & AndMask)
738 // It's only an insertion if all bits are covered or are known to be zero.
739 // The inner check covers all cases but is more expensive.
740 uint64_t Used = allOnes(Op.getValueSizeInBits());
741 if (Used != (AndMask | InsertMask)) {
742 KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));
743 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
747 Op = Op.getOperand(0);
751 bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
752 uint64_t Mask) const {
753 const SystemZInstrInfo *TII = getInstrInfo();
754 if (RxSBG.Rotate != 0)
755 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
757 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
764 // Return true if any bits of (RxSBG.Input & Mask) are significant.
765 static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
766 // Rotate the mask in the same way as RxSBG.Input is rotated.
767 if (RxSBG.Rotate != 0)
768 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
769 return (Mask & RxSBG.Mask) != 0;
772 bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
773 SDValue N = RxSBG.Input;
774 unsigned Opcode = N.getOpcode();
776 case ISD::TRUNCATE: {
777 if (RxSBG.Opcode == SystemZ::RNSBG)
779 uint64_t BitSize = N.getValueSizeInBits();
780 uint64_t Mask = allOnes(BitSize);
781 if (!refineRxSBGMask(RxSBG, Mask))
783 RxSBG.Input = N.getOperand(0);
787 if (RxSBG.Opcode == SystemZ::RNSBG)
790 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
794 SDValue Input = N.getOperand(0);
795 uint64_t Mask = MaskNode->getZExtValue();
796 if (!refineRxSBGMask(RxSBG, Mask)) {
797 // If some bits of Input are already known zeros, those bits will have
798 // been removed from the mask. See if adding them back in makes the
800 KnownBits Known = CurDAG->computeKnownBits(Input);
801 Mask |= Known.Zero.getZExtValue();
802 if (!refineRxSBGMask(RxSBG, Mask))
810 if (RxSBG.Opcode != SystemZ::RNSBG)
813 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
817 SDValue Input = N.getOperand(0);
818 uint64_t Mask = ~MaskNode->getZExtValue();
819 if (!refineRxSBGMask(RxSBG, Mask)) {
820 // If some bits of Input are already known ones, those bits will have
821 // been removed from the mask. See if adding them back in makes the
823 KnownBits Known = CurDAG->computeKnownBits(Input);
824 Mask &= ~Known.One.getZExtValue();
825 if (!refineRxSBGMask(RxSBG, Mask))
833 // Any 64-bit rotate left can be merged into the RxSBG.
834 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
836 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
840 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
841 RxSBG.Input = N.getOperand(0);
845 case ISD::ANY_EXTEND:
846 // Bits above the extended operand are don't-care.
847 RxSBG.Input = N.getOperand(0);
850 case ISD::ZERO_EXTEND:
851 if (RxSBG.Opcode != SystemZ::RNSBG) {
852 // Restrict the mask to the extended operand.
853 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
854 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
857 RxSBG.Input = N.getOperand(0);
862 case ISD::SIGN_EXTEND: {
863 // Check that the extension bits are don't-care (i.e. are masked out
864 // by the final mask).
865 unsigned BitSize = N.getValueSizeInBits();
866 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
867 if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) {
868 // In the case where only the sign bit is active, increase Rotate with
869 // the extension width.
870 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
871 RxSBG.Rotate += (BitSize - InnerBitSize);
876 RxSBG.Input = N.getOperand(0);
881 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
885 uint64_t Count = CountNode->getZExtValue();
886 unsigned BitSize = N.getValueSizeInBits();
887 if (Count < 1 || Count >= BitSize)
890 if (RxSBG.Opcode == SystemZ::RNSBG) {
891 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
892 // count bits from RxSBG.Input are ignored.
893 if (maskMatters(RxSBG, allOnes(Count)))
896 // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
897 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
901 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
902 RxSBG.Input = N.getOperand(0);
908 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
912 uint64_t Count = CountNode->getZExtValue();
913 unsigned BitSize = N.getValueSizeInBits();
914 if (Count < 1 || Count >= BitSize)
917 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
918 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
919 // count bits from RxSBG.Input are ignored.
920 if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
923 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
924 // which is similar to SLL above.
925 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
929 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
930 RxSBG.Input = N.getOperand(0);
938 SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
939 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
940 return SDValue(N, 0);
943 SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
945 if (N.getValueType() == MVT::i32 && VT == MVT::i64)
946 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
947 DL, VT, getUNDEF(DL, MVT::i64), N);
948 if (N.getValueType() == MVT::i64 && VT == MVT::i32)
949 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
950 assert(N.getValueType() == VT && "Unexpected value types");
954 bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
956 EVT VT = N->getValueType(0);
957 if (!VT.isInteger() || VT.getSizeInBits() > 64)
959 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
961 while (expandRxSBG(RISBG))
962 // The widening or narrowing is expected to be free.
963 // Counting widening or narrowing as a saved operation will result in
964 // preferring an R*SBG over a simple shift/logical instruction.
965 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
966 RISBG.Input.getOpcode() != ISD::TRUNCATE)
971 // Prefer to use normal shift instructions over RISBG, since they can handle
972 // all cases and are sometimes shorter.
973 if (Count == 1 && N->getOpcode() != ISD::AND)
976 // Prefer register extensions like LLC over RISBG. Also prefer to start
977 // out with normal ANDs if one instruction would be enough. We can convert
978 // these ANDs into an RISBG later if a three-address instruction is useful.
979 if (RISBG.Rotate == 0) {
980 bool PreferAnd = false;
981 // Prefer AND for any 32-bit and-immediate operation.
984 // As well as for any 64-bit operation that can be implemented via LLC(R),
985 // LLH(R), LLGT(R), or one of the and-immediate instructions.
986 else if (RISBG.Mask == 0xff ||
987 RISBG.Mask == 0xffff ||
988 RISBG.Mask == 0x7fffffff ||
989 SystemZ::isImmLF(~RISBG.Mask) ||
990 SystemZ::isImmHF(~RISBG.Mask))
992 // And likewise for the LLZRGF instruction, which doesn't have a register
993 // to register version.
994 else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
995 if (Load->getMemoryVT() == MVT::i32 &&
996 (Load->getExtensionType() == ISD::EXTLOAD ||
997 Load->getExtensionType() == ISD::ZEXTLOAD) &&
998 RISBG.Mask == 0xffffff00 &&
999 Subtarget->hasLoadAndZeroRightmostByte())
1003 // Replace the current node with an AND. Note that the current node
1004 // might already be that same AND, in which case it is already CSE'd
1005 // with it, and we must not call ReplaceNode.
1006 SDValue In = convertTo(DL, VT, RISBG.Input);
1007 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
1008 SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
1009 if (N != New.getNode()) {
1010 insertDAGNode(CurDAG, N, Mask);
1011 insertDAGNode(CurDAG, N, New);
1012 ReplaceNode(N, New.getNode());
1015 // Now, select the machine opcode to implement this operation.
1016 if (!N->isMachineOpcode())
1022 unsigned Opcode = SystemZ::RISBG;
1023 // Prefer RISBGN if available, since it does not clobber CC.
1024 if (Subtarget->hasMiscellaneousExtensions())
1025 Opcode = SystemZ::RISBGN;
1026 EVT OpcodeVT = MVT::i64;
1027 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1028 // We can only use the 32-bit instructions if all source bits are
1029 // in the low 32 bits without wrapping, both after rotation (because
1030 // of the smaller range for Start and End) and before rotation
1031 // (because the input value is truncated).
1032 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1033 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1034 ((RISBG.End + RISBG.Rotate) & 63) >=
1035 ((RISBG.Start + RISBG.Rotate) & 63)) {
1036 Opcode = SystemZ::RISBMux;
1037 OpcodeVT = MVT::i32;
1042 getUNDEF(DL, OpcodeVT),
1043 convertTo(DL, OpcodeVT, RISBG.Input),
1044 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1045 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1046 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1048 SDValue New = convertTo(
1049 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1050 ReplaceNode(N, New.getNode());
1054 bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1056 EVT VT = N->getValueType(0);
1057 if (!VT.isInteger() || VT.getSizeInBits() > 64)
1059 // Try treating each operand of N as the second operand of the RxSBG
1060 // and see which goes deepest.
1061 RxSBGOperands RxSBG[] = {
1062 RxSBGOperands(Opcode, N->getOperand(0)),
1063 RxSBGOperands(Opcode, N->getOperand(1))
1065 unsigned Count[] = { 0, 0 };
1066 for (unsigned I = 0; I < 2; ++I)
1067 while (expandRxSBG(RxSBG[I]))
1068 // The widening or narrowing is expected to be free.
1069 // Counting widening or narrowing as a saved operation will result in
1070 // preferring an R*SBG over a simple shift/logical instruction.
1071 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1072 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1075 // Do nothing if neither operand is suitable.
1076 if (Count[0] == 0 && Count[1] == 0)
1079 // Pick the deepest second operand.
1080 unsigned I = Count[0] > Count[1] ? 0 : 1;
1081 SDValue Op0 = N->getOperand(I ^ 1);
1083 // Prefer IC for character insertions from memory.
1084 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1085 if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1086 if (Load->getMemoryVT() == MVT::i8)
1089 // See whether we can avoid an AND in the first operand by converting
1091 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
1092 Opcode = SystemZ::RISBG;
1093 // Prefer RISBGN if available, since it does not clobber CC.
1094 if (Subtarget->hasMiscellaneousExtensions())
1095 Opcode = SystemZ::RISBGN;
1099 convertTo(DL, MVT::i64, Op0),
1100 convertTo(DL, MVT::i64, RxSBG[I].Input),
1101 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1102 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1103 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1105 SDValue New = convertTo(
1106 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1107 ReplaceNode(N, New.getNode());
1111 void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1112 SDValue Op0, uint64_t UpperVal,
1113 uint64_t LowerVal) {
1114 EVT VT = Node->getValueType(0);
1116 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1118 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1121 // When we haven't passed in Op0, Upper will be a constant. In order to
1122 // prevent folding back to the large immediate in `Or = getNode(...)` we run
1123 // SelectCode first and end up with an opaque machine node. This means that
1124 // we need to use a handle to keep track of Upper in case it gets CSE'd by
1127 // Note that in the case where Op0 is passed in we could just call
1128 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1129 // the handle at all, but it's fine to do it here.
1131 // TODO: This is a pretty hacky way to do this. Can we do something that
1132 // doesn't require a two paragraph explanation?
1133 HandleSDNode Handle(Upper);
1134 SelectCode(Upper.getNode());
1135 Upper = Handle.getValue();
1138 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1139 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1141 ReplaceNode(Node, Or.getNode());
1143 SelectCode(Or.getNode());
1146 void SystemZDAGToDAGISel::loadVectorConstant(
1147 const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1148 assert((VCI.Opcode == SystemZISD::BYTE_MASK ||
1149 VCI.Opcode == SystemZISD::REPLICATE ||
1150 VCI.Opcode == SystemZISD::ROTATE_MASK) &&
1152 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1153 EVT VT = Node->getValueType(0);
1155 SmallVector<SDValue, 2> Ops;
1156 for (unsigned OpVal : VCI.OpVals)
1157 Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32));
1158 SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
1160 if (VCI.VecVT == VT.getSimpleVT())
1161 ReplaceNode(Node, Op.getNode());
1162 else if (VT.getSizeInBits() == 128) {
1163 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
1164 ReplaceNode(Node, BitCast.getNode());
1165 SelectCode(BitCast.getNode());
1166 } else { // float or double
1167 unsigned SubRegIdx =
1168 (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1170 Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());
1172 SelectCode(Op.getNode());
1175 bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1176 SDValue ElemV = N->getOperand(2);
1177 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1181 unsigned Elem = ElemN->getZExtValue();
1182 EVT VT = N->getValueType(0);
1183 if (Elem >= VT.getVectorNumElements())
1186 auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1187 if (!Load || !Load->hasNUsesOfValue(1, 0))
1189 if (Load->getMemoryVT().getSizeInBits() !=
1190 Load->getValueType(0).getSizeInBits())
1193 SDValue Base, Disp, Index;
1194 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1195 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1200 N->getOperand(0), Base, Disp, Index,
1201 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1203 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1204 ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1205 ReplaceNode(N, Res);
1209 bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1210 SDValue Value = Store->getValue();
1211 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1213 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1216 SDValue ElemV = Value.getOperand(1);
1217 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1221 SDValue Vec = Value.getOperand(0);
1222 EVT VT = Vec.getValueType();
1223 unsigned Elem = ElemN->getZExtValue();
1224 if (Elem >= VT.getVectorNumElements())
1227 SDValue Base, Disp, Index;
1228 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1229 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1234 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1237 ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1241 // Check whether or not the chain ending in StoreNode is suitable for doing
1242 // the {load; op; store} to modify transformation.
1243 static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
1244 SDValue StoredVal, SelectionDAG *CurDAG,
1245 LoadSDNode *&LoadNode,
1246 SDValue &InputChain) {
1247 // Is the stored value result 0 of the operation?
1248 if (StoredVal.getResNo() != 0)
1251 // Are there other uses of the loaded value than the operation?
1252 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0))
1255 // Is the store non-extending and non-indexed?
1256 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
1259 SDValue Load = StoredVal->getOperand(0);
1260 // Is the stored value a non-extending and non-indexed load?
1261 if (!ISD::isNormalLoad(Load.getNode()))
1264 // Return LoadNode by reference.
1265 LoadNode = cast<LoadSDNode>(Load);
1267 // Is store the only read of the loaded value?
1268 if (!Load.hasOneUse())
1271 // Is the address of the store the same as the load?
1272 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1273 LoadNode->getOffset() != StoreNode->getOffset())
1276 // Check if the chain is produced by the load or is a TokenFactor with
1277 // the load output chain as an operand. Return InputChain by reference.
1278 SDValue Chain = StoreNode->getChain();
1280 bool ChainCheck = false;
1281 if (Chain == Load.getValue(1)) {
1283 InputChain = LoadNode->getChain();
1284 } else if (Chain.getOpcode() == ISD::TokenFactor) {
1285 SmallVector<SDValue, 4> ChainOps;
1286 SmallVector<const SDNode *, 4> LoopWorklist;
1287 SmallPtrSet<const SDNode *, 16> Visited;
1288 const unsigned int Max = 1024;
1289 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
1290 SDValue Op = Chain.getOperand(i);
1291 if (Op == Load.getValue(1)) {
1293 // Drop Load, but keep its chain. No cycle check necessary.
1294 ChainOps.push_back(Load.getOperand(0));
1297 LoopWorklist.push_back(Op.getNode());
1298 ChainOps.push_back(Op);
1302 // Add the other operand of StoredVal to worklist.
1303 for (SDValue Op : StoredVal->ops())
1304 if (Op.getNode() != LoadNode)
1305 LoopWorklist.push_back(Op.getNode());
1307 // Check if Load is reachable from any of the nodes in the worklist.
1308 if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max,
1312 // Make a new TokenFactor with all the other input chains except
1314 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain),
1315 MVT::Other, ChainOps);
1324 // Change a chain of {load; op; store} of the same value into a simple op
1325 // through memory of that value, if the uses of the modified value and its
1326 // address are suitable.
1328 // The tablegen pattern memory operand pattern is currently not able to match
1329 // the case where the CC on the original operation are used.
1331 // See the equivalent routine in X86ISelDAGToDAG for further comments.
1332 bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1333 StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
1334 SDValue StoredVal = StoreNode->getOperand(1);
1335 unsigned Opc = StoredVal->getOpcode();
1336 SDLoc DL(StoreNode);
1338 // Before we try to select anything, make sure this is memory operand size
1339 // and opcode we can handle. Note that this must match the code below that
1340 // actually lowers the opcodes.
1341 EVT MemVT = StoreNode->getMemoryVT();
1342 unsigned NewOpc = 0;
1343 bool NegateOperand = false;
1347 case SystemZISD::SSUBO:
1348 NegateOperand = true;
1350 case SystemZISD::SADDO:
1351 if (MemVT == MVT::i32)
1352 NewOpc = SystemZ::ASI;
1353 else if (MemVT == MVT::i64)
1354 NewOpc = SystemZ::AGSI;
1358 case SystemZISD::USUBO:
1359 NegateOperand = true;
1361 case SystemZISD::UADDO:
1362 if (MemVT == MVT::i32)
1363 NewOpc = SystemZ::ALSI;
1364 else if (MemVT == MVT::i64)
1365 NewOpc = SystemZ::ALGSI;
1371 LoadSDNode *LoadNode = nullptr;
1373 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1377 SDValue Operand = StoredVal.getOperand(1);
1378 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1381 auto OperandV = OperandC->getAPIntValue();
1383 OperandV = -OperandV;
1384 if (OperandV.getMinSignedBits() > 8)
1386 Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT);
1389 if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp))
1392 SDValue Ops[] = { Base, Disp, Operand, InputChain };
1393 MachineSDNode *Result =
1394 CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops);
1395 CurDAG->setNodeMemRefs(
1396 Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1398 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
1399 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0));
1400 CurDAG->RemoveDeadNode(Node);
1404 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1405 LoadSDNode *Load) const {
1406 // Check that the two memory operands have the same size.
1407 if (Load->getMemoryVT() != Store->getMemoryVT())
1410 // Volatility stops an access from being decomposed.
1411 if (Load->isVolatile() || Store->isVolatile())
1414 // There's no chance of overlap if the load is invariant.
1415 if (Load->isInvariant() && Load->isDereferenceable())
1418 // Otherwise we need to check whether there's an alias.
1419 const Value *V1 = Load->getMemOperand()->getValue();
1420 const Value *V2 = Store->getMemOperand()->getValue();
1425 uint64_t Size = Load->getMemoryVT().getStoreSize();
1426 int64_t End1 = Load->getSrcValueOffset() + Size;
1427 int64_t End2 = Store->getSrcValueOffset() + Size;
1428 if (V1 == V2 && End1 == End2)
1431 return !AA->alias(MemoryLocation(V1, End1, Load->getAAInfo()),
1432 MemoryLocation(V2, End2, Store->getAAInfo()));
1435 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1436 auto *Store = cast<StoreSDNode>(N);
1437 auto *Load = cast<LoadSDNode>(Store->getValue());
1439 // Prefer not to use MVC if either address can use ... RELATIVE LONG
1441 uint64_t Size = Load->getMemoryVT().getStoreSize();
1442 if (Size > 1 && Size <= 8) {
1443 // Prefer LHRL, LRL and LGRL.
1444 if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1446 // Prefer STHRL, STRL and STGRL.
1447 if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1451 return canUseBlockOperation(Store, Load);
1454 bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1456 auto *StoreA = cast<StoreSDNode>(N);
1457 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1458 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1459 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1460 canUseBlockOperation(StoreA, LoadB);
1463 void SystemZDAGToDAGISel::Select(SDNode *Node) {
1464 // If we have a custom node, we already have selected!
1465 if (Node->isMachineOpcode()) {
1466 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1467 Node->setNodeId(-1);
1471 unsigned Opcode = Node->getOpcode();
1474 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1475 if (tryRxSBG(Node, SystemZ::ROSBG))
1480 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1481 if (tryRxSBG(Node, SystemZ::RXSBG))
1485 // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1486 // split the operation into two. If both operands here happen to be
1487 // constant, leave this to common code to optimize.
1488 if (Node->getValueType(0) == MVT::i64 &&
1489 Node->getOperand(0).getOpcode() != ISD::Constant)
1490 if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1491 uint64_t Val = Op1->getZExtValue();
1492 // Don't split the operation if we can match one of the combined
1493 // logical operations provided by miscellaneous-extensions-3.
1494 if (Subtarget->hasMiscellaneousExtensions3()) {
1495 unsigned ChildOpcode = Node->getOperand(0).getOpcode();
1496 // Check whether this expression matches NAND/NOR/NXOR.
1497 if (Val == (uint64_t)-1 && Opcode == ISD::XOR)
1498 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||
1499 ChildOpcode == ISD::XOR)
1501 // Check whether this expression matches OR-with-complement
1502 // (or matches an alternate pattern for NXOR).
1503 if (ChildOpcode == ISD::XOR) {
1504 auto Op0 = Node->getOperand(0);
1505 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1)))
1506 if (Op0Op1->getZExtValue() == (uint64_t)-1)
1510 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1511 splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1512 Val - uint32_t(Val), uint32_t(Val));
1519 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1520 if (tryRxSBG(Node, SystemZ::RNSBG))
1526 case ISD::ZERO_EXTEND:
1527 if (tryRISBGZero(Node))
1532 // If this is a 64-bit constant that is out of the range of LLILF,
1533 // LLIHF and LGFI, split it into two 32-bit pieces.
1534 if (Node->getValueType(0) == MVT::i64) {
1535 uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue();
1536 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) {
1537 splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1544 case SystemZISD::SELECT_CCMASK: {
1545 SDValue Op0 = Node->getOperand(0);
1546 SDValue Op1 = Node->getOperand(1);
1547 // Prefer to put any load first, so that it can be matched as a
1548 // conditional load. Likewise for constants in range for LOCHI.
1549 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1550 (Subtarget->hasLoadStoreOnCond2() &&
1551 Node->getValueType(0).isInteger() &&
1552 Op1.getOpcode() == ISD::Constant &&
1553 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1554 !(Op0.getOpcode() == ISD::Constant &&
1555 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1556 SDValue CCValid = Node->getOperand(2);
1557 SDValue CCMask = Node->getOperand(3);
1558 uint64_t ConstCCValid =
1559 cast<ConstantSDNode>(CCValid.getNode())->getZExtValue();
1560 uint64_t ConstCCMask =
1561 cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
1562 // Invert the condition.
1563 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1564 SDLoc(Node), CCMask.getValueType());
1565 SDValue Op4 = Node->getOperand(4);
1566 SDNode *UpdatedNode =
1567 CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1568 if (UpdatedNode != Node) {
1569 // In case this node already exists then replace Node with it.
1570 ReplaceNode(Node, UpdatedNode);
1577 case ISD::INSERT_VECTOR_ELT: {
1578 EVT VT = Node->getValueType(0);
1579 unsigned ElemBitSize = VT.getScalarSizeInBits();
1580 if (ElemBitSize == 32) {
1581 if (tryGather(Node, SystemZ::VGEF))
1583 } else if (ElemBitSize == 64) {
1584 if (tryGather(Node, SystemZ::VGEG))
1590 case ISD::BUILD_VECTOR: {
1591 auto *BVN = cast<BuildVectorSDNode>(Node);
1592 SystemZVectorConstantInfo VCI(BVN);
1593 if (VCI.isVectorConstantLegal(*Subtarget)) {
1594 loadVectorConstant(VCI, Node);
1600 case ISD::ConstantFP: {
1601 APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF();
1602 if (Imm.isZero() || Imm.isNegZero())
1604 SystemZVectorConstantInfo VCI(Imm);
1605 bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success;
1606 assert(Success && "Expected legal FP immediate");
1607 loadVectorConstant(VCI, Node);
1612 if (tryFoldLoadStoreIntoMemOperand(Node))
1614 auto *Store = cast<StoreSDNode>(Node);
1615 unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1616 if (ElemBitSize == 32) {
1617 if (tryScatter(Store, SystemZ::VSCEF))
1619 } else if (ElemBitSize == 64) {
1620 if (tryScatter(Store, SystemZ::VSCEG))
1630 bool SystemZDAGToDAGISel::
1631 SelectInlineAsmMemoryOperand(const SDValue &Op,
1632 unsigned ConstraintID,
1633 std::vector<SDValue> &OutOps) {
1634 SystemZAddressingMode::AddrForm Form;
1635 SystemZAddressingMode::DispRange DispRange;
1636 SDValue Base, Disp, Index;
1638 switch(ConstraintID) {
1640 llvm_unreachable("Unexpected asm memory constraint");
1641 case InlineAsm::Constraint_i:
1642 case InlineAsm::Constraint_Q:
1643 // Accept an address with a short displacement, but no index.
1644 Form = SystemZAddressingMode::FormBD;
1645 DispRange = SystemZAddressingMode::Disp12Only;
1647 case InlineAsm::Constraint_R:
1648 // Accept an address with a short displacement and an index.
1649 Form = SystemZAddressingMode::FormBDXNormal;
1650 DispRange = SystemZAddressingMode::Disp12Only;
1652 case InlineAsm::Constraint_S:
1653 // Accept an address with a long displacement, but no index.
1654 Form = SystemZAddressingMode::FormBD;
1655 DispRange = SystemZAddressingMode::Disp20Only;
1657 case InlineAsm::Constraint_T:
1658 case InlineAsm::Constraint_m:
1659 case InlineAsm::Constraint_o:
1660 // Accept an address with a long displacement and an index.
1661 // m works the same as T, as this is the most general case.
1662 // We don't really have any special handling of "offsettable"
1663 // memory addresses, so just treat o the same as m.
1664 Form = SystemZAddressingMode::FormBDXNormal;
1665 DispRange = SystemZAddressingMode::Disp20Only;
1669 if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1670 const TargetRegisterClass *TRC =
1671 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1673 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1675 // Make sure that the base address doesn't go into %r0.
1676 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1677 if (Base.getOpcode() != ISD::TargetFrameIndex &&
1678 Base.getOpcode() != ISD::Register) {
1680 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1681 DL, Base.getValueType(),
1685 // Make sure that the index register isn't assigned to %r0 either.
1686 if (Index.getOpcode() != ISD::Register) {
1688 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1689 DL, Index.getValueType(),
1693 OutOps.push_back(Base);
1694 OutOps.push_back(Disp);
1695 OutOps.push_back(Index);
1702 // IsProfitableToFold - Returns true if is profitable to fold the specific
1703 // operand node N of U during instruction selection that starts at Root.
1705 SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1706 SDNode *Root) const {
1707 // We want to avoid folding a LOAD into an ICMP node if as a result
1708 // we would be forced to spill the condition code into a GPR.
1709 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {
1710 if (!N.hasOneUse() || !U->hasOneUse())
1713 // The user of the CC value will usually be a CopyToReg into the
1714 // physical CC register, which in turn is glued and chained to the
1715 // actual instruction that uses the CC value. Bail out if we have
1716 // anything else than that.
1717 SDNode *CCUser = *U->use_begin();
1718 SDNode *CCRegUser = nullptr;
1719 if (CCUser->getOpcode() == ISD::CopyToReg ||
1720 cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC) {
1721 for (auto *U : CCUser->uses()) {
1722 if (CCRegUser == nullptr)
1724 else if (CCRegUser != U)
1728 if (CCRegUser == nullptr)
1731 // If the actual instruction is a branch, the only thing that remains to be
1732 // checked is whether the CCUser chain is a predecessor of the load.
1733 if (CCRegUser->isMachineOpcode() &&
1734 CCRegUser->getMachineOpcode() == SystemZ::BRC)
1735 return !N->isPredecessorOf(CCUser->getOperand(0).getNode());
1737 // Otherwise, the instruction may have multiple operands, and we need to
1738 // verify that none of them are a predecessor of the load. This is exactly
1739 // the same check that would be done by common code if the CC setter were
1740 // glued to the CC user, so simply invoke that check here.
1741 if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false))
1749 // Represents a sequence for extracting a 0/1 value from an IPM result:
1750 // (((X ^ XORValue) + AddValue) >> Bit)
1751 struct IPMConversion {
1752 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1753 : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1759 } // end anonymous namespace
1761 // Return a sequence for getting a 1 from an IPM result when CC has a
1762 // value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
1763 // The handling of CC values outside CCValid doesn't matter.
1764 static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
1765 // Deal with cases where the result can be taken directly from a bit
1766 // of the IPM result.
1767 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1768 return IPMConversion(0, 0, SystemZ::IPM_CC);
1769 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1770 return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1772 // Deal with cases where we can add a value to force the sign bit
1773 // to contain the right value. Putting the bit in 31 means we can
1774 // use SRL rather than RISBG(L), and also makes it easier to get a
1775 // 0/-1 value, so it has priority over the other tests below.
1777 // These sequences rely on the fact that the upper two bits of the
1778 // IPM result are zero.
1779 uint64_t TopBit = uint64_t(1) << 31;
1780 if (CCMask == (CCValid & SystemZ::CCMASK_0))
1781 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1782 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1783 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1784 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1786 | SystemZ::CCMASK_2)))
1787 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1788 if (CCMask == (CCValid & SystemZ::CCMASK_3))
1789 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1790 if (CCMask == (CCValid & (SystemZ::CCMASK_1
1792 | SystemZ::CCMASK_3)))
1793 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1795 // Next try inverting the value and testing a bit. 0/1 could be
1796 // handled this way too, but we dealt with that case above.
1797 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
1798 return IPMConversion(-1, 0, SystemZ::IPM_CC);
1800 // Handle cases where adding a value forces a non-sign bit to contain
1802 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
1803 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
1804 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
1805 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
1807 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are
1808 // can be done by inverting the low CC bit and applying one of the
1809 // sign-based extractions above.
1810 if (CCMask == (CCValid & SystemZ::CCMASK_1))
1811 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
1812 if (CCMask == (CCValid & SystemZ::CCMASK_2))
1813 return IPMConversion(1 << SystemZ::IPM_CC,
1814 TopBit - (3 << SystemZ::IPM_CC), 31);
1815 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1817 | SystemZ::CCMASK_3)))
1818 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
1819 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1821 | SystemZ::CCMASK_3)))
1822 return IPMConversion(1 << SystemZ::IPM_CC,
1823 TopBit - (1 << SystemZ::IPM_CC), 31);
1825 llvm_unreachable("Unexpected CC combination");
1828 SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
1829 auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0));
1830 auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1831 if (!TrueOp || !FalseOp)
1833 if (FalseOp->getZExtValue() != 0)
1835 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1838 auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2));
1839 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3));
1840 if (!CCValidOp || !CCMaskOp)
1842 int CCValid = CCValidOp->getZExtValue();
1843 int CCMask = CCMaskOp->getZExtValue();
1846 SDValue CCReg = Node->getOperand(4);
1847 IPMConversion IPM = getIPMConversion(CCValid, CCMask);
1848 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
1851 Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result,
1852 CurDAG->getConstant(IPM.XORValue, DL, MVT::i32));
1855 Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result,
1856 CurDAG->getConstant(IPM.AddValue, DL, MVT::i32));
1858 EVT VT = Node->getValueType(0);
1859 if (VT == MVT::i32 && IPM.Bit == 31) {
1860 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;
1861 Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result,
1862 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1865 Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result);
1867 if (TrueOp->getSExtValue() == 1) {
1868 // The SHR/AND sequence should get optimized to an RISBG.
1869 Result = CurDAG->getNode(ISD::SRL, DL, VT, Result,
1870 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1871 Result = CurDAG->getNode(ISD::AND, DL, VT, Result,
1872 CurDAG->getConstant(1, DL, VT));
1874 // Sign-extend from IPM.Bit using a pair of shifts.
1875 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
1876 int SraAmt = VT.getSizeInBits() - 1;
1877 Result = CurDAG->getNode(ISD::SHL, DL, VT, Result,
1878 CurDAG->getConstant(ShlAmt, DL, MVT::i32));
1879 Result = CurDAG->getNode(ISD::SRA, DL, VT, Result,
1880 CurDAG->getConstant(SraAmt, DL, MVT::i32));
1887 void SystemZDAGToDAGISel::PreprocessISelDAG() {
1888 // If we have conditional immediate loads, we always prefer
1889 // using those over an IPM sequence.
1890 if (Subtarget->hasLoadStoreOnCond2())
1893 bool MadeChange = false;
1895 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
1896 E = CurDAG->allnodes_end();
1903 switch (N->getOpcode()) {
1905 case SystemZISD::SELECT_CCMASK:
1906 Res = expandSelectBoolean(N);
1911 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");
1912 LLVM_DEBUG(N->dump(CurDAG));
1913 LLVM_DEBUG(dbgs() << "\nNew: ");
1914 LLVM_DEBUG(Res.getNode()->dump(CurDAG));
1915 LLVM_DEBUG(dbgs() << "\n");
1917 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1923 CurDAG->RemoveDeadNodes();