1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 /// \file This file contains the AArch64 implementation of the DAG scheduling
10 /// mutation to pair instructions back to back.
12 //===----------------------------------------------------------------------===//
14 #include "AArch64MacroFusion.h"
15 #include "AArch64Subtarget.h"
16 #include "llvm/CodeGen/MacroFusion.h"
17 #include "llvm/CodeGen/TargetInstrInfo.h"
21 /// CMN, CMP, TST followed by Bcc
22 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
23 const MachineInstr &SecondMI, bool CmpOnly) {
24 if (SecondMI.getOpcode() != AArch64::Bcc)
27 // Assume the 1st instr to be a wildcard if it is unspecified.
28 if (FirstMI == nullptr)
31 // If we're in CmpOnly mode, we only fuse arithmetic instructions that
32 // discard their result.
33 if (CmpOnly && FirstMI->getOperand(0).isReg() &&
34 !(FirstMI->getOperand(0).getReg() == AArch64::XZR ||
35 FirstMI->getOperand(0).getReg() == AArch64::WZR)) {
39 switch (FirstMI->getOpcode()) {
40 case AArch64::ADDSWri:
41 case AArch64::ADDSWrr:
42 case AArch64::ADDSXri:
43 case AArch64::ADDSXrr:
44 case AArch64::ANDSWri:
45 case AArch64::ANDSWrr:
46 case AArch64::ANDSXri:
47 case AArch64::ANDSXrr:
48 case AArch64::SUBSWri:
49 case AArch64::SUBSWrr:
50 case AArch64::SUBSXri:
51 case AArch64::SUBSXrr:
52 case AArch64::BICSWrr:
53 case AArch64::BICSXrr:
55 case AArch64::ADDSWrs:
56 case AArch64::ADDSXrs:
57 case AArch64::ANDSWrs:
58 case AArch64::ANDSXrs:
59 case AArch64::SUBSWrs:
60 case AArch64::SUBSXrs:
61 case AArch64::BICSWrs:
62 case AArch64::BICSXrs:
63 // Shift value can be 0 making these behave like the "rr" variant...
64 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
70 /// ALU operations followed by CBZ/CBNZ.
71 static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
72 const MachineInstr &SecondMI) {
73 if (SecondMI.getOpcode() != AArch64::CBZW &&
74 SecondMI.getOpcode() != AArch64::CBZX &&
75 SecondMI.getOpcode() != AArch64::CBNZW &&
76 SecondMI.getOpcode() != AArch64::CBNZX)
79 // Assume the 1st instr to be a wildcard if it is unspecified.
80 if (FirstMI == nullptr)
83 switch (FirstMI->getOpcode()) {
100 case AArch64::SUBWri:
101 case AArch64::SUBWrr:
102 case AArch64::SUBXri:
103 case AArch64::SUBXrr:
105 case AArch64::ADDWrs:
106 case AArch64::ADDXrs:
107 case AArch64::ANDWrs:
108 case AArch64::ANDXrs:
109 case AArch64::SUBWrs:
110 case AArch64::SUBXrs:
111 case AArch64::BICWrs:
112 case AArch64::BICXrs:
113 // Shift value can be 0 making these behave like the "rr" variant...
114 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
120 /// AES crypto encoding or decoding.
121 static bool isAESPair(const MachineInstr *FirstMI,
122 const MachineInstr &SecondMI) {
123 // Assume the 1st instr to be a wildcard if it is unspecified.
124 switch (SecondMI.getOpcode()) {
126 case AArch64::AESMCrr:
127 case AArch64::AESMCrrTied:
128 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
130 case AArch64::AESIMCrr:
131 case AArch64::AESIMCrrTied:
132 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
138 /// AESE/AESD/PMULL + EOR.
139 static bool isCryptoEORPair(const MachineInstr *FirstMI,
140 const MachineInstr &SecondMI) {
141 if (SecondMI.getOpcode() != AArch64::EORv16i8)
144 // Assume the 1st instr to be a wildcard if it is unspecified.
145 if (FirstMI == nullptr)
148 switch (FirstMI->getOpcode()) {
149 case AArch64::AESErr:
150 case AArch64::AESDrr:
151 case AArch64::PMULLv16i8:
152 case AArch64::PMULLv8i8:
153 case AArch64::PMULLv1i64:
154 case AArch64::PMULLv2i64:
161 static bool isAdrpAddPair(const MachineInstr *FirstMI,
162 const MachineInstr &SecondMI) {
163 // Assume the 1st instr to be a wildcard if it is unspecified.
164 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
165 SecondMI.getOpcode() == AArch64::ADDXri)
170 /// Literal generation.
171 static bool isLiteralsPair(const MachineInstr *FirstMI,
172 const MachineInstr &SecondMI) {
173 // Assume the 1st instr to be a wildcard if it is unspecified.
175 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
176 (SecondMI.getOpcode() == AArch64::MOVKWi &&
177 SecondMI.getOperand(3).getImm() == 16))
180 // Lower half of 64 bit immediate.
181 if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
182 (SecondMI.getOpcode() == AArch64::MOVKXi &&
183 SecondMI.getOperand(3).getImm() == 16))
186 // Upper half of 64 bit immediate.
187 if ((FirstMI == nullptr ||
188 (FirstMI->getOpcode() == AArch64::MOVKXi &&
189 FirstMI->getOperand(3).getImm() == 32)) &&
190 (SecondMI.getOpcode() == AArch64::MOVKXi &&
191 SecondMI.getOperand(3).getImm() == 48))
197 /// Fuse address generation and loads or stores.
198 static bool isAddressLdStPair(const MachineInstr *FirstMI,
199 const MachineInstr &SecondMI) {
200 switch (SecondMI.getOpcode()) {
201 case AArch64::STRBBui:
202 case AArch64::STRBui:
203 case AArch64::STRDui:
204 case AArch64::STRHHui:
205 case AArch64::STRHui:
206 case AArch64::STRQui:
207 case AArch64::STRSui:
208 case AArch64::STRWui:
209 case AArch64::STRXui:
210 case AArch64::LDRBBui:
211 case AArch64::LDRBui:
212 case AArch64::LDRDui:
213 case AArch64::LDRHHui:
214 case AArch64::LDRHui:
215 case AArch64::LDRQui:
216 case AArch64::LDRSui:
217 case AArch64::LDRWui:
218 case AArch64::LDRXui:
219 case AArch64::LDRSBWui:
220 case AArch64::LDRSBXui:
221 case AArch64::LDRSHWui:
222 case AArch64::LDRSHXui:
223 case AArch64::LDRSWui:
224 // Assume the 1st instr to be a wildcard if it is unspecified.
225 if (FirstMI == nullptr)
228 switch (FirstMI->getOpcode()) {
230 return SecondMI.getOperand(2).getImm() == 0;
239 /// Compare and conditional select.
240 static bool isCCSelectPair(const MachineInstr *FirstMI,
241 const MachineInstr &SecondMI) {
243 if (SecondMI.getOpcode() == AArch64::CSELWr) {
244 // Assume the 1st instr to be a wildcard if it is unspecified.
245 if (FirstMI == nullptr)
248 if (FirstMI->definesRegister(AArch64::WZR))
249 switch (FirstMI->getOpcode()) {
250 case AArch64::SUBSWrs:
251 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
252 case AArch64::SUBSWrx:
253 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
254 case AArch64::SUBSWrr:
255 case AArch64::SUBSWri:
261 if (SecondMI.getOpcode() == AArch64::CSELXr) {
262 // Assume the 1st instr to be a wildcard if it is unspecified.
263 if (FirstMI == nullptr)
266 if (FirstMI->definesRegister(AArch64::XZR))
267 switch (FirstMI->getOpcode()) {
268 case AArch64::SUBSXrs:
269 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
270 case AArch64::SUBSXrx:
271 case AArch64::SUBSXrx64:
272 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
273 case AArch64::SUBSXrr:
274 case AArch64::SUBSXri:
282 // Arithmetic and logic.
283 static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
284 const MachineInstr &SecondMI) {
285 if (AArch64InstrInfo::hasShiftedReg(SecondMI))
288 switch (SecondMI.getOpcode()) {
290 case AArch64::ADDWrr:
291 case AArch64::ADDXrr:
292 case AArch64::SUBWrr:
293 case AArch64::SUBXrr:
294 case AArch64::ADDWrs:
295 case AArch64::ADDXrs:
296 case AArch64::SUBWrs:
297 case AArch64::SUBXrs:
299 case AArch64::ANDWrr:
300 case AArch64::ANDXrr:
301 case AArch64::BICWrr:
302 case AArch64::BICXrr:
303 case AArch64::EONWrr:
304 case AArch64::EONXrr:
305 case AArch64::EORWrr:
306 case AArch64::EORXrr:
307 case AArch64::ORNWrr:
308 case AArch64::ORNXrr:
309 case AArch64::ORRWrr:
310 case AArch64::ORRXrr:
311 case AArch64::ANDWrs:
312 case AArch64::ANDXrs:
313 case AArch64::BICWrs:
314 case AArch64::BICXrs:
315 case AArch64::EONWrs:
316 case AArch64::EONXrs:
317 case AArch64::EORWrs:
318 case AArch64::EORXrs:
319 case AArch64::ORNWrs:
320 case AArch64::ORNXrs:
321 case AArch64::ORRWrs:
322 case AArch64::ORRXrs:
323 // Assume the 1st instr to be a wildcard if it is unspecified.
324 if (FirstMI == nullptr)
328 switch (FirstMI->getOpcode()) {
329 case AArch64::ADDWrr:
330 case AArch64::ADDXrr:
331 case AArch64::ADDSWrr:
332 case AArch64::ADDSXrr:
333 case AArch64::SUBWrr:
334 case AArch64::SUBXrr:
335 case AArch64::SUBSWrr:
336 case AArch64::SUBSXrr:
338 case AArch64::ADDWrs:
339 case AArch64::ADDXrs:
340 case AArch64::ADDSWrs:
341 case AArch64::ADDSXrs:
342 case AArch64::SUBWrs:
343 case AArch64::SUBXrs:
344 case AArch64::SUBSWrs:
345 case AArch64::SUBSXrs:
346 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
350 // Arithmetic, setting flags.
351 case AArch64::ADDSWrr:
352 case AArch64::ADDSXrr:
353 case AArch64::SUBSWrr:
354 case AArch64::SUBSXrr:
355 case AArch64::ADDSWrs:
356 case AArch64::ADDSXrs:
357 case AArch64::SUBSWrs:
358 case AArch64::SUBSXrs:
359 // Assume the 1st instr to be a wildcard if it is unspecified.
360 if (FirstMI == nullptr)
363 // Arithmetic, not setting flags.
364 switch (FirstMI->getOpcode()) {
365 case AArch64::ADDWrr:
366 case AArch64::ADDXrr:
367 case AArch64::SUBWrr:
368 case AArch64::SUBXrr:
370 case AArch64::ADDWrs:
371 case AArch64::ADDXrs:
372 case AArch64::SUBWrs:
373 case AArch64::SUBXrs:
374 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
382 // "(A + B) + 1" or "(A - B) - 1"
383 static bool isAddSub2RegAndConstOnePair(const MachineInstr *FirstMI,
384 const MachineInstr &SecondMI) {
385 bool NeedsSubtract = false;
387 // The 2nd instr must be an add-immediate or subtract-immediate.
388 switch (SecondMI.getOpcode()) {
389 case AArch64::SUBWri:
390 case AArch64::SUBXri:
391 NeedsSubtract = true;
393 case AArch64::ADDWri:
394 case AArch64::ADDXri:
401 // The immediate in the 2nd instr must be "1".
402 if (!SecondMI.getOperand(2).isImm() || SecondMI.getOperand(2).getImm() != 1) {
406 // Assume the 1st instr to be a wildcard if it is unspecified.
407 if (FirstMI == nullptr) {
411 switch (FirstMI->getOpcode()) {
412 case AArch64::SUBWrs:
413 case AArch64::SUBXrs:
414 if (AArch64InstrInfo::hasShiftedReg(*FirstMI))
417 case AArch64::SUBWrr:
418 case AArch64::SUBXrr:
424 case AArch64::ADDWrs:
425 case AArch64::ADDXrs:
426 if (AArch64InstrInfo::hasShiftedReg(*FirstMI))
429 case AArch64::ADDWrr:
430 case AArch64::ADDXrr:
431 if (!NeedsSubtract) {
440 /// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
441 /// together. Given SecondMI, when FirstMI is unspecified, then check if
442 /// SecondMI may be part of a fused pair at all.
443 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
444 const TargetSubtargetInfo &TSI,
445 const MachineInstr *FirstMI,
446 const MachineInstr &SecondMI) {
447 const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
449 // All checking functions assume that the 1st instr is a wildcard if it is
451 if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {
452 bool CmpOnly = !ST.hasArithmeticBccFusion();
453 if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly))
456 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
458 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
460 if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
462 if (ST.hasFuseAdrpAdd() && isAdrpAddPair(FirstMI, SecondMI))
464 if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
466 if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
468 if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
470 if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
472 if (ST.hasFuseAddSub2RegAndConstOne() &&
473 isAddSub2RegAndConstOnePair(FirstMI, SecondMI))
479 std::unique_ptr<ScheduleDAGMutation>
480 llvm::createAArch64MacroFusionDAGMutation() {
481 return createMacroFusionDAGMutation(shouldScheduleAdjacent);