1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 /// \file This file contains the AArch64 implementation of the DAG scheduling
10 /// mutation to pair instructions back to back.
12 //===----------------------------------------------------------------------===//
14 #include "AArch64Subtarget.h"
15 #include "llvm/CodeGen/MacroFusion.h"
16 #include "llvm/CodeGen/TargetInstrInfo.h"
22 /// CMN, CMP, TST followed by Bcc
23 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
24 const MachineInstr &SecondMI) {
25 if (SecondMI.getOpcode() != AArch64::Bcc)
28 // Assume the 1st instr to be a wildcard if it is unspecified.
29 if (FirstMI == nullptr)
32 switch (FirstMI->getOpcode()) {
33 case AArch64::ADDSWri:
34 case AArch64::ADDSWrr:
35 case AArch64::ADDSXri:
36 case AArch64::ADDSXrr:
37 case AArch64::ANDSWri:
38 case AArch64::ANDSWrr:
39 case AArch64::ANDSXri:
40 case AArch64::ANDSXrr:
41 case AArch64::SUBSWri:
42 case AArch64::SUBSWrr:
43 case AArch64::SUBSXri:
44 case AArch64::SUBSXrr:
45 case AArch64::BICSWrr:
46 case AArch64::BICSXrr:
48 case AArch64::ADDSWrs:
49 case AArch64::ADDSXrs:
50 case AArch64::ANDSWrs:
51 case AArch64::ANDSXrs:
52 case AArch64::SUBSWrs:
53 case AArch64::SUBSXrs:
54 case AArch64::BICSWrs:
55 case AArch64::BICSXrs:
56 // Shift value can be 0 making these behave like the "rr" variant...
57 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
63 /// ALU operations followed by CBZ/CBNZ.
64 static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
65 const MachineInstr &SecondMI) {
66 if (SecondMI.getOpcode() != AArch64::CBZW &&
67 SecondMI.getOpcode() != AArch64::CBZX &&
68 SecondMI.getOpcode() != AArch64::CBNZW &&
69 SecondMI.getOpcode() != AArch64::CBNZX)
72 // Assume the 1st instr to be a wildcard if it is unspecified.
73 if (FirstMI == nullptr)
76 switch (FirstMI->getOpcode()) {
100 case AArch64::ANDWrs:
101 case AArch64::ANDXrs:
102 case AArch64::SUBWrs:
103 case AArch64::SUBXrs:
104 case AArch64::BICWrs:
105 case AArch64::BICXrs:
106 // Shift value can be 0 making these behave like the "rr" variant...
107 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
113 /// AES crypto encoding or decoding.
114 static bool isAESPair(const MachineInstr *FirstMI,
115 const MachineInstr &SecondMI) {
116 // Assume the 1st instr to be a wildcard if it is unspecified.
117 switch (SecondMI.getOpcode()) {
119 case AArch64::AESMCrr:
120 case AArch64::AESMCrrTied:
121 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
123 case AArch64::AESIMCrr:
124 case AArch64::AESIMCrrTied:
125 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
131 /// AESE/AESD/PMULL + EOR.
132 static bool isCryptoEORPair(const MachineInstr *FirstMI,
133 const MachineInstr &SecondMI) {
134 if (SecondMI.getOpcode() != AArch64::EORv16i8)
137 // Assume the 1st instr to be a wildcard if it is unspecified.
138 if (FirstMI == nullptr)
141 switch (FirstMI->getOpcode()) {
142 case AArch64::AESErr:
143 case AArch64::AESDrr:
144 case AArch64::PMULLv16i8:
145 case AArch64::PMULLv8i8:
146 case AArch64::PMULLv1i64:
147 case AArch64::PMULLv2i64:
154 /// Literal generation.
155 static bool isLiteralsPair(const MachineInstr *FirstMI,
156 const MachineInstr &SecondMI) {
157 // Assume the 1st instr to be a wildcard if it is unspecified.
159 // PC relative address.
160 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
161 SecondMI.getOpcode() == AArch64::ADDXri)
165 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
166 (SecondMI.getOpcode() == AArch64::MOVKWi &&
167 SecondMI.getOperand(3).getImm() == 16))
170 // Lower half of 64 bit immediate.
171 if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
172 (SecondMI.getOpcode() == AArch64::MOVKXi &&
173 SecondMI.getOperand(3).getImm() == 16))
176 // Upper half of 64 bit immediate.
177 if ((FirstMI == nullptr ||
178 (FirstMI->getOpcode() == AArch64::MOVKXi &&
179 FirstMI->getOperand(3).getImm() == 32)) &&
180 (SecondMI.getOpcode() == AArch64::MOVKXi &&
181 SecondMI.getOperand(3).getImm() == 48))
187 /// Fuse address generation and loads or stores.
188 static bool isAddressLdStPair(const MachineInstr *FirstMI,
189 const MachineInstr &SecondMI) {
190 switch (SecondMI.getOpcode()) {
191 case AArch64::STRBBui:
192 case AArch64::STRBui:
193 case AArch64::STRDui:
194 case AArch64::STRHHui:
195 case AArch64::STRHui:
196 case AArch64::STRQui:
197 case AArch64::STRSui:
198 case AArch64::STRWui:
199 case AArch64::STRXui:
200 case AArch64::LDRBBui:
201 case AArch64::LDRBui:
202 case AArch64::LDRDui:
203 case AArch64::LDRHHui:
204 case AArch64::LDRHui:
205 case AArch64::LDRQui:
206 case AArch64::LDRSui:
207 case AArch64::LDRWui:
208 case AArch64::LDRXui:
209 case AArch64::LDRSBWui:
210 case AArch64::LDRSBXui:
211 case AArch64::LDRSHWui:
212 case AArch64::LDRSHXui:
213 case AArch64::LDRSWui:
214 // Assume the 1st instr to be a wildcard if it is unspecified.
215 if (FirstMI == nullptr)
218 switch (FirstMI->getOpcode()) {
220 return SecondMI.getOperand(2).getImm() == 0;
229 /// Compare and conditional select.
230 static bool isCCSelectPair(const MachineInstr *FirstMI,
231 const MachineInstr &SecondMI) {
233 if (SecondMI.getOpcode() == AArch64::CSELWr) {
234 // Assume the 1st instr to be a wildcard if it is unspecified.
235 if (FirstMI == nullptr)
238 if (FirstMI->definesRegister(AArch64::WZR))
239 switch (FirstMI->getOpcode()) {
240 case AArch64::SUBSWrs:
241 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
242 case AArch64::SUBSWrx:
243 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
244 case AArch64::SUBSWrr:
245 case AArch64::SUBSWri:
251 if (SecondMI.getOpcode() == AArch64::CSELXr) {
252 // Assume the 1st instr to be a wildcard if it is unspecified.
253 if (FirstMI == nullptr)
256 if (FirstMI->definesRegister(AArch64::XZR))
257 switch (FirstMI->getOpcode()) {
258 case AArch64::SUBSXrs:
259 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
260 case AArch64::SUBSXrx:
261 case AArch64::SUBSXrx64:
262 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
263 case AArch64::SUBSXrr:
264 case AArch64::SUBSXri:
272 // Arithmetic and logic.
273 static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
274 const MachineInstr &SecondMI) {
275 if (AArch64InstrInfo::hasShiftedReg(SecondMI))
278 switch (SecondMI.getOpcode()) {
280 case AArch64::ADDWrr:
281 case AArch64::ADDXrr:
282 case AArch64::SUBWrr:
283 case AArch64::SUBXrr:
284 case AArch64::ADDWrs:
285 case AArch64::ADDXrs:
286 case AArch64::SUBWrs:
287 case AArch64::SUBXrs:
289 case AArch64::ANDWrr:
290 case AArch64::ANDXrr:
291 case AArch64::BICWrr:
292 case AArch64::BICXrr:
293 case AArch64::EONWrr:
294 case AArch64::EONXrr:
295 case AArch64::EORWrr:
296 case AArch64::EORXrr:
297 case AArch64::ORNWrr:
298 case AArch64::ORNXrr:
299 case AArch64::ORRWrr:
300 case AArch64::ORRXrr:
301 case AArch64::ANDWrs:
302 case AArch64::ANDXrs:
303 case AArch64::BICWrs:
304 case AArch64::BICXrs:
305 case AArch64::EONWrs:
306 case AArch64::EONXrs:
307 case AArch64::EORWrs:
308 case AArch64::EORXrs:
309 case AArch64::ORNWrs:
310 case AArch64::ORNXrs:
311 case AArch64::ORRWrs:
312 case AArch64::ORRXrs:
313 // Assume the 1st instr to be a wildcard if it is unspecified.
314 if (FirstMI == nullptr)
318 switch (FirstMI->getOpcode()) {
319 case AArch64::ADDWrr:
320 case AArch64::ADDXrr:
321 case AArch64::ADDSWrr:
322 case AArch64::ADDSXrr:
323 case AArch64::SUBWrr:
324 case AArch64::SUBXrr:
325 case AArch64::SUBSWrr:
326 case AArch64::SUBSXrr:
328 case AArch64::ADDWrs:
329 case AArch64::ADDXrs:
330 case AArch64::ADDSWrs:
331 case AArch64::ADDSXrs:
332 case AArch64::SUBWrs:
333 case AArch64::SUBXrs:
334 case AArch64::SUBSWrs:
335 case AArch64::SUBSXrs:
336 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
340 // Arithmetic, setting flags.
341 case AArch64::ADDSWrr:
342 case AArch64::ADDSXrr:
343 case AArch64::SUBSWrr:
344 case AArch64::SUBSXrr:
345 case AArch64::ADDSWrs:
346 case AArch64::ADDSXrs:
347 case AArch64::SUBSWrs:
348 case AArch64::SUBSXrs:
349 // Assume the 1st instr to be a wildcard if it is unspecified.
350 if (FirstMI == nullptr)
353 // Arithmetic, not setting flags.
354 switch (FirstMI->getOpcode()) {
355 case AArch64::ADDWrr:
356 case AArch64::ADDXrr:
357 case AArch64::SUBWrr:
358 case AArch64::SUBXrr:
360 case AArch64::ADDWrs:
361 case AArch64::ADDXrs:
362 case AArch64::SUBWrs:
363 case AArch64::SUBXrs:
364 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
372 /// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
373 /// together. Given SecondMI, when FirstMI is unspecified, then check if
374 /// SecondMI may be part of a fused pair at all.
375 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
376 const TargetSubtargetInfo &TSI,
377 const MachineInstr *FirstMI,
378 const MachineInstr &SecondMI) {
379 const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
381 // All checking functions assume that the 1st instr is a wildcard if it is
383 if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI))
385 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
387 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
389 if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
391 if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
393 if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
395 if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
397 if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
408 std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () {
409 return createMacroFusionDAGMutation(shouldScheduleAdjacent);
412 } // end namespace llvm