1 //===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file defines all of the AARCH64-specific intrinsics.
11 //===----------------------------------------------------------------------===//
13 let TargetPrefix = "aarch64" in {
15 def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
16 def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
17 def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
18 def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
20 def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
21 def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
22 def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
23 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
24 def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
25 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
27 def int_aarch64_clrex : Intrinsic<[]>;
29 def int_aarch64_sdiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
30 LLVMMatchType<0>], [IntrNoMem]>;
31 def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
32 LLVMMatchType<0>], [IntrNoMem]>;
34 def int_aarch64_fjcvtzs : Intrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>;
36 def int_aarch64_cls: Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
37 def int_aarch64_cls64: Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
39 //===----------------------------------------------------------------------===//
42 def int_aarch64_hint : Intrinsic<[], [llvm_i32_ty]>;
44 //===----------------------------------------------------------------------===//
45 // Data Barrier Instructions
47 def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>;
48 def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>;
49 def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>;
51 // A space-consuming intrinsic primarily for testing block and jump table
52 // placements. The first argument is the number of bytes this "instruction"
53 // takes up, the second and return value are essentially chains, used to force
54 // ordering during ISel.
55 def int_aarch64_space : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;
59 //===----------------------------------------------------------------------===//
60 // Advanced SIMD (NEON)
62 let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.".
63 class AdvSIMD_2Scalar_Float_Intrinsic
64 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
67 class AdvSIMD_FPToIntRounding_Intrinsic
68 : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
70 class AdvSIMD_1IntArg_Intrinsic
71 : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
72 class AdvSIMD_1FloatArg_Intrinsic
73 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
74 class AdvSIMD_1VectorArg_Intrinsic
75 : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
76 class AdvSIMD_1VectorArg_Expand_Intrinsic
77 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
78 class AdvSIMD_1VectorArg_Long_Intrinsic
79 : Intrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
80 class AdvSIMD_1IntArg_Narrow_Intrinsic
81 : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
82 class AdvSIMD_1VectorArg_Narrow_Intrinsic
83 : Intrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
84 class AdvSIMD_1VectorArg_Int_Across_Intrinsic
85 : Intrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
86 class AdvSIMD_1VectorArg_Float_Across_Intrinsic
87 : Intrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;
89 class AdvSIMD_2IntArg_Intrinsic
90 : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
92 class AdvSIMD_2FloatArg_Intrinsic
93 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
95 class AdvSIMD_2VectorArg_Intrinsic
96 : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
98 class AdvSIMD_2VectorArg_Compare_Intrinsic
99 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
101 class AdvSIMD_2Arg_FloatCompare_Intrinsic
102 : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
104 class AdvSIMD_2VectorArg_Long_Intrinsic
105 : Intrinsic<[llvm_anyvector_ty],
106 [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
108 class AdvSIMD_2VectorArg_Wide_Intrinsic
109 : Intrinsic<[llvm_anyvector_ty],
110 [LLVMMatchType<0>, LLVMTruncatedType<0>],
112 class AdvSIMD_2VectorArg_Narrow_Intrinsic
113 : Intrinsic<[llvm_anyvector_ty],
114 [LLVMExtendedType<0>, LLVMExtendedType<0>],
116 class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
117 : Intrinsic<[llvm_anyint_ty],
118 [LLVMExtendedType<0>, llvm_i32_ty],
120 class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
121 : Intrinsic<[llvm_anyvector_ty],
124 class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
125 : Intrinsic<[llvm_anyvector_ty],
126 [LLVMTruncatedType<0>],
128 class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
129 : Intrinsic<[llvm_anyvector_ty],
130 [LLVMTruncatedType<0>, llvm_i32_ty],
132 class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
133 : Intrinsic<[llvm_anyvector_ty],
134 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
136 class AdvSIMD_2VectorArg_Lane_Intrinsic
137 : Intrinsic<[llvm_anyint_ty],
138 [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty],
141 class AdvSIMD_3VectorArg_Intrinsic
142 : Intrinsic<[llvm_anyvector_ty],
143 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
145 class AdvSIMD_3VectorArg_Scalar_Intrinsic
146 : Intrinsic<[llvm_anyvector_ty],
147 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
149 class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
150 : Intrinsic<[llvm_anyvector_ty],
151 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
152 LLVMMatchType<1>], [IntrNoMem]>;
153 class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
154 : Intrinsic<[llvm_anyvector_ty],
155 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
157 class AdvSIMD_CvtFxToFP_Intrinsic
158 : Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
160 class AdvSIMD_CvtFPToFx_Intrinsic
161 : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
164 class AdvSIMD_1Arg_Intrinsic
165 : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;
167 class AdvSIMD_Dot_Intrinsic
168 : Intrinsic<[llvm_anyvector_ty],
169 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
172 class AdvSIMD_FP16FML_Intrinsic
173 : Intrinsic<[llvm_anyvector_ty],
174 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
177 class AdvSIMD_MatMul_Intrinsic
178 : Intrinsic<[llvm_anyvector_ty],
179 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
182 class AdvSIMD_FML_Intrinsic
183 : Intrinsic<[llvm_anyvector_ty],
184 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
191 let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
192 // Vector Add Across Lanes
193 def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
194 def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
195 def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
197 // Vector Long Add Across Lanes
198 def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
199 def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
201 // Vector Halving Add
202 def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
203 def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;
205 // Vector Rounding Halving Add
206 def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
207 def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;
209 // Vector Saturating Add
210 def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
211 def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
212 def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
213 def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;
215 // Vector Add High-Half
216 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
217 // header is no longer supported.
218 def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
220 // Vector Rounding Add High-Half
221 def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
223 // Vector Saturating Doubling Multiply High
224 def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
225 def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
226 def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
228 // Vector Saturating Rounding Doubling Multiply High
229 def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
230 def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
231 def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
233 // Vector Polynominal Multiply
234 def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
236 // Vector Long Multiply
237 def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
238 def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
239 def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
241 // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
243 def int_aarch64_neon_pmull64 :
244 Intrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
246 // Vector Extending Multiply
247 def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
248 let IntrProperties = [IntrNoMem, Commutative];
251 // Vector Saturating Doubling Long Multiply
252 def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
253 def int_aarch64_neon_sqdmulls_scalar
254 : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
256 // Vector Halving Subtract
257 def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
258 def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;
260 // Vector Saturating Subtract
261 def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
262 def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;
264 // Vector Subtract High-Half
265 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
266 // header is no longer supported.
267 def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
269 // Vector Rounding Subtract High-Half
270 def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
272 // Vector Compare Absolute Greater-than-or-equal
273 def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;
275 // Vector Compare Absolute Greater-than
276 def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;
278 // Vector Absolute Difference
279 def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
280 def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
281 def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;
283 // Scalar Absolute Difference
284 def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;
287 def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
288 def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
289 def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;
290 def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;
292 // Vector Max Across Lanes
293 def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
294 def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
295 def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
296 def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
299 def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
300 def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
301 def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;
302 def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;
304 // Vector Min/Max Number
305 def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
306 def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;
308 // Vector Min Across Lanes
309 def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
310 def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
311 def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
312 def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
315 def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
316 def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic;
319 // FIXME: In theory, we shouldn't need intrinsics for saddlp or
320 // uaddlp, but tblgen's type inference currently can't handle the
321 // pattern fragments this ends up generating.
322 def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
323 def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
326 def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
327 def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
328 def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;
331 def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
332 def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
333 def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;
335 // Reciprocal Estimate/Step
336 def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
337 def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;
339 // Reciprocal Exponent
340 def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;
342 // Vector Saturating Shift Left
343 def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
344 def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;
346 // Vector Rounding Shift Left
347 def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
348 def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;
350 // Vector Saturating Rounding Shift Left
351 def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
352 def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;
354 // Vector Signed->Unsigned Shift Left by Constant
355 def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;
357 // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
358 def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
360 // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
361 def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
363 // Vector Narrowing Shift Right by Constant
364 def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
365 def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
367 // Vector Rounding Narrowing Shift Right by Constant
368 def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
370 // Vector Rounding Narrowing Saturating Shift Right by Constant
371 def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
372 def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
375 def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
376 def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;
378 // Vector Widening Shift Left by Constant
379 def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
380 def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
381 def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
383 // Vector Shift Right by Constant and Insert
384 def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;
386 // Vector Shift Left by Constant and Insert
387 def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;
389 // Vector Saturating Narrow
390 def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
391 def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
392 def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
393 def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
395 // Vector Saturating Extract and Unsigned Narrow
396 def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
397 def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;
399 // Vector Absolute Value
400 def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;
402 // Vector Saturating Absolute Value
403 def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;
405 // Vector Saturating Negation
406 def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;
408 // Vector Count Leading Sign Bits
409 def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;
411 // Vector Reciprocal Estimate
412 def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
413 def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;
415 // Vector Square Root Estimate
416 def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
417 def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;
419 // Vector Bitwise Reverse
420 def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic;
422 // Vector Conversions Between Half-Precision and Single-Precision.
423 def int_aarch64_neon_vcvtfp2hf
424 : Intrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
425 def int_aarch64_neon_vcvthf2fp
426 : Intrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;
428 // Vector Conversions Between Floating-point and Fixed-point.
429 def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
430 def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
431 def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
432 def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;
434 // Vector FP->Int Conversions
435 def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
436 def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
437 def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
438 def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
439 def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
440 def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
441 def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
442 def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
443 def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
444 def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
446 // Vector FP Rounding: only ties to even is unrepresented by a normal
448 def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic;
450 // Scalar FP->Int conversions
452 // Vector FP Inexact Narrowing
453 def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;
455 // Scalar FP Inexact Narrowing
456 def int_aarch64_sisd_fcvtxn : Intrinsic<[llvm_float_ty], [llvm_double_ty],
459 // v8.2-A Dot Product
460 def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;
461 def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;
463 // v8.6-A Matrix Multiply Intrinsics
464 def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic;
465 def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic;
466 def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic;
467 def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic;
468 def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic;
469 def int_aarch64_neon_bfmmla : AdvSIMD_MatMul_Intrinsic;
470 def int_aarch64_neon_bfmlalb : AdvSIMD_FML_Intrinsic;
471 def int_aarch64_neon_bfmlalt : AdvSIMD_FML_Intrinsic;
474 // v8.6-A Bfloat Intrinsics
475 def int_aarch64_neon_bfcvt
476 : Intrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>;
477 def int_aarch64_neon_bfcvtn
478 : Intrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
479 def int_aarch64_neon_bfcvtn2
480 : Intrinsic<[llvm_v8bf16_ty],
481 [llvm_v8bf16_ty, llvm_v4f32_ty],
484 // v8.2-A FP16 Fused Multiply-Add Long
485 def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
486 def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
487 def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;
488 def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;
490 // v8.3-A Floating-point complex add
491 def int_aarch64_neon_vcadd_rot90 : AdvSIMD_2VectorArg_Intrinsic;
492 def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic;
495 let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.".
496 class AdvSIMD_2Vector2Index_Intrinsic
497 : Intrinsic<[llvm_anyvector_ty],
498 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
502 // Vector element to element moves
503 def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;
505 let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.".
506 class AdvSIMD_1Vec_Load_Intrinsic
507 : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
508 [IntrReadMem, IntrArgMemOnly]>;
509 class AdvSIMD_1Vec_Store_Lane_Intrinsic
510 : Intrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
511 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
513 class AdvSIMD_2Vec_Load_Intrinsic
514 : Intrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],
515 [LLVMAnyPointerType<LLVMMatchType<0>>],
516 [IntrReadMem, IntrArgMemOnly]>;
517 class AdvSIMD_2Vec_Load_Lane_Intrinsic
518 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
519 [LLVMMatchType<0>, llvm_anyvector_ty,
520 llvm_i64_ty, llvm_anyptr_ty],
521 [IntrReadMem, IntrArgMemOnly]>;
522 class AdvSIMD_2Vec_Store_Intrinsic
523 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
524 LLVMAnyPointerType<LLVMMatchType<0>>],
525 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
526 class AdvSIMD_2Vec_Store_Lane_Intrinsic
527 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
528 llvm_i64_ty, llvm_anyptr_ty],
529 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
531 class AdvSIMD_3Vec_Load_Intrinsic
532 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
533 [LLVMAnyPointerType<LLVMMatchType<0>>],
534 [IntrReadMem, IntrArgMemOnly]>;
535 class AdvSIMD_3Vec_Load_Lane_Intrinsic
536 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
537 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty,
538 llvm_i64_ty, llvm_anyptr_ty],
539 [IntrReadMem, IntrArgMemOnly]>;
540 class AdvSIMD_3Vec_Store_Intrinsic
541 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
542 LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
543 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
544 class AdvSIMD_3Vec_Store_Lane_Intrinsic
545 : Intrinsic<[], [llvm_anyvector_ty,
546 LLVMMatchType<0>, LLVMMatchType<0>,
547 llvm_i64_ty, llvm_anyptr_ty],
548 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
550 class AdvSIMD_4Vec_Load_Intrinsic
551 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
552 LLVMMatchType<0>, llvm_anyvector_ty],
553 [LLVMAnyPointerType<LLVMMatchType<0>>],
554 [IntrReadMem, IntrArgMemOnly]>;
555 class AdvSIMD_4Vec_Load_Lane_Intrinsic
556 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
557 LLVMMatchType<0>, LLVMMatchType<0>],
558 [LLVMMatchType<0>, LLVMMatchType<0>,
559 LLVMMatchType<0>, llvm_anyvector_ty,
560 llvm_i64_ty, llvm_anyptr_ty],
561 [IntrReadMem, IntrArgMemOnly]>;
562 class AdvSIMD_4Vec_Store_Intrinsic
563 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
564 LLVMMatchType<0>, LLVMMatchType<0>,
565 LLVMAnyPointerType<LLVMMatchType<0>>],
566 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
567 class AdvSIMD_4Vec_Store_Lane_Intrinsic
568 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
569 LLVMMatchType<0>, LLVMMatchType<0>,
570 llvm_i64_ty, llvm_anyptr_ty],
571 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
576 def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
577 def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
578 def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;
580 def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
581 def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
582 def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;
584 def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
585 def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
586 def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;
588 def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
589 def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
590 def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;
592 def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
593 def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
594 def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;
596 def int_aarch64_neon_st2 : AdvSIMD_2Vec_Store_Intrinsic;
597 def int_aarch64_neon_st3 : AdvSIMD_3Vec_Store_Intrinsic;
598 def int_aarch64_neon_st4 : AdvSIMD_4Vec_Store_Intrinsic;
600 def int_aarch64_neon_st2lane : AdvSIMD_2Vec_Store_Lane_Intrinsic;
601 def int_aarch64_neon_st3lane : AdvSIMD_3Vec_Store_Lane_Intrinsic;
602 def int_aarch64_neon_st4lane : AdvSIMD_4Vec_Store_Lane_Intrinsic;
604 let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.".
605 class AdvSIMD_Tbl1_Intrinsic
606 : Intrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
608 class AdvSIMD_Tbl2_Intrinsic
609 : Intrinsic<[llvm_anyvector_ty],
610 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
611 class AdvSIMD_Tbl3_Intrinsic
612 : Intrinsic<[llvm_anyvector_ty],
613 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
616 class AdvSIMD_Tbl4_Intrinsic
617 : Intrinsic<[llvm_anyvector_ty],
618 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
622 class AdvSIMD_Tbx1_Intrinsic
623 : Intrinsic<[llvm_anyvector_ty],
624 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
626 class AdvSIMD_Tbx2_Intrinsic
627 : Intrinsic<[llvm_anyvector_ty],
628 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
631 class AdvSIMD_Tbx3_Intrinsic
632 : Intrinsic<[llvm_anyvector_ty],
633 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
634 llvm_v16i8_ty, LLVMMatchType<0>],
636 class AdvSIMD_Tbx4_Intrinsic
637 : Intrinsic<[llvm_anyvector_ty],
638 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
639 llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
642 def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
643 def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
644 def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
645 def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;
647 def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
648 def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
649 def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
650 def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
652 let TargetPrefix = "aarch64" in {
653 class FPCR_Get_Intrinsic
654 : Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>;
658 def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
660 let TargetPrefix = "aarch64" in {
661 class Crypto_AES_DataKey_Intrinsic
662 : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
664 class Crypto_AES_Data_Intrinsic
665 : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
667 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
669 class Crypto_SHA_5Hash4Schedule_Intrinsic
670 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
673 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
675 class Crypto_SHA_1Hash_Intrinsic
676 : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
678 // SHA intrinsic taking 8 words of the schedule
679 class Crypto_SHA_8Schedule_Intrinsic
680 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
682 // SHA intrinsic taking 12 words of the schedule
683 class Crypto_SHA_12Schedule_Intrinsic
684 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
687 // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
688 class Crypto_SHA_8Hash4Schedule_Intrinsic
689 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
694 def int_aarch64_crypto_aese : Crypto_AES_DataKey_Intrinsic;
695 def int_aarch64_crypto_aesd : Crypto_AES_DataKey_Intrinsic;
696 def int_aarch64_crypto_aesmc : Crypto_AES_Data_Intrinsic;
697 def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;
700 def int_aarch64_crypto_sha1c : Crypto_SHA_5Hash4Schedule_Intrinsic;
701 def int_aarch64_crypto_sha1p : Crypto_SHA_5Hash4Schedule_Intrinsic;
702 def int_aarch64_crypto_sha1m : Crypto_SHA_5Hash4Schedule_Intrinsic;
703 def int_aarch64_crypto_sha1h : Crypto_SHA_1Hash_Intrinsic;
705 def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
706 def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;
709 def int_aarch64_crypto_sha256h : Crypto_SHA_8Hash4Schedule_Intrinsic;
710 def int_aarch64_crypto_sha256h2 : Crypto_SHA_8Hash4Schedule_Intrinsic;
711 def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
712 def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;
714 //===----------------------------------------------------------------------===//
717 let TargetPrefix = "aarch64" in {
719 def int_aarch64_crc32b : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
721 def int_aarch64_crc32cb : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
723 def int_aarch64_crc32h : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
725 def int_aarch64_crc32ch : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
727 def int_aarch64_crc32w : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
729 def int_aarch64_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
731 def int_aarch64_crc32x : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
733 def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
737 //===----------------------------------------------------------------------===//
738 // Memory Tagging Extensions (MTE) Intrinsics
739 let TargetPrefix = "aarch64" in {
740 def int_aarch64_irg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
741 [IntrNoMem, IntrHasSideEffects]>;
742 def int_aarch64_addg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
744 def int_aarch64_gmi : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty],
746 def int_aarch64_ldg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty],
748 def int_aarch64_stg : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
750 def int_aarch64_subp : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
753 // The following are codegen-only intrinsics for stack instrumentation.
755 // Generate a randomly tagged stack base pointer.
756 def int_aarch64_irg_sp : Intrinsic<[llvm_ptr_ty], [llvm_i64_ty],
757 [IntrNoMem, IntrHasSideEffects]>;
759 // Transfer pointer tag with offset.
760 // ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where
761 // * address is the address in ptr0
762 // * tag is a function of (tag in baseptr, tag_offset).
763 // Address bits in baseptr and tag bits in ptr0 are ignored.
764 // When offset between ptr0 and baseptr is a compile time constant, this can be emitted as
765 // ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset
766 // It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp.
767 def int_aarch64_tagp : Intrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty],
768 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
770 // Update allocation tags for the memory range to match the tag in the pointer argument.
771 def int_aarch64_settag : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
772 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
774 // Update allocation tags for the memory range to match the tag in the pointer argument,
775 // and set memory contents to zero.
776 def int_aarch64_settag_zero : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
777 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
779 // Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values.
780 def int_aarch64_stgp : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
781 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
784 // Transactional Memory Extension (TME) Intrinsics
785 let TargetPrefix = "aarch64" in {
786 def int_aarch64_tstart : GCCBuiltin<"__builtin_arm_tstart">,
787 Intrinsic<[llvm_i64_ty]>;
789 def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[]>;
791 def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">,
792 Intrinsic<[], [llvm_i64_ty], [ImmArg<ArgIndex<0>>]>;
794 def int_aarch64_ttest : GCCBuiltin<"__builtin_arm_ttest">,
795 Intrinsic<[llvm_i64_ty], [],
796 [IntrNoMem, IntrHasSideEffects]>;
799 def llvm_nxv2i1_ty : LLVMType<nxv2i1>;
800 def llvm_nxv4i1_ty : LLVMType<nxv4i1>;
801 def llvm_nxv8i1_ty : LLVMType<nxv8i1>;
802 def llvm_nxv16i1_ty : LLVMType<nxv16i1>;
803 def llvm_nxv16i8_ty : LLVMType<nxv16i8>;
804 def llvm_nxv4i32_ty : LLVMType<nxv4i32>;
805 def llvm_nxv2i64_ty : LLVMType<nxv2i64>;
806 def llvm_nxv8f16_ty : LLVMType<nxv8f16>;
807 def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>;
808 def llvm_nxv4f32_ty : LLVMType<nxv4f32>;
809 def llvm_nxv2f64_ty : LLVMType<nxv2f64>;
811 let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.".
813 class AdvSIMD_SVE_Create_2Vector_Tuple
814 : Intrinsic<[llvm_anyvector_ty],
815 [llvm_anyvector_ty, LLVMMatchType<1>],
818 class AdvSIMD_SVE_Create_3Vector_Tuple
819 : Intrinsic<[llvm_anyvector_ty],
820 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>],
823 class AdvSIMD_SVE_Create_4Vector_Tuple
824 : Intrinsic<[llvm_anyvector_ty],
825 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
829 class AdvSIMD_SVE_Set_Vector_Tuple
830 : Intrinsic<[llvm_anyvector_ty],
831 [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty],
832 [IntrReadMem, ImmArg<ArgIndex<1>>]>;
834 class AdvSIMD_SVE_Get_Vector_Tuple
835 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty],
836 [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
838 class AdvSIMD_ManyVec_PredLoad_Intrinsic
839 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMPointerToElt<0>],
840 [IntrReadMem, IntrArgMemOnly]>;
842 class AdvSIMD_1Vec_PredLoad_Intrinsic
843 : Intrinsic<[llvm_anyvector_ty],
844 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
845 LLVMPointerToElt<0>],
846 [IntrReadMem, IntrArgMemOnly]>;
848 class AdvSIMD_1Vec_PredStore_Intrinsic
851 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
852 LLVMPointerToElt<0>],
853 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
855 class AdvSIMD_2Vec_PredStore_Intrinsic
857 [llvm_anyvector_ty, LLVMMatchType<0>,
858 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
859 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
861 class AdvSIMD_3Vec_PredStore_Intrinsic
863 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
864 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
865 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
867 class AdvSIMD_4Vec_PredStore_Intrinsic
869 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
871 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
872 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
874 class AdvSIMD_SVE_Index_Intrinsic
875 : Intrinsic<[llvm_anyvector_ty],
876 [LLVMVectorElementType<0>,
877 LLVMVectorElementType<0>],
880 class AdvSIMD_Merged1VectorArg_Intrinsic
881 : Intrinsic<[llvm_anyvector_ty],
883 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
887 class AdvSIMD_2VectorArgIndexed_Intrinsic
888 : Intrinsic<[llvm_anyvector_ty],
892 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
894 class AdvSIMD_3VectorArgIndexed_Intrinsic
895 : Intrinsic<[llvm_anyvector_ty],
900 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
902 class AdvSIMD_Pred1VectorArg_Intrinsic
903 : Intrinsic<[llvm_anyvector_ty],
904 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
908 class AdvSIMD_Pred2VectorArg_Intrinsic
909 : Intrinsic<[llvm_anyvector_ty],
910 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
915 class AdvSIMD_Pred3VectorArg_Intrinsic
916 : Intrinsic<[llvm_anyvector_ty],
917 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
923 class AdvSIMD_SVE_Compare_Intrinsic
924 : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
925 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
930 class AdvSIMD_SVE_CompareWide_Intrinsic
931 : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
932 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
937 class AdvSIMD_SVE_Saturating_Intrinsic
938 : Intrinsic<[llvm_anyvector_ty],
940 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
943 class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic
944 : Intrinsic<[llvm_anyvector_ty],
948 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
950 class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T>
952 [T, llvm_anyvector_ty],
955 class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T>
957 [T, llvm_i32_ty, llvm_i32_ty],
958 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
960 class AdvSIMD_SVE_CNT_Intrinsic
961 : Intrinsic<[LLVMVectorOfBitcastsToInt<0>],
962 [LLVMVectorOfBitcastsToInt<0>,
963 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
967 class AdvSIMD_SVE_ReduceWithInit_Intrinsic
968 : Intrinsic<[LLVMVectorElementType<0>],
969 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
970 LLVMVectorElementType<0>,
974 class AdvSIMD_SVE_ShiftByImm_Intrinsic
975 : Intrinsic<[llvm_anyvector_ty],
976 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
979 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
981 class AdvSIMD_SVE_ShiftWide_Intrinsic
982 : Intrinsic<[llvm_anyvector_ty],
983 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
988 class AdvSIMD_SVE_Unpack_Intrinsic
989 : Intrinsic<[llvm_anyvector_ty],
990 [LLVMSubdivide2VectorType<0>],
993 class AdvSIMD_SVE_CADD_Intrinsic
994 : Intrinsic<[llvm_anyvector_ty],
995 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
999 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1001 class AdvSIMD_SVE_CMLA_Intrinsic
1002 : Intrinsic<[llvm_anyvector_ty],
1003 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1008 [IntrNoMem, ImmArg<ArgIndex<4>>]>;
1010 class AdvSIMD_SVE_CMLA_LANE_Intrinsic
1011 : Intrinsic<[llvm_anyvector_ty],
1017 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1019 class AdvSIMD_SVE_DUP_Intrinsic
1020 : Intrinsic<[llvm_anyvector_ty],
1022 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1023 LLVMVectorElementType<0>],
1026 class AdvSIMD_SVE_DUP_Unpred_Intrinsic
1027 : Intrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>],
1030 class AdvSIMD_SVE_DUPQ_Intrinsic
1031 : Intrinsic<[llvm_anyvector_ty],
1036 class AdvSIMD_SVE_EXPA_Intrinsic
1037 : Intrinsic<[llvm_anyvector_ty],
1038 [LLVMVectorOfBitcastsToInt<0>],
1041 class AdvSIMD_SVE_FCVT_Intrinsic
1042 : Intrinsic<[llvm_anyvector_ty],
1044 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1048 class AdvSIMD_SVE_FCVTZS_Intrinsic
1049 : Intrinsic<[llvm_anyvector_ty],
1050 [LLVMVectorOfBitcastsToInt<0>,
1051 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1055 class AdvSIMD_SVE_INSR_Intrinsic
1056 : Intrinsic<[llvm_anyvector_ty],
1058 LLVMVectorElementType<0>],
1061 class AdvSIMD_SVE_PTRUE_Intrinsic
1062 : Intrinsic<[llvm_anyvector_ty],
1064 [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1066 class AdvSIMD_SVE_PUNPKHI_Intrinsic
1067 : Intrinsic<[LLVMHalfElementsVectorType<0>],
1068 [llvm_anyvector_ty],
1071 class AdvSIMD_SVE_SCALE_Intrinsic
1072 : Intrinsic<[llvm_anyvector_ty],
1073 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1075 LLVMVectorOfBitcastsToInt<0>],
1078 class AdvSIMD_SVE_SCVTF_Intrinsic
1079 : Intrinsic<[llvm_anyvector_ty],
1081 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1085 class AdvSIMD_SVE_TSMUL_Intrinsic
1086 : Intrinsic<[llvm_anyvector_ty],
1088 LLVMVectorOfBitcastsToInt<0>],
1091 class AdvSIMD_SVE_CNTB_Intrinsic
1092 : Intrinsic<[llvm_i64_ty],
1094 [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1096 class AdvSIMD_SVE_CNTP_Intrinsic
1097 : Intrinsic<[llvm_i64_ty],
1098 [llvm_anyvector_ty, LLVMMatchType<0>],
1101 class AdvSIMD_SVE_DOT_Intrinsic
1102 : Intrinsic<[llvm_anyvector_ty],
1104 LLVMSubdivide4VectorType<0>,
1105 LLVMSubdivide4VectorType<0>],
1108 class AdvSIMD_SVE_DOT_Indexed_Intrinsic
1109 : Intrinsic<[llvm_anyvector_ty],
1111 LLVMSubdivide4VectorType<0>,
1112 LLVMSubdivide4VectorType<0>,
1114 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1116 class AdvSIMD_SVE_PTEST_Intrinsic
1117 : Intrinsic<[llvm_i1_ty],
1122 class AdvSIMD_SVE_TBL_Intrinsic
1123 : Intrinsic<[llvm_anyvector_ty],
1125 LLVMVectorOfBitcastsToInt<0>],
1128 class AdvSIMD_SVE2_TBX_Intrinsic
1129 : Intrinsic<[llvm_anyvector_ty],
1132 LLVMVectorOfBitcastsToInt<0>],
1135 class SVE2_1VectorArg_Long_Intrinsic
1136 : Intrinsic<[llvm_anyvector_ty],
1137 [LLVMSubdivide2VectorType<0>,
1139 [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1141 class SVE2_2VectorArg_Long_Intrinsic
1142 : Intrinsic<[llvm_anyvector_ty],
1143 [LLVMSubdivide2VectorType<0>,
1144 LLVMSubdivide2VectorType<0>],
1147 class SVE2_2VectorArgIndexed_Long_Intrinsic
1148 : Intrinsic<[llvm_anyvector_ty],
1149 [LLVMSubdivide2VectorType<0>,
1150 LLVMSubdivide2VectorType<0>,
1152 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1154 class SVE2_2VectorArg_Wide_Intrinsic
1155 : Intrinsic<[llvm_anyvector_ty],
1157 LLVMSubdivide2VectorType<0>],
1160 class SVE2_2VectorArg_Pred_Long_Intrinsic
1161 : Intrinsic<[llvm_anyvector_ty],
1162 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1164 LLVMSubdivide2VectorType<0>],
1167 class SVE2_3VectorArg_Long_Intrinsic
1168 : Intrinsic<[llvm_anyvector_ty],
1170 LLVMSubdivide2VectorType<0>,
1171 LLVMSubdivide2VectorType<0>],
1174 class SVE2_3VectorArgIndexed_Long_Intrinsic
1175 : Intrinsic<[llvm_anyvector_ty],
1177 LLVMSubdivide2VectorType<0>,
1178 LLVMSubdivide2VectorType<0>,
1180 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1182 class SVE2_1VectorArg_Narrowing_Intrinsic
1183 : Intrinsic<[LLVMSubdivide2VectorType<0>],
1184 [llvm_anyvector_ty],
1187 class SVE2_Merged1VectorArg_Narrowing_Intrinsic
1188 : Intrinsic<[LLVMSubdivide2VectorType<0>],
1189 [LLVMSubdivide2VectorType<0>,
1192 class SVE2_2VectorArg_Narrowing_Intrinsic
1194 [LLVMSubdivide2VectorType<0>],
1195 [llvm_anyvector_ty, LLVMMatchType<0>],
1198 class SVE2_Merged2VectorArg_Narrowing_Intrinsic
1200 [LLVMSubdivide2VectorType<0>],
1201 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>],
1204 class SVE2_1VectorArg_Imm_Narrowing_Intrinsic
1205 : Intrinsic<[LLVMSubdivide2VectorType<0>],
1206 [llvm_anyvector_ty, llvm_i32_ty],
1207 [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1209 class SVE2_2VectorArg_Imm_Narrowing_Intrinsic
1210 : Intrinsic<[LLVMSubdivide2VectorType<0>],
1211 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty,
1213 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1215 class SVE2_CONFLICT_DETECT_Intrinsic
1216 : Intrinsic<[llvm_anyvector_ty],
1217 [LLVMAnyPointerType<llvm_any_ty>,
1220 class SVE2_3VectorArg_Indexed_Intrinsic
1221 : Intrinsic<[llvm_anyvector_ty],
1223 LLVMSubdivide2VectorType<0>,
1224 LLVMSubdivide2VectorType<0>,
1226 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1228 class AdvSIMD_SVE_CDOT_LANE_Intrinsic
1229 : Intrinsic<[llvm_anyvector_ty],
1231 LLVMSubdivide4VectorType<0>,
1232 LLVMSubdivide4VectorType<0>,
1235 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1237 // NOTE: There is no relationship between these intrinsics beyond an attempt
1238 // to reuse currently identical class definitions.
1239 class AdvSIMD_SVE_LOGB_Intrinsic : AdvSIMD_SVE_CNT_Intrinsic;
1240 class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic;
1241 class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic;
1243 // This class of intrinsics are not intended to be useful within LLVM IR but
1244 // are instead here to support some of the more regid parts of the ACLE.
1245 class Builtin_SVCVT<string name, LLVMType OUT, LLVMType PRED, LLVMType IN>
1246 : Intrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>;
1249 //===----------------------------------------------------------------------===//
1252 let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.".
1254 class AdvSIMD_SVE_Reduce_Intrinsic
1255 : Intrinsic<[LLVMVectorElementType<0>],
1256 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1260 class AdvSIMD_SVE_SADDV_Reduce_Intrinsic
1261 : Intrinsic<[llvm_i64_ty],
1262 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1266 class AdvSIMD_SVE_WHILE_Intrinsic
1267 : Intrinsic<[llvm_anyvector_ty],
1268 [llvm_anyint_ty, LLVMMatchType<1>],
1271 class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic
1272 : Intrinsic<[llvm_anyvector_ty],
1274 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1275 LLVMPointerToElt<0>,
1276 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1278 [IntrReadMem, IntrArgMemOnly]>;
1280 class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic
1281 : Intrinsic<[llvm_anyvector_ty],
1283 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1284 LLVMPointerToElt<0>,
1285 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1287 [IntrReadMem, IntrArgMemOnly]>;
1289 class AdvSIMD_GatherLoad_VS_Intrinsic
1290 : Intrinsic<[llvm_anyvector_ty],
1292 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1296 [IntrReadMem, IntrArgMemOnly]>;
1298 class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic
1302 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1303 LLVMPointerToElt<0>,
1304 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1306 [IntrWriteMem, IntrArgMemOnly]>;
1308 class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic
1312 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1313 LLVMPointerToElt<0>,
1314 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1316 [IntrWriteMem, IntrArgMemOnly]>;
1318 class AdvSIMD_ScatterStore_VS_Intrinsic
1322 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1323 llvm_anyvector_ty, llvm_i64_ty
1325 [IntrWriteMem, IntrArgMemOnly]>;
1328 class SVE_gather_prf_SV
1331 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1332 llvm_ptr_ty, // Base address
1333 llvm_anyvector_ty, // Offsets
1334 llvm_i32_ty // Prfop
1336 [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>;
1338 class SVE_gather_prf_VS
1341 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1342 llvm_anyvector_ty, // Base addresses
1343 llvm_i64_ty, // Scalar offset
1344 llvm_i32_ty // Prfop
1346 [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>;
1348 class SVE_MatMul_Intrinsic
1349 : Intrinsic<[llvm_anyvector_ty],
1350 [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>],
1354 : Intrinsic<[llvm_nxv4f32_ty],
1355 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty],
1358 class SVE_4Vec_BF16_Indexed
1359 : Intrinsic<[llvm_nxv4f32_ty],
1360 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty],
1361 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1364 // Vector tuple creation intrinsics (ACLE)
1367 def int_aarch64_sve_tuple_create2 : AdvSIMD_SVE_Create_2Vector_Tuple;
1368 def int_aarch64_sve_tuple_create3 : AdvSIMD_SVE_Create_3Vector_Tuple;
1369 def int_aarch64_sve_tuple_create4 : AdvSIMD_SVE_Create_4Vector_Tuple;
1372 // Vector tuple insertion/extraction intrinsics (ACLE)
1375 def int_aarch64_sve_tuple_get : AdvSIMD_SVE_Get_Vector_Tuple;
1376 def int_aarch64_sve_tuple_set : AdvSIMD_SVE_Set_Vector_Tuple;
1382 def int_aarch64_sve_ld1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1384 def int_aarch64_sve_ld2 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1385 def int_aarch64_sve_ld3 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1386 def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1388 def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1389 def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1390 def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1392 def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic;
1393 def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic;
1399 def int_aarch64_sve_st1 : AdvSIMD_1Vec_PredStore_Intrinsic;
1400 def int_aarch64_sve_st2 : AdvSIMD_2Vec_PredStore_Intrinsic;
1401 def int_aarch64_sve_st3 : AdvSIMD_3Vec_PredStore_Intrinsic;
1402 def int_aarch64_sve_st4 : AdvSIMD_4Vec_PredStore_Intrinsic;
1404 def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic;
1410 def int_aarch64_sve_prf
1411 : Intrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty],
1412 [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>;
1414 // Scalar + 32-bit scaled offset vector, zero extend, packed and
1416 def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV;
1417 def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV;
1418 def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV;
1419 def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV;
1421 // Scalar + 32-bit scaled offset vector, sign extend, packed and
1423 def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV;
1424 def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV;
1425 def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV;
1426 def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV;
1428 // Scalar + 64-bit scaled offset vector.
1429 def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV;
1430 def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV;
1431 def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV;
1432 def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV;
1435 def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS;
1436 def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS;
1437 def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS;
1438 def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS;
1441 // Scalar to vector operations
1444 def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic;
1445 def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic;
1448 def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic;
1451 // Address calculation
1454 def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic;
1455 def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic;
1456 def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic;
1457 def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic;
1460 // Integer arithmetic
1463 def int_aarch64_sve_add : AdvSIMD_Pred2VectorArg_Intrinsic;
1464 def int_aarch64_sve_sub : AdvSIMD_Pred2VectorArg_Intrinsic;
1465 def int_aarch64_sve_subr : AdvSIMD_Pred2VectorArg_Intrinsic;
1467 def int_aarch64_sve_pmul : AdvSIMD_2VectorArg_Intrinsic;
1469 def int_aarch64_sve_mul : AdvSIMD_Pred2VectorArg_Intrinsic;
1470 def int_aarch64_sve_mul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
1471 def int_aarch64_sve_smulh : AdvSIMD_Pred2VectorArg_Intrinsic;
1472 def int_aarch64_sve_umulh : AdvSIMD_Pred2VectorArg_Intrinsic;
1474 def int_aarch64_sve_sdiv : AdvSIMD_Pred2VectorArg_Intrinsic;
1475 def int_aarch64_sve_udiv : AdvSIMD_Pred2VectorArg_Intrinsic;
1476 def int_aarch64_sve_sdivr : AdvSIMD_Pred2VectorArg_Intrinsic;
1477 def int_aarch64_sve_udivr : AdvSIMD_Pred2VectorArg_Intrinsic;
1479 def int_aarch64_sve_smax : AdvSIMD_Pred2VectorArg_Intrinsic;
1480 def int_aarch64_sve_umax : AdvSIMD_Pred2VectorArg_Intrinsic;
1481 def int_aarch64_sve_smin : AdvSIMD_Pred2VectorArg_Intrinsic;
1482 def int_aarch64_sve_umin : AdvSIMD_Pred2VectorArg_Intrinsic;
1483 def int_aarch64_sve_sabd : AdvSIMD_Pred2VectorArg_Intrinsic;
1484 def int_aarch64_sve_uabd : AdvSIMD_Pred2VectorArg_Intrinsic;
1486 def int_aarch64_sve_mad : AdvSIMD_Pred3VectorArg_Intrinsic;
1487 def int_aarch64_sve_msb : AdvSIMD_Pred3VectorArg_Intrinsic;
1488 def int_aarch64_sve_mla : AdvSIMD_Pred3VectorArg_Intrinsic;
1489 def int_aarch64_sve_mla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
1490 def int_aarch64_sve_mls : AdvSIMD_Pred3VectorArg_Intrinsic;
1491 def int_aarch64_sve_mls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
1493 def int_aarch64_sve_saddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1494 def int_aarch64_sve_uaddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1496 def int_aarch64_sve_smaxv : AdvSIMD_SVE_Reduce_Intrinsic;
1497 def int_aarch64_sve_umaxv : AdvSIMD_SVE_Reduce_Intrinsic;
1498 def int_aarch64_sve_sminv : AdvSIMD_SVE_Reduce_Intrinsic;
1499 def int_aarch64_sve_uminv : AdvSIMD_SVE_Reduce_Intrinsic;
1501 def int_aarch64_sve_orv : AdvSIMD_SVE_Reduce_Intrinsic;
1502 def int_aarch64_sve_eorv : AdvSIMD_SVE_Reduce_Intrinsic;
1503 def int_aarch64_sve_andv : AdvSIMD_SVE_Reduce_Intrinsic;
1505 def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic;
1506 def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic;
1508 def int_aarch64_sve_sdot : AdvSIMD_SVE_DOT_Intrinsic;
1509 def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1511 def int_aarch64_sve_udot : AdvSIMD_SVE_DOT_Intrinsic;
1512 def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1514 def int_aarch64_sve_sqadd_x : AdvSIMD_2VectorArg_Intrinsic;
1515 def int_aarch64_sve_sqsub_x : AdvSIMD_2VectorArg_Intrinsic;
1516 def int_aarch64_sve_uqadd_x : AdvSIMD_2VectorArg_Intrinsic;
1517 def int_aarch64_sve_uqsub_x : AdvSIMD_2VectorArg_Intrinsic;
1521 def int_aarch64_sve_asr : AdvSIMD_Pred2VectorArg_Intrinsic;
1522 def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1523 def int_aarch64_sve_asrd : AdvSIMD_SVE_ShiftByImm_Intrinsic;
1524 def int_aarch64_sve_insr : AdvSIMD_SVE_INSR_Intrinsic;
1525 def int_aarch64_sve_lsl : AdvSIMD_Pred2VectorArg_Intrinsic;
1526 def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1527 def int_aarch64_sve_lsr : AdvSIMD_Pred2VectorArg_Intrinsic;
1528 def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1531 // Integer comparisons
1534 def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1535 def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic;
1536 def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1537 def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic;
1538 def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic;
1539 def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic;
1541 def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1542 def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1543 def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1544 def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1545 def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1546 def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1547 def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1548 def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1549 def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1550 def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1556 def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic;
1557 def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic;
1558 def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic;
1561 // Counting elements
1564 def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic;
1565 def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic;
1566 def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic;
1567 def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic;
1569 def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic;
1575 def int_aarch64_sve_rdffr : GCCBuiltin<"__builtin_sve_svrdffr">, Intrinsic<[llvm_nxv16i1_ty], []>;
1576 def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, Intrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>;
1577 def int_aarch64_sve_setffr : GCCBuiltin<"__builtin_sve_svsetffr">, Intrinsic<[], []>;
1578 def int_aarch64_sve_wrffr : GCCBuiltin<"__builtin_sve_svwrffr">, Intrinsic<[], [llvm_nxv16i1_ty]>;
1581 // Saturating scalar arithmetic
1584 def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1585 def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1586 def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1587 def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1589 def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1590 def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1591 def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1592 def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1593 def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1594 def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1595 def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1596 def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1597 def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1598 def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1600 def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1601 def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1602 def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1603 def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1605 def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1606 def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1607 def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1608 def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1609 def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1610 def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1611 def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1612 def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1613 def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1614 def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1616 def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1617 def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1618 def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1619 def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1621 def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1622 def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1623 def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1624 def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1625 def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1626 def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1627 def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1628 def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1629 def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1630 def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1632 def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1633 def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1634 def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1635 def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1637 def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1638 def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1639 def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1640 def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1641 def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1642 def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1643 def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1644 def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1645 def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1646 def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1652 def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic;
1653 def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic;
1654 def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic;
1655 def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic;
1658 // Permutations and selection
1661 def int_aarch64_sve_clasta : AdvSIMD_Pred2VectorArg_Intrinsic;
1662 def int_aarch64_sve_clasta_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1663 def int_aarch64_sve_clastb : AdvSIMD_Pred2VectorArg_Intrinsic;
1664 def int_aarch64_sve_clastb_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1665 def int_aarch64_sve_compact : AdvSIMD_Pred1VectorArg_Intrinsic;
1666 def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic;
1667 def int_aarch64_sve_ext : AdvSIMD_2VectorArgIndexed_Intrinsic;
1668 def int_aarch64_sve_sel : AdvSIMD_Pred2VectorArg_Intrinsic;
1669 def int_aarch64_sve_lasta : AdvSIMD_SVE_Reduce_Intrinsic;
1670 def int_aarch64_sve_lastb : AdvSIMD_SVE_Reduce_Intrinsic;
1671 def int_aarch64_sve_rev : AdvSIMD_1VectorArg_Intrinsic;
1672 def int_aarch64_sve_splice : AdvSIMD_Pred2VectorArg_Intrinsic;
1673 def int_aarch64_sve_sunpkhi : AdvSIMD_SVE_Unpack_Intrinsic;
1674 def int_aarch64_sve_sunpklo : AdvSIMD_SVE_Unpack_Intrinsic;
1675 def int_aarch64_sve_tbl : AdvSIMD_SVE_TBL_Intrinsic;
1676 def int_aarch64_sve_trn1 : AdvSIMD_2VectorArg_Intrinsic;
1677 def int_aarch64_sve_trn2 : AdvSIMD_2VectorArg_Intrinsic;
1678 def int_aarch64_sve_trn1q : AdvSIMD_2VectorArg_Intrinsic;
1679 def int_aarch64_sve_trn2q : AdvSIMD_2VectorArg_Intrinsic;
1680 def int_aarch64_sve_uunpkhi : AdvSIMD_SVE_Unpack_Intrinsic;
1681 def int_aarch64_sve_uunpklo : AdvSIMD_SVE_Unpack_Intrinsic;
1682 def int_aarch64_sve_uzp1 : AdvSIMD_2VectorArg_Intrinsic;
1683 def int_aarch64_sve_uzp2 : AdvSIMD_2VectorArg_Intrinsic;
1684 def int_aarch64_sve_uzp1q : AdvSIMD_2VectorArg_Intrinsic;
1685 def int_aarch64_sve_uzp2q : AdvSIMD_2VectorArg_Intrinsic;
1686 def int_aarch64_sve_zip1 : AdvSIMD_2VectorArg_Intrinsic;
1687 def int_aarch64_sve_zip2 : AdvSIMD_2VectorArg_Intrinsic;
1688 def int_aarch64_sve_zip1q : AdvSIMD_2VectorArg_Intrinsic;
1689 def int_aarch64_sve_zip2q : AdvSIMD_2VectorArg_Intrinsic;
1692 // Logical operations
1695 def int_aarch64_sve_and : AdvSIMD_Pred2VectorArg_Intrinsic;
1696 def int_aarch64_sve_bic : AdvSIMD_Pred2VectorArg_Intrinsic;
1697 def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic;
1698 def int_aarch64_sve_eor : AdvSIMD_Pred2VectorArg_Intrinsic;
1699 def int_aarch64_sve_not : AdvSIMD_Merged1VectorArg_Intrinsic;
1700 def int_aarch64_sve_orr : AdvSIMD_Pred2VectorArg_Intrinsic;
1706 def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1707 def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1708 def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1709 def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1710 def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1711 def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1714 // While comparisons
1717 def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic;
1718 def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic;
1719 def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic;
1720 def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic;
1721 def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic;
1722 def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic;
1723 def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic;
1724 def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic;
1727 // Floating-point arithmetic
1730 def int_aarch64_sve_fabd : AdvSIMD_Pred2VectorArg_Intrinsic;
1731 def int_aarch64_sve_fabs : AdvSIMD_Merged1VectorArg_Intrinsic;
1732 def int_aarch64_sve_fadd : AdvSIMD_Pred2VectorArg_Intrinsic;
1733 def int_aarch64_sve_fcadd : AdvSIMD_SVE_CADD_Intrinsic;
1734 def int_aarch64_sve_fcmla : AdvSIMD_SVE_CMLA_Intrinsic;
1735 def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
1736 def int_aarch64_sve_fdiv : AdvSIMD_Pred2VectorArg_Intrinsic;
1737 def int_aarch64_sve_fdivr : AdvSIMD_Pred2VectorArg_Intrinsic;
1738 def int_aarch64_sve_fexpa_x : AdvSIMD_SVE_EXPA_Intrinsic;
1739 def int_aarch64_sve_fmad : AdvSIMD_Pred3VectorArg_Intrinsic;
1740 def int_aarch64_sve_fmax : AdvSIMD_Pred2VectorArg_Intrinsic;
1741 def int_aarch64_sve_fmaxnm : AdvSIMD_Pred2VectorArg_Intrinsic;
1742 def int_aarch64_sve_fmin : AdvSIMD_Pred2VectorArg_Intrinsic;
1743 def int_aarch64_sve_fminnm : AdvSIMD_Pred2VectorArg_Intrinsic;
1744 def int_aarch64_sve_fmla : AdvSIMD_Pred3VectorArg_Intrinsic;
1745 def int_aarch64_sve_fmla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
1746 def int_aarch64_sve_fmls : AdvSIMD_Pred3VectorArg_Intrinsic;
1747 def int_aarch64_sve_fmls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
1748 def int_aarch64_sve_fmsb : AdvSIMD_Pred3VectorArg_Intrinsic;
1749 def int_aarch64_sve_fmul : AdvSIMD_Pred2VectorArg_Intrinsic;
1750 def int_aarch64_sve_fmulx : AdvSIMD_Pred2VectorArg_Intrinsic;
1751 def int_aarch64_sve_fneg : AdvSIMD_Merged1VectorArg_Intrinsic;
1752 def int_aarch64_sve_fmul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
1753 def int_aarch64_sve_fnmad : AdvSIMD_Pred3VectorArg_Intrinsic;
1754 def int_aarch64_sve_fnmla : AdvSIMD_Pred3VectorArg_Intrinsic;
1755 def int_aarch64_sve_fnmls : AdvSIMD_Pred3VectorArg_Intrinsic;
1756 def int_aarch64_sve_fnmsb : AdvSIMD_Pred3VectorArg_Intrinsic;
1757 def int_aarch64_sve_frecpe_x : AdvSIMD_1VectorArg_Intrinsic;
1758 def int_aarch64_sve_frecps_x : AdvSIMD_2VectorArg_Intrinsic;
1759 def int_aarch64_sve_frecpx : AdvSIMD_Merged1VectorArg_Intrinsic;
1760 def int_aarch64_sve_frinta : AdvSIMD_Merged1VectorArg_Intrinsic;
1761 def int_aarch64_sve_frinti : AdvSIMD_Merged1VectorArg_Intrinsic;
1762 def int_aarch64_sve_frintm : AdvSIMD_Merged1VectorArg_Intrinsic;
1763 def int_aarch64_sve_frintn : AdvSIMD_Merged1VectorArg_Intrinsic;
1764 def int_aarch64_sve_frintp : AdvSIMD_Merged1VectorArg_Intrinsic;
1765 def int_aarch64_sve_frintx : AdvSIMD_Merged1VectorArg_Intrinsic;
1766 def int_aarch64_sve_frintz : AdvSIMD_Merged1VectorArg_Intrinsic;
1767 def int_aarch64_sve_frsqrte_x : AdvSIMD_1VectorArg_Intrinsic;
1768 def int_aarch64_sve_frsqrts_x : AdvSIMD_2VectorArg_Intrinsic;
1769 def int_aarch64_sve_fscale : AdvSIMD_SVE_SCALE_Intrinsic;
1770 def int_aarch64_sve_fsqrt : AdvSIMD_Merged1VectorArg_Intrinsic;
1771 def int_aarch64_sve_fsub : AdvSIMD_Pred2VectorArg_Intrinsic;
1772 def int_aarch64_sve_fsubr : AdvSIMD_Pred2VectorArg_Intrinsic;
1773 def int_aarch64_sve_ftmad_x : AdvSIMD_2VectorArgIndexed_Intrinsic;
1774 def int_aarch64_sve_ftsmul_x : AdvSIMD_SVE_TSMUL_Intrinsic;
1775 def int_aarch64_sve_ftssel_x : AdvSIMD_SVE_TSMUL_Intrinsic;
1778 // Floating-point reductions
1781 def int_aarch64_sve_fadda : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1782 def int_aarch64_sve_faddv : AdvSIMD_SVE_Reduce_Intrinsic;
1783 def int_aarch64_sve_fmaxv : AdvSIMD_SVE_Reduce_Intrinsic;
1784 def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1785 def int_aarch64_sve_fminv : AdvSIMD_SVE_Reduce_Intrinsic;
1786 def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1789 // Floating-point conversions
1792 def int_aarch64_sve_fcvt : AdvSIMD_SVE_FCVT_Intrinsic;
1793 def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic;
1794 def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic;
1795 def int_aarch64_sve_scvtf : AdvSIMD_SVE_SCVTF_Intrinsic;
1796 def int_aarch64_sve_ucvtf : AdvSIMD_SVE_SCVTF_Intrinsic;
1799 // Floating-point comparisons
1802 def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic;
1803 def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic;
1805 def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1806 def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic;
1807 def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1808 def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic;
1809 def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic;
1811 def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1812 def int_aarch64_sve_fcvtzs_i32f64 : Builtin_SVCVT<"svcvt_s32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1813 def int_aarch64_sve_fcvtzs_i64f16 : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1814 def int_aarch64_sve_fcvtzs_i64f32 : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1816 def int_aarch64_sve_fcvt_bf16f32 : Builtin_SVCVT<"svcvt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
1817 def int_aarch64_sve_fcvtnt_bf16f32 : Builtin_SVCVT<"svcvtnt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
1819 def int_aarch64_sve_fcvtzu_i32f16 : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1820 def int_aarch64_sve_fcvtzu_i32f64 : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1821 def int_aarch64_sve_fcvtzu_i64f16 : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1822 def int_aarch64_sve_fcvtzu_i64f32 : Builtin_SVCVT<"svcvt_u64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1824 def int_aarch64_sve_fcvt_f16f32 : Builtin_SVCVT<"svcvt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
1825 def int_aarch64_sve_fcvt_f16f64 : Builtin_SVCVT<"svcvt_f16_f64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1826 def int_aarch64_sve_fcvt_f32f64 : Builtin_SVCVT<"svcvt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1828 def int_aarch64_sve_fcvt_f32f16 : Builtin_SVCVT<"svcvt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1829 def int_aarch64_sve_fcvt_f64f16 : Builtin_SVCVT<"svcvt_f64_f16_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1830 def int_aarch64_sve_fcvt_f64f32 : Builtin_SVCVT<"svcvt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1832 def int_aarch64_sve_fcvtlt_f32f16 : Builtin_SVCVT<"svcvtlt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1833 def int_aarch64_sve_fcvtlt_f64f32 : Builtin_SVCVT<"svcvtlt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1834 def int_aarch64_sve_fcvtnt_f16f32 : Builtin_SVCVT<"svcvtnt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
1835 def int_aarch64_sve_fcvtnt_f32f64 : Builtin_SVCVT<"svcvtnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1837 def int_aarch64_sve_fcvtx_f32f64 : Builtin_SVCVT<"svcvtx_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1838 def int_aarch64_sve_fcvtxnt_f32f64 : Builtin_SVCVT<"svcvtxnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1840 def int_aarch64_sve_scvtf_f16i32 : Builtin_SVCVT<"svcvt_f16_s32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
1841 def int_aarch64_sve_scvtf_f16i64 : Builtin_SVCVT<"svcvt_f16_s64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1842 def int_aarch64_sve_scvtf_f32i64 : Builtin_SVCVT<"svcvt_f32_s64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1843 def int_aarch64_sve_scvtf_f64i32 : Builtin_SVCVT<"svcvt_f64_s32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
1845 def int_aarch64_sve_ucvtf_f16i32 : Builtin_SVCVT<"svcvt_f16_u32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
1846 def int_aarch64_sve_ucvtf_f16i64 : Builtin_SVCVT<"svcvt_f16_u64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1847 def int_aarch64_sve_ucvtf_f32i64 : Builtin_SVCVT<"svcvt_f32_u64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1848 def int_aarch64_sve_ucvtf_f64i32 : Builtin_SVCVT<"svcvt_f64_u32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
1851 // Predicate creation
1854 def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic;
1857 // Predicate operations
1860 def int_aarch64_sve_and_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1861 def int_aarch64_sve_bic_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1862 def int_aarch64_sve_brka : AdvSIMD_Merged1VectorArg_Intrinsic;
1863 def int_aarch64_sve_brka_z : AdvSIMD_Pred1VectorArg_Intrinsic;
1864 def int_aarch64_sve_brkb : AdvSIMD_Merged1VectorArg_Intrinsic;
1865 def int_aarch64_sve_brkb_z : AdvSIMD_Pred1VectorArg_Intrinsic;
1866 def int_aarch64_sve_brkn_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1867 def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1868 def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1869 def int_aarch64_sve_eor_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1870 def int_aarch64_sve_nand_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1871 def int_aarch64_sve_nor_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1872 def int_aarch64_sve_orn_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1873 def int_aarch64_sve_orr_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1874 def int_aarch64_sve_pfirst : AdvSIMD_Pred1VectorArg_Intrinsic;
1875 def int_aarch64_sve_pnext : AdvSIMD_Pred1VectorArg_Intrinsic;
1876 def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
1877 def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;
1880 // Testing predicates
1883 def int_aarch64_sve_ptest_any : AdvSIMD_SVE_PTEST_Intrinsic;
1884 def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic;
1885 def int_aarch64_sve_ptest_last : AdvSIMD_SVE_PTEST_Intrinsic;
1888 // Reinterpreting data
1891 def int_aarch64_sve_convert_from_svbool : Intrinsic<[llvm_anyvector_ty],
1895 def int_aarch64_sve_convert_to_svbool : Intrinsic<[llvm_nxv16i1_ty],
1896 [llvm_anyvector_ty],
1900 // Gather loads: scalar base + vector offsets
1903 // 64 bit unscaled offsets
1904 def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1906 // 64 bit scaled offsets
1907 def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1909 // 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1910 def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1911 def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1913 // 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1914 def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1915 def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1918 // Gather loads: vector base + scalar offset
1921 def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1925 // First-faulting gather loads: scalar base + vector offsets
1928 // 64 bit unscaled offsets
1929 def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1931 // 64 bit scaled offsets
1932 def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1934 // 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
1935 def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1936 def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1938 // 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
1939 def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1940 def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1943 // First-faulting gather loads: vector base + scalar offset
1946 def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1950 // Non-temporal gather loads: scalar base + vector offsets
1953 // 64 bit unscaled offsets
1954 def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1957 def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1959 // 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
1960 def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1963 // Non-temporal gather loads: vector base + scalar offset
1966 def int_aarch64_sve_ldnt1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1969 // Scatter stores: scalar base + vector offsets
1972 // 64 bit unscaled offsets
1973 def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
1975 // 64 bit scaled offsets
1976 def int_aarch64_sve_st1_scatter_index
1977 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
1979 // 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1980 def int_aarch64_sve_st1_scatter_sxtw
1981 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1983 def int_aarch64_sve_st1_scatter_uxtw
1984 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1986 // 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1987 def int_aarch64_sve_st1_scatter_sxtw_index
1988 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1990 def int_aarch64_sve_st1_scatter_uxtw_index
1991 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1994 // Scatter stores: vector base + scalar offset
1997 def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic;
2000 // Non-temporal scatter stores: scalar base + vector offsets
2003 // 64 bit unscaled offsets
2004 def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2007 def int_aarch64_sve_stnt1_scatter_index
2008 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2010 // 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
2011 def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2014 // Non-temporal scatter stores: vector base + scalar offset
2017 def int_aarch64_sve_stnt1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic;
2020 // SVE2 - Uniform DSP operations
2023 def int_aarch64_sve_saba : AdvSIMD_3VectorArg_Intrinsic;
2024 def int_aarch64_sve_shadd : AdvSIMD_Pred2VectorArg_Intrinsic;
2025 def int_aarch64_sve_shsub : AdvSIMD_Pred2VectorArg_Intrinsic;
2026 def int_aarch64_sve_shsubr : AdvSIMD_Pred2VectorArg_Intrinsic;
2027 def int_aarch64_sve_sli : AdvSIMD_2VectorArgIndexed_Intrinsic;
2028 def int_aarch64_sve_sqabs : AdvSIMD_Merged1VectorArg_Intrinsic;
2029 def int_aarch64_sve_sqadd : AdvSIMD_Pred2VectorArg_Intrinsic;
2030 def int_aarch64_sve_sqdmulh : AdvSIMD_2VectorArg_Intrinsic;
2031 def int_aarch64_sve_sqdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
2032 def int_aarch64_sve_sqneg : AdvSIMD_Merged1VectorArg_Intrinsic;
2033 def int_aarch64_sve_sqrdmlah : AdvSIMD_3VectorArg_Intrinsic;
2034 def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2035 def int_aarch64_sve_sqrdmlsh : AdvSIMD_3VectorArg_Intrinsic;
2036 def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2037 def int_aarch64_sve_sqrdmulh : AdvSIMD_2VectorArg_Intrinsic;
2038 def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
2039 def int_aarch64_sve_sqrshl : AdvSIMD_Pred2VectorArg_Intrinsic;
2040 def int_aarch64_sve_sqshl : AdvSIMD_Pred2VectorArg_Intrinsic;
2041 def int_aarch64_sve_sqshlu : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2042 def int_aarch64_sve_sqsub : AdvSIMD_Pred2VectorArg_Intrinsic;
2043 def int_aarch64_sve_sqsubr : AdvSIMD_Pred2VectorArg_Intrinsic;
2044 def int_aarch64_sve_srhadd : AdvSIMD_Pred2VectorArg_Intrinsic;
2045 def int_aarch64_sve_sri : AdvSIMD_2VectorArgIndexed_Intrinsic;
2046 def int_aarch64_sve_srshl : AdvSIMD_Pred2VectorArg_Intrinsic;
2047 def int_aarch64_sve_srshr : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2048 def int_aarch64_sve_srsra : AdvSIMD_2VectorArgIndexed_Intrinsic;
2049 def int_aarch64_sve_ssra : AdvSIMD_2VectorArgIndexed_Intrinsic;
2050 def int_aarch64_sve_suqadd : AdvSIMD_Pred2VectorArg_Intrinsic;
2051 def int_aarch64_sve_uaba : AdvSIMD_3VectorArg_Intrinsic;
2052 def int_aarch64_sve_uhadd : AdvSIMD_Pred2VectorArg_Intrinsic;
2053 def int_aarch64_sve_uhsub : AdvSIMD_Pred2VectorArg_Intrinsic;
2054 def int_aarch64_sve_uhsubr : AdvSIMD_Pred2VectorArg_Intrinsic;
2055 def int_aarch64_sve_uqadd : AdvSIMD_Pred2VectorArg_Intrinsic;
2056 def int_aarch64_sve_uqrshl : AdvSIMD_Pred2VectorArg_Intrinsic;
2057 def int_aarch64_sve_uqshl : AdvSIMD_Pred2VectorArg_Intrinsic;
2058 def int_aarch64_sve_uqsub : AdvSIMD_Pred2VectorArg_Intrinsic;
2059 def int_aarch64_sve_uqsubr : AdvSIMD_Pred2VectorArg_Intrinsic;
2060 def int_aarch64_sve_urecpe : AdvSIMD_Merged1VectorArg_Intrinsic;
2061 def int_aarch64_sve_urhadd : AdvSIMD_Pred2VectorArg_Intrinsic;
2062 def int_aarch64_sve_urshl : AdvSIMD_Pred2VectorArg_Intrinsic;
2063 def int_aarch64_sve_urshr : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2064 def int_aarch64_sve_ursqrte : AdvSIMD_Merged1VectorArg_Intrinsic;
2065 def int_aarch64_sve_ursra : AdvSIMD_2VectorArgIndexed_Intrinsic;
2066 def int_aarch64_sve_usqadd : AdvSIMD_Pred2VectorArg_Intrinsic;
2067 def int_aarch64_sve_usra : AdvSIMD_2VectorArgIndexed_Intrinsic;
2070 // SVE2 - Widening DSP operations
2073 def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic;
2074 def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic;
2075 def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic;
2076 def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic;
2077 def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic;
2078 def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic;
2079 def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic;
2080 def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic;
2081 def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic;
2082 def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic;
2083 def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic;
2084 def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic;
2085 def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic;
2086 def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic;
2087 def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic;
2088 def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic;
2089 def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic;
2090 def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic;
2091 def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic;
2092 def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic;
2093 def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic;
2094 def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic;
2095 def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic;
2096 def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic;
2097 def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic;
2098 def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic;
2099 def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic;
2100 def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic;
2103 // SVE2 - Non-widening pairwise arithmetic
2106 def int_aarch64_sve_addp : AdvSIMD_Pred2VectorArg_Intrinsic;
2107 def int_aarch64_sve_faddp : AdvSIMD_Pred2VectorArg_Intrinsic;
2108 def int_aarch64_sve_fmaxp : AdvSIMD_Pred2VectorArg_Intrinsic;
2109 def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2110 def int_aarch64_sve_fminp : AdvSIMD_Pred2VectorArg_Intrinsic;
2111 def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2112 def int_aarch64_sve_smaxp : AdvSIMD_Pred2VectorArg_Intrinsic;
2113 def int_aarch64_sve_sminp : AdvSIMD_Pred2VectorArg_Intrinsic;
2114 def int_aarch64_sve_umaxp : AdvSIMD_Pred2VectorArg_Intrinsic;
2115 def int_aarch64_sve_uminp : AdvSIMD_Pred2VectorArg_Intrinsic;
2118 // SVE2 - Widening pairwise arithmetic
2121 def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2122 def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2125 // SVE2 - Uniform complex integer arithmetic
2128 def int_aarch64_sve_cadd_x : AdvSIMD_SVE2_CADD_Intrinsic;
2129 def int_aarch64_sve_sqcadd_x : AdvSIMD_SVE2_CADD_Intrinsic;
2130 def int_aarch64_sve_cmla_x : AdvSIMD_SVE2_CMLA_Intrinsic;
2131 def int_aarch64_sve_cmla_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2132 def int_aarch64_sve_sqrdcmlah_x : AdvSIMD_SVE2_CMLA_Intrinsic;
2133 def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2136 // SVE2 - Widening complex integer arithmetic
2139 def int_aarch64_sve_saddlbt : SVE2_2VectorArg_Long_Intrinsic;
2140 def int_aarch64_sve_ssublbt : SVE2_2VectorArg_Long_Intrinsic;
2141 def int_aarch64_sve_ssubltb : SVE2_2VectorArg_Long_Intrinsic;
2144 // SVE2 - Widening complex integer dot product
2147 def int_aarch64_sve_cdot : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2148 def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic;
2151 // SVE2 - Floating-point widening multiply-accumulate
2154 def int_aarch64_sve_fmlalb : SVE2_3VectorArg_Long_Intrinsic;
2155 def int_aarch64_sve_fmlalb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic;
2156 def int_aarch64_sve_fmlalt : SVE2_3VectorArg_Long_Intrinsic;
2157 def int_aarch64_sve_fmlalt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic;
2158 def int_aarch64_sve_fmlslb : SVE2_3VectorArg_Long_Intrinsic;
2159 def int_aarch64_sve_fmlslb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic;
2160 def int_aarch64_sve_fmlslt : SVE2_3VectorArg_Long_Intrinsic;
2161 def int_aarch64_sve_fmlslt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic;
2164 // SVE2 - Floating-point integer binary logarithm
2167 def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic;
2170 // SVE2 - Vector histogram count
2173 def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic;
2174 def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic;
2177 // SVE2 - Character match
2180 def int_aarch64_sve_match : AdvSIMD_SVE_Compare_Intrinsic;
2181 def int_aarch64_sve_nmatch : AdvSIMD_SVE_Compare_Intrinsic;
2184 // SVE2 - Unary narrowing operations
2187 def int_aarch64_sve_sqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic;
2188 def int_aarch64_sve_sqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2189 def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic;
2190 def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2191 def int_aarch64_sve_uqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic;
2192 def int_aarch64_sve_uqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2195 // SVE2 - Binary narrowing DSP operations
2197 def int_aarch64_sve_addhnb : SVE2_2VectorArg_Narrowing_Intrinsic;
2198 def int_aarch64_sve_addhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2200 def int_aarch64_sve_raddhnb : SVE2_2VectorArg_Narrowing_Intrinsic;
2201 def int_aarch64_sve_raddhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2203 def int_aarch64_sve_subhnb : SVE2_2VectorArg_Narrowing_Intrinsic;
2204 def int_aarch64_sve_subhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2206 def int_aarch64_sve_rsubhnb : SVE2_2VectorArg_Narrowing_Intrinsic;
2207 def int_aarch64_sve_rsubhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2209 // Narrowing shift right
2210 def int_aarch64_sve_shrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2211 def int_aarch64_sve_shrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2213 def int_aarch64_sve_rshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2214 def int_aarch64_sve_rshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2216 // Saturating shift right - signed input/output
2217 def int_aarch64_sve_sqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2218 def int_aarch64_sve_sqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2220 def int_aarch64_sve_sqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2221 def int_aarch64_sve_sqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2223 // Saturating shift right - unsigned input/output
2224 def int_aarch64_sve_uqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2225 def int_aarch64_sve_uqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2227 def int_aarch64_sve_uqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2228 def int_aarch64_sve_uqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2230 // Saturating shift right - signed input, unsigned output
2231 def int_aarch64_sve_sqshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2232 def int_aarch64_sve_sqshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2234 def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2235 def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2238 def int_aarch64_sve_smlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2239 def int_aarch64_sve_smlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2240 def int_aarch64_sve_umlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2241 def int_aarch64_sve_umlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2242 def int_aarch64_sve_smlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2243 def int_aarch64_sve_smlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2244 def int_aarch64_sve_umlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2245 def int_aarch64_sve_umlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2246 def int_aarch64_sve_smullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2247 def int_aarch64_sve_smullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2248 def int_aarch64_sve_umullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2249 def int_aarch64_sve_umullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2250 def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2251 def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2252 def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2253 def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2254 def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2255 def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2257 // SVE2 MLA Unpredicated.
2258 def int_aarch64_sve_smlalb : SVE2_3VectorArg_Long_Intrinsic;
2259 def int_aarch64_sve_smlalt : SVE2_3VectorArg_Long_Intrinsic;
2260 def int_aarch64_sve_umlalb : SVE2_3VectorArg_Long_Intrinsic;
2261 def int_aarch64_sve_umlalt : SVE2_3VectorArg_Long_Intrinsic;
2262 def int_aarch64_sve_smlslb : SVE2_3VectorArg_Long_Intrinsic;
2263 def int_aarch64_sve_smlslt : SVE2_3VectorArg_Long_Intrinsic;
2264 def int_aarch64_sve_umlslb : SVE2_3VectorArg_Long_Intrinsic;
2265 def int_aarch64_sve_umlslt : SVE2_3VectorArg_Long_Intrinsic;
2266 def int_aarch64_sve_smullb : SVE2_2VectorArg_Long_Intrinsic;
2267 def int_aarch64_sve_smullt : SVE2_2VectorArg_Long_Intrinsic;
2268 def int_aarch64_sve_umullb : SVE2_2VectorArg_Long_Intrinsic;
2269 def int_aarch64_sve_umullt : SVE2_2VectorArg_Long_Intrinsic;
2271 def int_aarch64_sve_sqdmlalb : SVE2_3VectorArg_Long_Intrinsic;
2272 def int_aarch64_sve_sqdmlalt : SVE2_3VectorArg_Long_Intrinsic;
2273 def int_aarch64_sve_sqdmlslb : SVE2_3VectorArg_Long_Intrinsic;
2274 def int_aarch64_sve_sqdmlslt : SVE2_3VectorArg_Long_Intrinsic;
2275 def int_aarch64_sve_sqdmullb : SVE2_2VectorArg_Long_Intrinsic;
2276 def int_aarch64_sve_sqdmullt : SVE2_2VectorArg_Long_Intrinsic;
2277 def int_aarch64_sve_sqdmlalbt : SVE2_3VectorArg_Long_Intrinsic;
2278 def int_aarch64_sve_sqdmlslbt : SVE2_3VectorArg_Long_Intrinsic;
2280 // SVE2 ADDSUB Long Unpredicated.
2281 def int_aarch64_sve_adclb : AdvSIMD_3VectorArg_Intrinsic;
2282 def int_aarch64_sve_adclt : AdvSIMD_3VectorArg_Intrinsic;
2283 def int_aarch64_sve_sbclb : AdvSIMD_3VectorArg_Intrinsic;
2284 def int_aarch64_sve_sbclt : AdvSIMD_3VectorArg_Intrinsic;
2287 // SVE2 - Polynomial arithmetic
2289 def int_aarch64_sve_eorbt : AdvSIMD_3VectorArg_Intrinsic;
2290 def int_aarch64_sve_eortb : AdvSIMD_3VectorArg_Intrinsic;
2291 def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic;
2292 def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic;
2295 // SVE2 bitwise ternary operations.
2297 def int_aarch64_sve_eor3 : AdvSIMD_3VectorArg_Intrinsic;
2298 def int_aarch64_sve_bcax : AdvSIMD_3VectorArg_Intrinsic;
2299 def int_aarch64_sve_bsl : AdvSIMD_3VectorArg_Intrinsic;
2300 def int_aarch64_sve_bsl1n : AdvSIMD_3VectorArg_Intrinsic;
2301 def int_aarch64_sve_bsl2n : AdvSIMD_3VectorArg_Intrinsic;
2302 def int_aarch64_sve_nbsl : AdvSIMD_3VectorArg_Intrinsic;
2303 def int_aarch64_sve_xar : AdvSIMD_2VectorArgIndexed_Intrinsic;
2306 // SVE2 - Optional AES, SHA-3 and SM4
2309 def int_aarch64_sve_aesd : GCCBuiltin<"__builtin_sve_svaesd_u8">,
2310 Intrinsic<[llvm_nxv16i8_ty],
2311 [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2313 def int_aarch64_sve_aesimc : GCCBuiltin<"__builtin_sve_svaesimc_u8">,
2314 Intrinsic<[llvm_nxv16i8_ty],
2317 def int_aarch64_sve_aese : GCCBuiltin<"__builtin_sve_svaese_u8">,
2318 Intrinsic<[llvm_nxv16i8_ty],
2319 [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2321 def int_aarch64_sve_aesmc : GCCBuiltin<"__builtin_sve_svaesmc_u8">,
2322 Intrinsic<[llvm_nxv16i8_ty],
2325 def int_aarch64_sve_rax1 : GCCBuiltin<"__builtin_sve_svrax1_u64">,
2326 Intrinsic<[llvm_nxv2i64_ty],
2327 [llvm_nxv2i64_ty, llvm_nxv2i64_ty],
2329 def int_aarch64_sve_sm4e : GCCBuiltin<"__builtin_sve_svsm4e_u32">,
2330 Intrinsic<[llvm_nxv4i32_ty],
2331 [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2333 def int_aarch64_sve_sm4ekey : GCCBuiltin<"__builtin_sve_svsm4ekey_u32">,
2334 Intrinsic<[llvm_nxv4i32_ty],
2335 [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2338 // SVE2 - Extended table lookup/permute
2341 def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic;
2342 def int_aarch64_sve_tbx : AdvSIMD_SVE2_TBX_Intrinsic;
2345 // SVE2 - Optional bit permutation
2348 def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic;
2349 def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic;
2350 def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic;
2354 // SVE ACLE: 7.3. INT8 matrix multiply extensions
2356 def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic;
2357 def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic;
2358 def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic;
2360 def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic;
2361 def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2362 def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2365 // SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions
2367 def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic;
2370 // SVE ACLE: 7.2. BFloat16 extensions
2373 def int_aarch64_sve_bfdot : SVE_4Vec_BF16;
2374 def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16;
2375 def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16;
2377 def int_aarch64_sve_bfmmla : SVE_4Vec_BF16;
2379 def int_aarch64_sve_bfdot_lane : SVE_4Vec_BF16_Indexed;
2380 def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed;
2381 def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed;
2385 // SVE2 - Contiguous conflict detection
2388 def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic;
2389 def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic;
2390 def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic;
2391 def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic;
2392 def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic;
2393 def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic;
2394 def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic;
2395 def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic;