1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
3 ; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
5 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
7 define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
8 ; GFX7-LABEL: udot8_acc32:
9 ; GFX7: ; %bb.0: ; %entry
10 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
11 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
12 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13 ; GFX7-NEXT: s_mov_b32 s6, -1
14 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15 ; GFX7-NEXT: s_load_dword s0, s[8:9], 0x0
16 ; GFX7-NEXT: s_load_dword s1, s[10:11], 0x0
17 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
18 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19 ; GFX7-NEXT: s_lshr_b32 s8, s0, 28
20 ; GFX7-NEXT: s_lshr_b32 s15, s1, 28
21 ; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40018
22 ; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40014
23 ; GFX7-NEXT: s_bfe_u32 s18, s1, 0x40010
24 ; GFX7-NEXT: s_bfe_u32 s19, s1, 0x4000c
25 ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40008
26 ; GFX7-NEXT: s_bfe_u32 s21, s1, 0x40004
27 ; GFX7-NEXT: s_and_b32 s1, s1, 15
28 ; GFX7-NEXT: s_bfe_u32 s9, s0, 0x40018
29 ; GFX7-NEXT: s_bfe_u32 s10, s0, 0x40014
30 ; GFX7-NEXT: s_bfe_u32 s11, s0, 0x40010
31 ; GFX7-NEXT: s_bfe_u32 s12, s0, 0x4000c
32 ; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40008
33 ; GFX7-NEXT: s_bfe_u32 s14, s0, 0x40004
34 ; GFX7-NEXT: s_and_b32 s0, s0, 15
35 ; GFX7-NEXT: v_mov_b32_e32 v0, s1
36 ; GFX7-NEXT: v_mov_b32_e32 v1, s2
37 ; GFX7-NEXT: v_mad_u32_u24 v0, s0, v0, v1
38 ; GFX7-NEXT: v_mov_b32_e32 v1, s21
39 ; GFX7-NEXT: v_mad_u32_u24 v0, s14, v1, v0
40 ; GFX7-NEXT: v_mov_b32_e32 v1, s20
41 ; GFX7-NEXT: v_mad_u32_u24 v0, s13, v1, v0
42 ; GFX7-NEXT: v_mov_b32_e32 v1, s19
43 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v1, v0
44 ; GFX7-NEXT: v_mov_b32_e32 v1, s18
45 ; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0
46 ; GFX7-NEXT: v_mov_b32_e32 v1, s17
47 ; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
48 ; GFX7-NEXT: v_mov_b32_e32 v1, s16
49 ; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0
50 ; GFX7-NEXT: v_mov_b32_e32 v1, s15
51 ; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0
52 ; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
55 ; GFX8-LABEL: udot8_acc32:
56 ; GFX8: ; %bb.0: ; %entry
57 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
58 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
59 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
60 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
61 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
62 ; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
63 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
64 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
65 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
66 ; GFX8-NEXT: s_lshr_b32 s0, s2, 28
67 ; GFX8-NEXT: s_lshr_b32 s11, s4, 28
68 ; GFX8-NEXT: s_bfe_u32 s12, s4, 0x40018
69 ; GFX8-NEXT: s_bfe_u32 s13, s4, 0x40014
70 ; GFX8-NEXT: s_bfe_u32 s14, s4, 0x40010
71 ; GFX8-NEXT: s_bfe_u32 s15, s4, 0x4000c
72 ; GFX8-NEXT: s_bfe_u32 s16, s4, 0x40008
73 ; GFX8-NEXT: s_bfe_u32 s17, s4, 0x40004
74 ; GFX8-NEXT: s_and_b32 s4, s4, 15
75 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40018
76 ; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40014
77 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40010
78 ; GFX8-NEXT: s_bfe_u32 s8, s2, 0x4000c
79 ; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40008
80 ; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40004
81 ; GFX8-NEXT: s_and_b32 s2, s2, 15
82 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
83 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
84 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v2, v3
85 ; GFX8-NEXT: v_mov_b32_e32 v3, s17
86 ; GFX8-NEXT: v_mad_u32_u24 v2, s10, v3, v2
87 ; GFX8-NEXT: v_mov_b32_e32 v3, s16
88 ; GFX8-NEXT: v_mad_u32_u24 v2, s9, v3, v2
89 ; GFX8-NEXT: v_mov_b32_e32 v3, s15
90 ; GFX8-NEXT: v_mad_u32_u24 v2, s8, v3, v2
91 ; GFX8-NEXT: v_mov_b32_e32 v3, s14
92 ; GFX8-NEXT: v_mad_u32_u24 v2, s7, v3, v2
93 ; GFX8-NEXT: v_mov_b32_e32 v3, s13
94 ; GFX8-NEXT: v_mad_u32_u24 v2, s6, v3, v2
95 ; GFX8-NEXT: v_mov_b32_e32 v3, s12
96 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2
97 ; GFX8-NEXT: v_mov_b32_e32 v3, s11
98 ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
99 ; GFX8-NEXT: flat_store_dword v[0:1], v2
100 ; GFX8-NEXT: s_endpgm
102 ; GFX9-LABEL: udot8_acc32:
103 ; GFX9: ; %bb.0: ; %entry
104 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
105 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
106 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
107 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
108 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
109 ; GFX9-NEXT: s_load_dword s5, s[0:1], 0x0
110 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
111 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
112 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
113 ; GFX9-NEXT: s_lshr_b32 s0, s2, 28
114 ; GFX9-NEXT: s_lshr_b32 s11, s4, 28
115 ; GFX9-NEXT: s_bfe_u32 s12, s4, 0x40018
116 ; GFX9-NEXT: s_bfe_u32 s13, s4, 0x40014
117 ; GFX9-NEXT: s_bfe_u32 s14, s4, 0x40010
118 ; GFX9-NEXT: s_bfe_u32 s15, s4, 0x4000c
119 ; GFX9-NEXT: s_bfe_u32 s16, s4, 0x40008
120 ; GFX9-NEXT: s_bfe_u32 s17, s4, 0x40004
121 ; GFX9-NEXT: s_and_b32 s4, s4, 15
122 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40018
123 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40014
124 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40010
125 ; GFX9-NEXT: s_bfe_u32 s8, s2, 0x4000c
126 ; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40008
127 ; GFX9-NEXT: s_bfe_u32 s10, s2, 0x40004
128 ; GFX9-NEXT: s_and_b32 s2, s2, 15
129 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
130 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
131 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v2, v3
132 ; GFX9-NEXT: v_mov_b32_e32 v3, s17
133 ; GFX9-NEXT: v_mad_u32_u24 v2, s10, v3, v2
134 ; GFX9-NEXT: v_mov_b32_e32 v3, s16
135 ; GFX9-NEXT: v_mad_u32_u24 v2, s9, v3, v2
136 ; GFX9-NEXT: v_mov_b32_e32 v3, s15
137 ; GFX9-NEXT: v_mad_u32_u24 v2, s8, v3, v2
138 ; GFX9-NEXT: v_mov_b32_e32 v3, s14
139 ; GFX9-NEXT: v_mad_u32_u24 v2, s7, v3, v2
140 ; GFX9-NEXT: v_mov_b32_e32 v3, s13
141 ; GFX9-NEXT: v_mad_u32_u24 v2, s6, v3, v2
142 ; GFX9-NEXT: v_mov_b32_e32 v3, s12
143 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v3, v2
144 ; GFX9-NEXT: v_mov_b32_e32 v3, s11
145 ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2
146 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
147 ; GFX9-NEXT: s_endpgm
149 ; GFX9-DL-LABEL: udot8_acc32:
150 ; GFX9-DL: ; %bb.0: ; %entry
151 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
152 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
153 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
154 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
155 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
156 ; GFX9-DL-NEXT: s_load_dword s5, s[0:1], 0x0
157 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
158 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
159 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
160 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
161 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5
162 ; GFX9-DL-NEXT: v_dot8_u32_u4 v2, s2, v2, v3
163 ; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off
164 ; GFX9-DL-NEXT: s_endpgm
165 <8 x i4> addrspace(1)* %src2,
166 i32 addrspace(1)* nocapture %dst) {
168 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
169 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
171 %v1e0 = extractelement <8 x i4> %vec1, i64 0
172 %cv1e0 = zext i4 %v1e0 to i32
173 %v2e0 = extractelement <8 x i4> %vec2, i64 0
174 %cv2e0 = zext i4 %v2e0 to i32
175 %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
177 %v1e1 = extractelement <8 x i4> %vec1, i64 1
178 %cv1e1 = zext i4 %v1e1 to i32
179 %v2e1 = extractelement <8 x i4> %vec2, i64 1
180 %cv2e1 = zext i4 %v2e1 to i32
181 %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
183 %v1e2 = extractelement <8 x i4> %vec1, i64 2
184 %cv1e2 = zext i4 %v1e2 to i32
185 %v2e2 = extractelement <8 x i4> %vec2, i64 2
186 %cv2e2 = zext i4 %v2e2 to i32
187 %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
189 %v1e3 = extractelement <8 x i4> %vec1, i64 3
190 %cv1e3 = zext i4 %v1e3 to i32
191 %v2e3 = extractelement <8 x i4> %vec2, i64 3
192 %cv2e3 = zext i4 %v2e3 to i32
193 %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
195 %v1e4 = extractelement <8 x i4> %vec1, i64 4
196 %cv1e4 = zext i4 %v1e4 to i32
197 %v2e4 = extractelement <8 x i4> %vec2, i64 4
198 %cv2e4 = zext i4 %v2e4 to i32
199 %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
201 %v1e5 = extractelement <8 x i4> %vec1, i64 5
202 %cv1e5 = zext i4 %v1e5 to i32
203 %v2e5 = extractelement <8 x i4> %vec2, i64 5
204 %cv2e5 = zext i4 %v2e5 to i32
205 %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
207 %v1e6 = extractelement <8 x i4> %vec1, i64 6
208 %cv1e6 = zext i4 %v1e6 to i32
209 %v2e6 = extractelement <8 x i4> %vec2, i64 6
210 %cv2e6 = zext i4 %v2e6 to i32
211 %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
213 %v1e7 = extractelement <8 x i4> %vec1, i64 7
214 %cv1e7 = zext i4 %v1e7 to i32
215 %v2e7 = extractelement <8 x i4> %vec2, i64 7
216 %cv2e7 = zext i4 %v2e7 to i32
217 %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
219 %acc = load i32, i32 addrspace(1)* %dst, align 4
220 %add1 = add i32 %mul0, %acc
221 %add2 = add i32 %add1, %mul1
222 %add3 = add i32 %add2, %mul2
223 %add4 = add i32 %add3, %mul3
224 %add5 = add i32 %add4, %mul4
225 %add6 = add i32 %add5, %mul5
226 %add7 = add i32 %add6, %mul6
227 %add8 = add i32 %add7, %mul7
229 store i32 %add8, i32 addrspace(1)* %dst, align 4
233 ; TODO: Remove the unnecessary instruction(that is zero-extending the
234 ; 2nd MAD) to have the pattern-recognizer to kick in.
235 define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
236 ; GFX7-LABEL: udot8_acc16:
237 ; GFX7: ; %bb.0: ; %entry
238 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
239 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd
240 ; GFX7-NEXT: s_mov_b32 s11, 0xf000
241 ; GFX7-NEXT: s_mov_b32 s10, -1
242 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
243 ; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
244 ; GFX7-NEXT: buffer_load_ushort v0, off, s[8:11], 0
245 ; GFX7-NEXT: s_load_dword s1, s[6:7], 0x0
246 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
247 ; GFX7-NEXT: s_lshr_b32 s2, s0, 28
248 ; GFX7-NEXT: s_bfe_u32 s4, s0, 0x40018
249 ; GFX7-NEXT: s_bfe_u32 s15, s1, 0x40018
250 ; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40014
251 ; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40010
252 ; GFX7-NEXT: s_bfe_u32 s18, s1, 0x4000c
253 ; GFX7-NEXT: s_bfe_u32 s19, s1, 0x40008
254 ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40004
255 ; GFX7-NEXT: s_lshr_b32 s14, s1, 28
256 ; GFX7-NEXT: s_and_b32 s1, s1, 15
257 ; GFX7-NEXT: s_bfe_u32 s5, s0, 0x40014
258 ; GFX7-NEXT: s_bfe_u32 s6, s0, 0x40010
259 ; GFX7-NEXT: s_bfe_u32 s7, s0, 0x4000c
260 ; GFX7-NEXT: s_bfe_u32 s12, s0, 0x40008
261 ; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40004
262 ; GFX7-NEXT: s_and_b32 s0, s0, 15
263 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
264 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
265 ; GFX7-NEXT: v_mov_b32_e32 v3, s19
266 ; GFX7-NEXT: v_mov_b32_e32 v4, s18
267 ; GFX7-NEXT: v_mov_b32_e32 v5, s17
268 ; GFX7-NEXT: v_mov_b32_e32 v6, s16
269 ; GFX7-NEXT: v_mov_b32_e32 v7, s15
270 ; GFX7-NEXT: s_waitcnt vmcnt(0)
271 ; GFX7-NEXT: v_mad_u32_u24 v0, s0, v1, v0
272 ; GFX7-NEXT: v_mad_u32_u24 v0, s13, v2, v0
273 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v3, v0
274 ; GFX7-NEXT: v_mad_u32_u24 v0, s7, v4, v0
275 ; GFX7-NEXT: v_mad_u32_u24 v0, s6, v5, v0
276 ; GFX7-NEXT: v_mad_u32_u24 v0, s5, v6, v0
277 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v7, v0
278 ; GFX7-NEXT: v_mov_b32_e32 v1, s14
279 ; GFX7-NEXT: v_mad_u32_u24 v0, s2, v1, v0
280 ; GFX7-NEXT: buffer_store_short v0, off, s[8:11], 0
281 ; GFX7-NEXT: s_endpgm
283 ; GFX8-LABEL: udot8_acc16:
284 ; GFX8: ; %bb.0: ; %entry
285 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
286 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
287 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
288 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
289 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
290 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
291 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
292 ; GFX8-NEXT: flat_load_ushort v2, v[0:1]
293 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
294 ; GFX8-NEXT: s_and_b32 s0, s2, 15
295 ; GFX8-NEXT: s_and_b32 s1, s4, 15
296 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
297 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004
298 ; GFX8-NEXT: v_mov_b32_e32 v4, s5
299 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004
300 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40008
301 ; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40010
302 ; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40014
303 ; GFX8-NEXT: s_bfe_u32 s12, s4, 0x40018
304 ; GFX8-NEXT: s_lshr_b32 s14, s4, 28
305 ; GFX8-NEXT: s_bfe_u32 s4, s4, 0x4000c
306 ; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40008
307 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
308 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x4000c
309 ; GFX8-NEXT: v_mov_b32_e32 v6, s4
310 ; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40010
311 ; GFX8-NEXT: v_mov_b32_e32 v7, s8
312 ; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40014
313 ; GFX8-NEXT: v_mov_b32_e32 v8, s10
314 ; GFX8-NEXT: s_bfe_u32 s13, s2, 0x40018
315 ; GFX8-NEXT: v_mov_b32_e32 v9, s12
316 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28
317 ; GFX8-NEXT: s_waitcnt vmcnt(0)
318 ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
319 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v4, v2
320 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v2
321 ; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2
322 ; GFX8-NEXT: v_mad_u32_u24 v2, s7, v6, v2
323 ; GFX8-NEXT: v_mad_u32_u24 v2, s9, v7, v2
324 ; GFX8-NEXT: v_mad_u32_u24 v2, s11, v8, v2
325 ; GFX8-NEXT: v_mad_u32_u24 v2, s13, v9, v2
326 ; GFX8-NEXT: v_mov_b32_e32 v3, s14
327 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
328 ; GFX8-NEXT: flat_store_short v[0:1], v2
329 ; GFX8-NEXT: s_endpgm
331 ; GFX9-LABEL: udot8_acc16:
332 ; GFX9: ; %bb.0: ; %entry
333 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
334 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
335 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
336 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
337 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
338 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
339 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
340 ; GFX9-NEXT: global_load_ushort v2, v[0:1], off
341 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
342 ; GFX9-NEXT: s_and_b32 s0, s2, 15
343 ; GFX9-NEXT: s_and_b32 s1, s4, 15
344 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
345 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004
346 ; GFX9-NEXT: v_mov_b32_e32 v4, s5
347 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004
348 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40008
349 ; GFX9-NEXT: s_bfe_u32 s8, s4, 0x40010
350 ; GFX9-NEXT: s_bfe_u32 s10, s4, 0x40014
351 ; GFX9-NEXT: s_bfe_u32 s12, s4, 0x40018
352 ; GFX9-NEXT: s_lshr_b32 s14, s4, 28
353 ; GFX9-NEXT: s_bfe_u32 s4, s4, 0x4000c
354 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40008
355 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
356 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x4000c
357 ; GFX9-NEXT: v_mov_b32_e32 v6, s4
358 ; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40010
359 ; GFX9-NEXT: v_mov_b32_e32 v7, s8
360 ; GFX9-NEXT: s_bfe_u32 s11, s2, 0x40014
361 ; GFX9-NEXT: v_mov_b32_e32 v8, s10
362 ; GFX9-NEXT: s_bfe_u32 s13, s2, 0x40018
363 ; GFX9-NEXT: v_mov_b32_e32 v9, s12
364 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28
365 ; GFX9-NEXT: s_waitcnt vmcnt(0)
366 ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2
367 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v4, v2
368 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
369 ; GFX9-NEXT: v_mad_u32_u24 v2, s6, v5, v2
370 ; GFX9-NEXT: v_mad_u32_u24 v2, s7, v6, v2
371 ; GFX9-NEXT: v_mad_u32_u24 v2, s9, v7, v2
372 ; GFX9-NEXT: v_mad_u32_u24 v2, s11, v8, v2
373 ; GFX9-NEXT: v_mad_u32_u24 v2, s13, v9, v2
374 ; GFX9-NEXT: v_mov_b32_e32 v3, s14
375 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v3, v2
376 ; GFX9-NEXT: global_store_short v[0:1], v2, off
377 ; GFX9-NEXT: s_endpgm
379 ; GFX9-DL-LABEL: udot8_acc16:
380 ; GFX9-DL: ; %bb.0: ; %entry
381 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
382 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
383 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
384 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
385 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
386 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
387 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
388 ; GFX9-DL-NEXT: global_load_ushort v2, v[0:1], off
389 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
390 ; GFX9-DL-NEXT: s_and_b32 s0, s2, 15
391 ; GFX9-DL-NEXT: s_and_b32 s1, s4, 15
392 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
393 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004
394 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s5
395 ; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004
396 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40008
397 ; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x40010
398 ; GFX9-DL-NEXT: s_bfe_u32 s10, s4, 0x40014
399 ; GFX9-DL-NEXT: s_bfe_u32 s12, s4, 0x40018
400 ; GFX9-DL-NEXT: s_lshr_b32 s14, s4, 28
401 ; GFX9-DL-NEXT: s_bfe_u32 s4, s4, 0x4000c
402 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x40008
403 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s5
404 ; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x4000c
405 ; GFX9-DL-NEXT: v_mov_b32_e32 v6, s4
406 ; GFX9-DL-NEXT: s_bfe_u32 s9, s2, 0x40010
407 ; GFX9-DL-NEXT: v_mov_b32_e32 v7, s8
408 ; GFX9-DL-NEXT: s_bfe_u32 s11, s2, 0x40014
409 ; GFX9-DL-NEXT: v_mov_b32_e32 v8, s10
410 ; GFX9-DL-NEXT: s_bfe_u32 s13, s2, 0x40018
411 ; GFX9-DL-NEXT: v_mov_b32_e32 v9, s12
412 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28
413 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
414 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2
415 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v4, v2
416 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2
417 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v5, v2
418 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s7, v6, v2
419 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s9, v7, v2
420 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s11, v8, v2
421 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s13, v9, v2
422 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s14
423 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2
424 ; GFX9-DL-NEXT: global_store_short v[0:1], v2, off
425 ; GFX9-DL-NEXT: s_endpgm
426 <8 x i4> addrspace(1)* %src2,
427 i16 addrspace(1)* nocapture %dst) {
429 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
430 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
432 %v1e0 = extractelement <8 x i4> %vec1, i64 0
433 %cv1e0 = zext i4 %v1e0 to i16
434 %v2e0 = extractelement <8 x i4> %vec2, i64 0
435 %cv2e0 = zext i4 %v2e0 to i16
436 %mul0 = mul nuw nsw i16 %cv1e0, %cv2e0
438 %v1e1 = extractelement <8 x i4> %vec1, i64 1
439 %cv1e1 = zext i4 %v1e1 to i16
440 %v2e1 = extractelement <8 x i4> %vec2, i64 1
441 %cv2e1 = zext i4 %v2e1 to i16
442 %mul1 = mul nuw nsw i16 %cv1e1, %cv2e1
444 %v1e2 = extractelement <8 x i4> %vec1, i64 2
445 %cv1e2 = zext i4 %v1e2 to i16
446 %v2e2 = extractelement <8 x i4> %vec2, i64 2
447 %cv2e2 = zext i4 %v2e2 to i16
448 %mul2 = mul nuw nsw i16 %cv1e2, %cv2e2
450 %v1e3 = extractelement <8 x i4> %vec1, i64 3
451 %cv1e3 = zext i4 %v1e3 to i16
452 %v2e3 = extractelement <8 x i4> %vec2, i64 3
453 %cv2e3 = zext i4 %v2e3 to i16
454 %mul3 = mul nuw nsw i16 %cv1e3, %cv2e3
456 %v1e4 = extractelement <8 x i4> %vec1, i64 4
457 %cv1e4 = zext i4 %v1e4 to i16
458 %v2e4 = extractelement <8 x i4> %vec2, i64 4
459 %cv2e4 = zext i4 %v2e4 to i16
460 %mul4 = mul nuw nsw i16 %cv1e4, %cv2e4
462 %v1e5 = extractelement <8 x i4> %vec1, i64 5
463 %cv1e5 = zext i4 %v1e5 to i16
464 %v2e5 = extractelement <8 x i4> %vec2, i64 5
465 %cv2e5 = zext i4 %v2e5 to i16
466 %mul5 = mul nuw nsw i16 %cv1e5, %cv2e5
468 %v1e6 = extractelement <8 x i4> %vec1, i64 6
469 %cv1e6 = zext i4 %v1e6 to i16
470 %v2e6 = extractelement <8 x i4> %vec2, i64 6
471 %cv2e6 = zext i4 %v2e6 to i16
472 %mul6 = mul nuw nsw i16 %cv1e6, %cv2e6
474 %v1e7 = extractelement <8 x i4> %vec1, i64 7
475 %cv1e7 = zext i4 %v1e7 to i16
476 %v2e7 = extractelement <8 x i4> %vec2, i64 7
477 %cv2e7 = zext i4 %v2e7 to i16
478 %mul7 = mul nuw nsw i16 %cv1e7, %cv2e7
480 %acc = load i16, i16 addrspace(1)* %dst, align 4
481 %add1 = add i16 %mul0, %acc
482 %add2 = add i16 %add1, %mul1
483 %add3 = add i16 %add2, %mul2
484 %add4 = add i16 %add3, %mul3
485 %add5 = add i16 %add4, %mul4
486 %add6 = add i16 %add5, %mul5
487 %add7 = add i16 %add6, %mul6
488 %add8 = add i16 %add7, %mul7
490 store i16 %add8, i16 addrspace(1)* %dst, align 4
494 ; TODO: Remove the unnecessary instruction(that is zero-extending the
495 ; 2nd MAD) to have the pattern-recognizer to kick in.
496 define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
497 ; GFX7-LABEL: udot8_acc8:
498 ; GFX7: ; %bb.0: ; %entry
499 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
500 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd
501 ; GFX7-NEXT: s_mov_b32 s11, 0xf000
502 ; GFX7-NEXT: s_mov_b32 s10, -1
503 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
504 ; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
505 ; GFX7-NEXT: buffer_load_ubyte v0, off, s[8:11], 0
506 ; GFX7-NEXT: s_load_dword s1, s[6:7], 0x0
507 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
508 ; GFX7-NEXT: s_lshr_b32 s2, s0, 28
509 ; GFX7-NEXT: s_bfe_u32 s4, s0, 0x40018
510 ; GFX7-NEXT: s_bfe_u32 s15, s1, 0x40018
511 ; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40014
512 ; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40010
513 ; GFX7-NEXT: s_bfe_u32 s18, s1, 0x4000c
514 ; GFX7-NEXT: s_bfe_u32 s19, s1, 0x40008
515 ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40004
516 ; GFX7-NEXT: s_lshr_b32 s14, s1, 28
517 ; GFX7-NEXT: s_and_b32 s1, s1, 15
518 ; GFX7-NEXT: s_bfe_u32 s5, s0, 0x40014
519 ; GFX7-NEXT: s_bfe_u32 s6, s0, 0x40010
520 ; GFX7-NEXT: s_bfe_u32 s7, s0, 0x4000c
521 ; GFX7-NEXT: s_bfe_u32 s12, s0, 0x40008
522 ; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40004
523 ; GFX7-NEXT: s_and_b32 s0, s0, 15
524 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
525 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
526 ; GFX7-NEXT: v_mov_b32_e32 v3, s19
527 ; GFX7-NEXT: v_mov_b32_e32 v4, s18
528 ; GFX7-NEXT: v_mov_b32_e32 v5, s17
529 ; GFX7-NEXT: v_mov_b32_e32 v6, s16
530 ; GFX7-NEXT: v_mov_b32_e32 v7, s15
531 ; GFX7-NEXT: s_waitcnt vmcnt(0)
532 ; GFX7-NEXT: v_mad_u32_u24 v0, s0, v1, v0
533 ; GFX7-NEXT: v_mad_u32_u24 v0, s13, v2, v0
534 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v3, v0
535 ; GFX7-NEXT: v_mad_u32_u24 v0, s7, v4, v0
536 ; GFX7-NEXT: v_mad_u32_u24 v0, s6, v5, v0
537 ; GFX7-NEXT: v_mad_u32_u24 v0, s5, v6, v0
538 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v7, v0
539 ; GFX7-NEXT: v_mov_b32_e32 v1, s14
540 ; GFX7-NEXT: v_mad_u32_u24 v0, s2, v1, v0
541 ; GFX7-NEXT: buffer_store_byte v0, off, s[8:11], 0
542 ; GFX7-NEXT: s_endpgm
544 ; GFX8-LABEL: udot8_acc8:
545 ; GFX8: ; %bb.0: ; %entry
546 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
547 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
548 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
549 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
550 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
551 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
552 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
553 ; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
554 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
555 ; GFX8-NEXT: s_and_b32 s0, s2, 15
556 ; GFX8-NEXT: s_and_b32 s1, s4, 15
557 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
558 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004
559 ; GFX8-NEXT: v_mov_b32_e32 v4, s5
560 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004
561 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40008
562 ; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40010
563 ; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40014
564 ; GFX8-NEXT: s_bfe_u32 s12, s4, 0x40018
565 ; GFX8-NEXT: s_lshr_b32 s14, s4, 28
566 ; GFX8-NEXT: s_bfe_u32 s4, s4, 0x4000c
567 ; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40008
568 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
569 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x4000c
570 ; GFX8-NEXT: v_mov_b32_e32 v6, s4
571 ; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40010
572 ; GFX8-NEXT: v_mov_b32_e32 v7, s8
573 ; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40014
574 ; GFX8-NEXT: v_mov_b32_e32 v8, s10
575 ; GFX8-NEXT: s_bfe_u32 s13, s2, 0x40018
576 ; GFX8-NEXT: v_mov_b32_e32 v9, s12
577 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28
578 ; GFX8-NEXT: s_waitcnt vmcnt(0)
579 ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
580 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v4, v2
581 ; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v2
582 ; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2
583 ; GFX8-NEXT: v_mad_u32_u24 v2, s7, v6, v2
584 ; GFX8-NEXT: v_mad_u32_u24 v2, s9, v7, v2
585 ; GFX8-NEXT: v_mad_u32_u24 v2, s11, v8, v2
586 ; GFX8-NEXT: v_mad_u32_u24 v2, s13, v9, v2
587 ; GFX8-NEXT: v_mov_b32_e32 v3, s14
588 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
589 ; GFX8-NEXT: flat_store_byte v[0:1], v2
590 ; GFX8-NEXT: s_endpgm
592 ; GFX9-LABEL: udot8_acc8:
593 ; GFX9: ; %bb.0: ; %entry
594 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
595 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
596 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
597 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
598 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
599 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
600 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
601 ; GFX9-NEXT: global_load_ubyte v2, v[0:1], off
602 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
603 ; GFX9-NEXT: s_and_b32 s0, s2, 15
604 ; GFX9-NEXT: s_and_b32 s1, s4, 15
605 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
606 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004
607 ; GFX9-NEXT: v_mov_b32_e32 v4, s5
608 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004
609 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40008
610 ; GFX9-NEXT: s_bfe_u32 s8, s4, 0x40010
611 ; GFX9-NEXT: s_bfe_u32 s10, s4, 0x40014
612 ; GFX9-NEXT: s_bfe_u32 s12, s4, 0x40018
613 ; GFX9-NEXT: s_lshr_b32 s14, s4, 28
614 ; GFX9-NEXT: s_bfe_u32 s4, s4, 0x4000c
615 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40008
616 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
617 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x4000c
618 ; GFX9-NEXT: v_mov_b32_e32 v6, s4
619 ; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40010
620 ; GFX9-NEXT: v_mov_b32_e32 v7, s8
621 ; GFX9-NEXT: s_bfe_u32 s11, s2, 0x40014
622 ; GFX9-NEXT: v_mov_b32_e32 v8, s10
623 ; GFX9-NEXT: s_bfe_u32 s13, s2, 0x40018
624 ; GFX9-NEXT: v_mov_b32_e32 v9, s12
625 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28
626 ; GFX9-NEXT: s_waitcnt vmcnt(0)
627 ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2
628 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v4, v2
629 ; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v2
630 ; GFX9-NEXT: v_mad_u32_u24 v2, s6, v5, v2
631 ; GFX9-NEXT: v_mad_u32_u24 v2, s7, v6, v2
632 ; GFX9-NEXT: v_mad_u32_u24 v2, s9, v7, v2
633 ; GFX9-NEXT: v_mad_u32_u24 v2, s11, v8, v2
634 ; GFX9-NEXT: v_mad_u32_u24 v2, s13, v9, v2
635 ; GFX9-NEXT: v_mov_b32_e32 v3, s14
636 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v3, v2
637 ; GFX9-NEXT: global_store_byte v[0:1], v2, off
638 ; GFX9-NEXT: s_endpgm
640 ; GFX9-DL-LABEL: udot8_acc8:
641 ; GFX9-DL: ; %bb.0: ; %entry
642 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
643 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
644 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
645 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
646 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
647 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
648 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
649 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off
650 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
651 ; GFX9-DL-NEXT: s_and_b32 s0, s2, 15
652 ; GFX9-DL-NEXT: s_and_b32 s1, s4, 15
653 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
654 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004
655 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s5
656 ; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004
657 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40008
658 ; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x40010
659 ; GFX9-DL-NEXT: s_bfe_u32 s10, s4, 0x40014
660 ; GFX9-DL-NEXT: s_bfe_u32 s12, s4, 0x40018
661 ; GFX9-DL-NEXT: s_lshr_b32 s14, s4, 28
662 ; GFX9-DL-NEXT: s_bfe_u32 s4, s4, 0x4000c
663 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x40008
664 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s5
665 ; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x4000c
666 ; GFX9-DL-NEXT: v_mov_b32_e32 v6, s4
667 ; GFX9-DL-NEXT: s_bfe_u32 s9, s2, 0x40010
668 ; GFX9-DL-NEXT: v_mov_b32_e32 v7, s8
669 ; GFX9-DL-NEXT: s_bfe_u32 s11, s2, 0x40014
670 ; GFX9-DL-NEXT: v_mov_b32_e32 v8, s10
671 ; GFX9-DL-NEXT: s_bfe_u32 s13, s2, 0x40018
672 ; GFX9-DL-NEXT: v_mov_b32_e32 v9, s12
673 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28
674 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
675 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2
676 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v4, v2
677 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 0xff, v2
678 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v5, v2
679 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s7, v6, v2
680 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s9, v7, v2
681 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s11, v8, v2
682 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s13, v9, v2
683 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s14
684 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2
685 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off
686 ; GFX9-DL-NEXT: s_endpgm
687 <8 x i4> addrspace(1)* %src2,
688 i8 addrspace(1)* nocapture %dst) {
690 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
691 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
693 %v1e0 = extractelement <8 x i4> %vec1, i64 0
694 %cv1e0 = zext i4 %v1e0 to i8
695 %v2e0 = extractelement <8 x i4> %vec2, i64 0
696 %cv2e0 = zext i4 %v2e0 to i8
697 %mul0 = mul nuw nsw i8 %cv1e0, %cv2e0
699 %v1e1 = extractelement <8 x i4> %vec1, i64 1
700 %cv1e1 = zext i4 %v1e1 to i8
701 %v2e1 = extractelement <8 x i4> %vec2, i64 1
702 %cv2e1 = zext i4 %v2e1 to i8
703 %mul1 = mul nuw nsw i8 %cv1e1, %cv2e1
705 %v1e2 = extractelement <8 x i4> %vec1, i64 2
706 %cv1e2 = zext i4 %v1e2 to i8
707 %v2e2 = extractelement <8 x i4> %vec2, i64 2
708 %cv2e2 = zext i4 %v2e2 to i8
709 %mul2 = mul nuw nsw i8 %cv1e2, %cv2e2
711 %v1e3 = extractelement <8 x i4> %vec1, i64 3
712 %cv1e3 = zext i4 %v1e3 to i8
713 %v2e3 = extractelement <8 x i4> %vec2, i64 3
714 %cv2e3 = zext i4 %v2e3 to i8
715 %mul3 = mul nuw nsw i8 %cv1e3, %cv2e3
717 %v1e4 = extractelement <8 x i4> %vec1, i64 4
718 %cv1e4 = zext i4 %v1e4 to i8
719 %v2e4 = extractelement <8 x i4> %vec2, i64 4
720 %cv2e4 = zext i4 %v2e4 to i8
721 %mul4 = mul nuw nsw i8 %cv1e4, %cv2e4
723 %v1e5 = extractelement <8 x i4> %vec1, i64 5
724 %cv1e5 = zext i4 %v1e5 to i8
725 %v2e5 = extractelement <8 x i4> %vec2, i64 5
726 %cv2e5 = zext i4 %v2e5 to i8
727 %mul5 = mul nuw nsw i8 %cv1e5, %cv2e5
729 %v1e6 = extractelement <8 x i4> %vec1, i64 6
730 %cv1e6 = zext i4 %v1e6 to i8
731 %v2e6 = extractelement <8 x i4> %vec2, i64 6
732 %cv2e6 = zext i4 %v2e6 to i8
733 %mul6 = mul nuw nsw i8 %cv1e6, %cv2e6
735 %v1e7 = extractelement <8 x i4> %vec1, i64 7
736 %cv1e7 = zext i4 %v1e7 to i8
737 %v2e7 = extractelement <8 x i4> %vec2, i64 7
738 %cv2e7 = zext i4 %v2e7 to i8
739 %mul7 = mul nuw nsw i8 %cv1e7, %cv2e7
741 %acc = load i8, i8 addrspace(1)* %dst, align 4
742 %add1 = add i8 %mul0, %acc
743 %add2 = add i8 %add1, %mul1
744 %add3 = add i8 %add2, %mul2
745 %add4 = add i8 %add3, %mul3
746 %add5 = add i8 %add4, %mul4
747 %add6 = add i8 %add5, %mul5
748 %add7 = add i8 %add6, %mul6
749 %add8 = add i8 %add7, %mul7
751 store i8 %add8, i8 addrspace(1)* %dst, align 4
755 ; TODO: Remove the two unnecessary instructions(and+add after 2nd MAD)
756 ; to have the pattern-recognizer to kick in.
757 define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
758 ; GFX7-LABEL: udot8_acc4:
759 ; GFX7: ; %bb.0: ; %entry
760 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
761 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd
762 ; GFX7-NEXT: s_mov_b32 s11, 0xf000
763 ; GFX7-NEXT: s_mov_b32 s10, -1
764 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
765 ; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
766 ; GFX7-NEXT: buffer_load_ubyte v0, off, s[8:11], 0
767 ; GFX7-NEXT: s_load_dword s1, s[6:7], 0x0
768 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
769 ; GFX7-NEXT: s_lshr_b32 s2, s0, 28
770 ; GFX7-NEXT: s_bfe_u32 s4, s0, 0x40018
771 ; GFX7-NEXT: s_bfe_u32 s15, s1, 0x40018
772 ; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40014
773 ; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40010
774 ; GFX7-NEXT: s_bfe_u32 s18, s1, 0x4000c
775 ; GFX7-NEXT: s_bfe_u32 s19, s1, 0x40008
776 ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40004
777 ; GFX7-NEXT: s_lshr_b32 s14, s1, 28
778 ; GFX7-NEXT: s_and_b32 s1, s1, 15
779 ; GFX7-NEXT: s_bfe_u32 s5, s0, 0x40014
780 ; GFX7-NEXT: s_bfe_u32 s6, s0, 0x40010
781 ; GFX7-NEXT: s_bfe_u32 s7, s0, 0x4000c
782 ; GFX7-NEXT: s_bfe_u32 s12, s0, 0x40008
783 ; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40004
784 ; GFX7-NEXT: s_and_b32 s0, s0, 15
785 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
786 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
787 ; GFX7-NEXT: v_mov_b32_e32 v3, s19
788 ; GFX7-NEXT: v_mov_b32_e32 v4, s18
789 ; GFX7-NEXT: v_mov_b32_e32 v5, s17
790 ; GFX7-NEXT: v_mov_b32_e32 v6, s16
791 ; GFX7-NEXT: v_mov_b32_e32 v7, s15
792 ; GFX7-NEXT: s_waitcnt vmcnt(0)
793 ; GFX7-NEXT: v_mad_u32_u24 v0, s0, v1, v0
794 ; GFX7-NEXT: v_mad_u32_u24 v0, s13, v2, v0
795 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v3, v0
796 ; GFX7-NEXT: v_mad_u32_u24 v0, s7, v4, v0
797 ; GFX7-NEXT: v_mad_u32_u24 v0, s6, v5, v0
798 ; GFX7-NEXT: v_mad_u32_u24 v0, s5, v6, v0
799 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v7, v0
800 ; GFX7-NEXT: v_mov_b32_e32 v1, s14
801 ; GFX7-NEXT: v_mad_u32_u24 v0, s2, v1, v0
802 ; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
803 ; GFX7-NEXT: buffer_store_byte v0, off, s[8:11], 0
804 ; GFX7-NEXT: s_endpgm
806 ; GFX8-LABEL: udot8_acc4:
807 ; GFX8: ; %bb.0: ; %entry
808 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
809 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
810 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
811 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
812 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
813 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
814 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
815 ; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
816 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
817 ; GFX8-NEXT: s_and_b32 s0, s2, 15
818 ; GFX8-NEXT: s_and_b32 s1, s4, 15
819 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
820 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004
821 ; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40008
822 ; GFX8-NEXT: v_mov_b32_e32 v4, s6
823 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40008
824 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
825 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004
826 ; GFX8-NEXT: v_mul_u32_u24_e32 v4, s7, v4
827 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x4000c
828 ; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
829 ; GFX8-NEXT: s_bfe_u32 s7, s4, 0x40010
830 ; GFX8-NEXT: v_mov_b32_e32 v6, s5
831 ; GFX8-NEXT: s_bfe_u32 s6, s2, 0x4000c
832 ; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40014
833 ; GFX8-NEXT: v_mov_b32_e32 v7, s7
834 ; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40010
835 ; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40018
836 ; GFX8-NEXT: v_mov_b32_e32 v8, s8
837 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40014
838 ; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40018
839 ; GFX8-NEXT: s_lshr_b32 s4, s4, 28
840 ; GFX8-NEXT: v_mov_b32_e32 v9, s9
841 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28
842 ; GFX8-NEXT: s_waitcnt vmcnt(0)
843 ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
844 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v5, v2
845 ; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
846 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v4, v2
847 ; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2
848 ; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2
849 ; GFX8-NEXT: v_mad_u32_u24 v2, s7, v8, v2
850 ; GFX8-NEXT: v_mad_u32_u24 v2, s8, v9, v2
851 ; GFX8-NEXT: v_mov_b32_e32 v3, s4
852 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
853 ; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
854 ; GFX8-NEXT: flat_store_byte v[0:1], v2
855 ; GFX8-NEXT: s_endpgm
857 ; GFX9-LABEL: udot8_acc4:
858 ; GFX9: ; %bb.0: ; %entry
859 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
860 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
861 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
862 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
863 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
864 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
865 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
866 ; GFX9-NEXT: global_load_ubyte v2, v[0:1], off
867 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
868 ; GFX9-NEXT: s_and_b32 s0, s2, 15
869 ; GFX9-NEXT: s_and_b32 s1, s4, 15
870 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
871 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004
872 ; GFX9-NEXT: s_bfe_u32 s6, s4, 0x40008
873 ; GFX9-NEXT: v_mov_b32_e32 v4, s6
874 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40008
875 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
876 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004
877 ; GFX9-NEXT: v_mul_u32_u24_e32 v4, s7, v4
878 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x4000c
879 ; GFX9-NEXT: v_and_b32_e32 v4, 15, v4
880 ; GFX9-NEXT: s_bfe_u32 s7, s4, 0x40010
881 ; GFX9-NEXT: v_mov_b32_e32 v6, s5
882 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x4000c
883 ; GFX9-NEXT: s_bfe_u32 s8, s4, 0x40014
884 ; GFX9-NEXT: v_mov_b32_e32 v7, s7
885 ; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40010
886 ; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40018
887 ; GFX9-NEXT: v_mov_b32_e32 v8, s8
888 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40014
889 ; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40018
890 ; GFX9-NEXT: s_lshr_b32 s4, s4, 28
891 ; GFX9-NEXT: v_mov_b32_e32 v9, s9
892 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28
893 ; GFX9-NEXT: s_waitcnt vmcnt(0)
894 ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2
895 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v5, v2
896 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
897 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v4
898 ; GFX9-NEXT: v_mad_u32_u24 v2, s6, v6, v2
899 ; GFX9-NEXT: v_mad_u32_u24 v2, s5, v7, v2
900 ; GFX9-NEXT: v_mad_u32_u24 v2, s7, v8, v2
901 ; GFX9-NEXT: v_mad_u32_u24 v2, s8, v9, v2
902 ; GFX9-NEXT: v_mov_b32_e32 v3, s4
903 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v3, v2
904 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
905 ; GFX9-NEXT: global_store_byte v[0:1], v2, off
906 ; GFX9-NEXT: s_endpgm
908 ; GFX9-DL-LABEL: udot8_acc4:
909 ; GFX9-DL: ; %bb.0: ; %entry
910 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
911 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
912 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
913 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
914 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
915 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
916 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
917 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off
918 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
919 ; GFX9-DL-NEXT: s_and_b32 s0, s2, 15
920 ; GFX9-DL-NEXT: s_and_b32 s1, s4, 15
921 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
922 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004
923 ; GFX9-DL-NEXT: s_bfe_u32 s6, s4, 0x40008
924 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s6
925 ; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x40008
926 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s5
927 ; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004
928 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, s7, v4
929 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x4000c
930 ; GFX9-DL-NEXT: v_and_b32_e32 v4, 15, v4
931 ; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x40010
932 ; GFX9-DL-NEXT: v_mov_b32_e32 v6, s5
933 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x4000c
934 ; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x40014
935 ; GFX9-DL-NEXT: v_mov_b32_e32 v7, s7
936 ; GFX9-DL-NEXT: s_bfe_u32 s5, s2, 0x40010
937 ; GFX9-DL-NEXT: s_bfe_u32 s9, s4, 0x40018
938 ; GFX9-DL-NEXT: v_mov_b32_e32 v8, s8
939 ; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x40014
940 ; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x40018
941 ; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28
942 ; GFX9-DL-NEXT: v_mov_b32_e32 v9, s9
943 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28
944 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
945 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2
946 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v5, v2
947 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
948 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v4
949 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v6, v2
950 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v7, v2
951 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s7, v8, v2
952 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s8, v9, v2
953 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4
954 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2
955 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
956 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off
957 ; GFX9-DL-NEXT: s_endpgm
958 <8 x i4> addrspace(1)* %src2,
959 i4 addrspace(1)* nocapture %dst) {
961 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
962 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
964 %v1e0 = extractelement <8 x i4> %vec1, i64 0
965 %v2e0 = extractelement <8 x i4> %vec2, i64 0
966 %mul0 = mul nuw nsw i4 %v1e0, %v2e0
968 %v1e1 = extractelement <8 x i4> %vec1, i64 1
969 %v2e1 = extractelement <8 x i4> %vec2, i64 1
970 %mul1 = mul nuw nsw i4 %v1e1, %v2e1
972 %v1e2 = extractelement <8 x i4> %vec1, i64 2
973 %v2e2 = extractelement <8 x i4> %vec2, i64 2
974 %mul2 = mul nuw nsw i4 %v1e2, %v2e2
976 %v1e3 = extractelement <8 x i4> %vec1, i64 3
977 %v2e3 = extractelement <8 x i4> %vec2, i64 3
978 %mul3 = mul nuw nsw i4 %v1e3, %v2e3
980 %v1e4 = extractelement <8 x i4> %vec1, i64 4
981 %v2e4 = extractelement <8 x i4> %vec2, i64 4
982 %mul4 = mul nuw nsw i4 %v1e4, %v2e4
984 %v1e5 = extractelement <8 x i4> %vec1, i64 5
985 %v2e5 = extractelement <8 x i4> %vec2, i64 5
986 %mul5 = mul nuw nsw i4 %v1e5, %v2e5
988 %v1e6 = extractelement <8 x i4> %vec1, i64 6
989 %v2e6 = extractelement <8 x i4> %vec2, i64 6
990 %mul6 = mul nuw nsw i4 %v1e6, %v2e6
992 %v1e7 = extractelement <8 x i4> %vec1, i64 7
993 %v2e7 = extractelement <8 x i4> %vec2, i64 7
994 %mul7 = mul nuw nsw i4 %v1e7, %v2e7
996 %acc = load i4, i4 addrspace(1)* %dst, align 4
997 %add1 = add i4 %mul0, %acc
998 %add2 = add i4 %add1, %mul1
999 %add3 = add i4 %add2, %mul2
1000 %add4 = add i4 %add3, %mul3
1001 %add5 = add i4 %add4, %mul4
1002 %add6 = add i4 %add5, %mul5
1003 %add7 = add i4 %add6, %mul6
1004 %add8 = add i4 %add7, %mul7
1006 store i4 %add8, i4 addrspace(1)* %dst, align 4
1010 ; TODO: Currently, permutation of udot8 is turned off due to a huge increase
1011 ; in the compile time.
1012 define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %src1,
1013 ; GFX7-LABEL: udot8_CommutationInsideMAD:
1014 ; GFX7: ; %bb.0: ; %entry
1015 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
1016 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd
1017 ; GFX7-NEXT: s_mov_b32 s11, 0xf000
1018 ; GFX7-NEXT: s_mov_b32 s10, -1
1019 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1020 ; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
1021 ; GFX7-NEXT: buffer_load_ubyte v0, off, s[8:11], 0
1022 ; GFX7-NEXT: s_load_dword s1, s[6:7], 0x0
1023 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1024 ; GFX7-NEXT: s_lshr_b32 s2, s0, 28
1025 ; GFX7-NEXT: s_bfe_u32 s4, s0, 0x40018
1026 ; GFX7-NEXT: s_bfe_u32 s15, s1, 0x40018
1027 ; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40014
1028 ; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40010
1029 ; GFX7-NEXT: s_bfe_u32 s18, s1, 0x4000c
1030 ; GFX7-NEXT: s_bfe_u32 s19, s1, 0x40008
1031 ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40004
1032 ; GFX7-NEXT: s_lshr_b32 s14, s1, 28
1033 ; GFX7-NEXT: s_and_b32 s1, s1, 15
1034 ; GFX7-NEXT: s_bfe_u32 s5, s0, 0x40014
1035 ; GFX7-NEXT: s_bfe_u32 s6, s0, 0x40010
1036 ; GFX7-NEXT: s_bfe_u32 s7, s0, 0x4000c
1037 ; GFX7-NEXT: s_bfe_u32 s12, s0, 0x40008
1038 ; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40004
1039 ; GFX7-NEXT: s_and_b32 s0, s0, 15
1040 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1041 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
1042 ; GFX7-NEXT: v_mov_b32_e32 v3, s19
1043 ; GFX7-NEXT: v_mov_b32_e32 v4, s18
1044 ; GFX7-NEXT: v_mov_b32_e32 v5, s17
1045 ; GFX7-NEXT: v_mov_b32_e32 v6, s16
1046 ; GFX7-NEXT: v_mov_b32_e32 v7, s15
1047 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1048 ; GFX7-NEXT: v_mad_u32_u24 v0, s0, v1, v0
1049 ; GFX7-NEXT: v_mad_u32_u24 v0, s13, v2, v0
1050 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v3, v0
1051 ; GFX7-NEXT: v_mad_u32_u24 v0, s7, v4, v0
1052 ; GFX7-NEXT: v_mad_u32_u24 v0, s6, v5, v0
1053 ; GFX7-NEXT: v_mad_u32_u24 v0, s5, v6, v0
1054 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v7, v0
1055 ; GFX7-NEXT: v_mov_b32_e32 v1, s14
1056 ; GFX7-NEXT: v_mad_u32_u24 v0, s2, v1, v0
1057 ; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
1058 ; GFX7-NEXT: buffer_store_byte v0, off, s[8:11], 0
1059 ; GFX7-NEXT: s_endpgm
1061 ; GFX8-LABEL: udot8_CommutationInsideMAD:
1062 ; GFX8: ; %bb.0: ; %entry
1063 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1064 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1065 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1066 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
1067 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
1068 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1069 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1070 ; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
1071 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1072 ; GFX8-NEXT: s_and_b32 s0, s2, 15
1073 ; GFX8-NEXT: s_and_b32 s1, s4, 15
1074 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
1075 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004
1076 ; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40008
1077 ; GFX8-NEXT: v_mov_b32_e32 v4, s5
1078 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004
1079 ; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c
1080 ; GFX8-NEXT: v_mov_b32_e32 v5, s6
1081 ; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40008
1082 ; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40010
1083 ; GFX8-NEXT: v_mov_b32_e32 v6, s7
1084 ; GFX8-NEXT: s_bfe_u32 s6, s2, 0x4000c
1085 ; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40014
1086 ; GFX8-NEXT: v_mov_b32_e32 v7, s8
1087 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40010
1088 ; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40018
1089 ; GFX8-NEXT: v_mov_b32_e32 v8, s9
1090 ; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40014
1091 ; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40018
1092 ; GFX8-NEXT: s_lshr_b32 s4, s4, 28
1093 ; GFX8-NEXT: v_mov_b32_e32 v9, s10
1094 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28
1095 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1096 ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
1097 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v4, v2
1098 ; GFX8-NEXT: v_mad_u32_u24 v2, s5, v5, v2
1099 ; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2
1100 ; GFX8-NEXT: v_mad_u32_u24 v2, s7, v7, v2
1101 ; GFX8-NEXT: v_mad_u32_u24 v2, s8, v8, v2
1102 ; GFX8-NEXT: v_mad_u32_u24 v2, s9, v9, v2
1103 ; GFX8-NEXT: v_mov_b32_e32 v3, s4
1104 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
1105 ; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
1106 ; GFX8-NEXT: flat_store_byte v[0:1], v2
1107 ; GFX8-NEXT: s_endpgm
1109 ; GFX9-LABEL: udot8_CommutationInsideMAD:
1110 ; GFX9: ; %bb.0: ; %entry
1111 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1112 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1113 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1114 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
1115 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
1116 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
1117 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
1118 ; GFX9-NEXT: global_load_ubyte v2, v[0:1], off
1119 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1120 ; GFX9-NEXT: s_and_b32 s0, s2, 15
1121 ; GFX9-NEXT: s_and_b32 s1, s4, 15
1122 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
1123 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004
1124 ; GFX9-NEXT: s_bfe_u32 s6, s4, 0x40008
1125 ; GFX9-NEXT: v_mov_b32_e32 v4, s5
1126 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004
1127 ; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c
1128 ; GFX9-NEXT: v_mov_b32_e32 v5, s6
1129 ; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40008
1130 ; GFX9-NEXT: s_bfe_u32 s8, s4, 0x40010
1131 ; GFX9-NEXT: v_mov_b32_e32 v6, s7
1132 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x4000c
1133 ; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40014
1134 ; GFX9-NEXT: v_mov_b32_e32 v7, s8
1135 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40010
1136 ; GFX9-NEXT: s_bfe_u32 s10, s4, 0x40018
1137 ; GFX9-NEXT: v_mov_b32_e32 v8, s9
1138 ; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40014
1139 ; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40018
1140 ; GFX9-NEXT: s_lshr_b32 s4, s4, 28
1141 ; GFX9-NEXT: v_mov_b32_e32 v9, s10
1142 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28
1143 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1144 ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2
1145 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v4, v2
1146 ; GFX9-NEXT: v_mad_u32_u24 v2, s5, v5, v2
1147 ; GFX9-NEXT: v_mad_u32_u24 v2, s6, v6, v2
1148 ; GFX9-NEXT: v_mad_u32_u24 v2, s7, v7, v2
1149 ; GFX9-NEXT: v_mad_u32_u24 v2, s8, v8, v2
1150 ; GFX9-NEXT: v_mad_u32_u24 v2, s9, v9, v2
1151 ; GFX9-NEXT: v_mov_b32_e32 v3, s4
1152 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v3, v2
1153 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
1154 ; GFX9-NEXT: global_store_byte v[0:1], v2, off
1155 ; GFX9-NEXT: s_endpgm
1157 ; GFX9-DL-LABEL: udot8_CommutationInsideMAD:
1158 ; GFX9-DL: ; %bb.0: ; %entry
1159 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1160 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1161 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1162 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
1163 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
1164 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
1165 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
1166 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off
1167 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1168 ; GFX9-DL-NEXT: s_and_b32 s0, s2, 15
1169 ; GFX9-DL-NEXT: s_and_b32 s1, s4, 15
1170 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
1171 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004
1172 ; GFX9-DL-NEXT: s_bfe_u32 s6, s4, 0x40008
1173 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s5
1174 ; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004
1175 ; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x4000c
1176 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s6
1177 ; GFX9-DL-NEXT: s_bfe_u32 s5, s2, 0x40008
1178 ; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x40010
1179 ; GFX9-DL-NEXT: v_mov_b32_e32 v6, s7
1180 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x4000c
1181 ; GFX9-DL-NEXT: s_bfe_u32 s9, s4, 0x40014
1182 ; GFX9-DL-NEXT: v_mov_b32_e32 v7, s8
1183 ; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x40010
1184 ; GFX9-DL-NEXT: s_bfe_u32 s10, s4, 0x40018
1185 ; GFX9-DL-NEXT: v_mov_b32_e32 v8, s9
1186 ; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x40014
1187 ; GFX9-DL-NEXT: s_bfe_u32 s9, s2, 0x40018
1188 ; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28
1189 ; GFX9-DL-NEXT: v_mov_b32_e32 v9, s10
1190 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28
1191 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
1192 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2
1193 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v4, v2
1194 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v5, v2
1195 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v6, v2
1196 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s7, v7, v2
1197 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s8, v8, v2
1198 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s9, v9, v2
1199 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4
1200 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2
1201 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
1202 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off
1203 ; GFX9-DL-NEXT: s_endpgm
1204 <8 x i4> addrspace(1)* %src2,
1205 i4 addrspace(1)* nocapture %dst) {
1207 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1208 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1210 %v1e0 = extractelement <8 x i4> %vec1, i64 0
1211 %v2e0 = extractelement <8 x i4> %vec2, i64 0
1212 %mul0 = mul nuw nsw i4 %v1e0, %v2e0
1214 %v1e1 = extractelement <8 x i4> %vec1, i64 1
1215 %v2e1 = extractelement <8 x i4> %vec2, i64 1
1216 %mul1 = mul nuw nsw i4 %v1e1, %v2e1
1218 %v1e2 = extractelement <8 x i4> %vec1, i64 2
1219 %v2e2 = extractelement <8 x i4> %vec2, i64 2
1220 %mul2 = mul nuw nsw i4 %v1e2, %v2e2
1222 %v1e3 = extractelement <8 x i4> %vec1, i64 3
1223 %v2e3 = extractelement <8 x i4> %vec2, i64 3
1224 %mul3 = mul nuw nsw i4 %v1e3, %v2e3
1226 %v1e4 = extractelement <8 x i4> %vec1, i64 4
1227 %v2e4 = extractelement <8 x i4> %vec2, i64 4
1228 %mul4 = mul nuw nsw i4 %v1e4, %v2e4
1230 %v1e5 = extractelement <8 x i4> %vec1, i64 5
1231 %v2e5 = extractelement <8 x i4> %vec2, i64 5
1232 %mul5 = mul nuw nsw i4 %v1e5, %v2e5
1234 %v1e6 = extractelement <8 x i4> %vec1, i64 6
1235 %v2e6 = extractelement <8 x i4> %vec2, i64 6
1236 %mul6 = mul nuw nsw i4 %v1e6, %v2e6
1238 %v1e7 = extractelement <8 x i4> %vec1, i64 7
1239 %v2e7 = extractelement <8 x i4> %vec2, i64 7
1240 %mul7 = mul nuw nsw i4 %v1e7, %v2e7
1242 %acc = load i4, i4 addrspace(1)* %dst, align 4
1243 %add1 = add i4 %mul0, %acc
1244 %add2 = add i4 %mul1, %add1
1245 %add3 = add i4 %mul2, %add2
1246 %add4 = add i4 %mul3, %add3
1247 %add5 = add i4 %mul4, %add4
1248 %add6 = add i4 %mul5, %add5
1249 %add7 = add i4 %mul6, %add6
1250 %add8 = add i4 %mul7, %add7
1252 store i4 %add8, i4 addrspace(1)* %dst, align 4
1256 define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
1257 ; GFX7-LABEL: udot8_multiuses_mul1:
1258 ; GFX7: ; %bb.0: ; %entry
1259 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
1260 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
1261 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1262 ; GFX7-NEXT: s_mov_b32 s6, -1
1263 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1264 ; GFX7-NEXT: s_load_dword s0, s[8:9], 0x0
1265 ; GFX7-NEXT: s_load_dword s1, s[10:11], 0x0
1266 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
1267 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1268 ; GFX7-NEXT: s_lshr_b32 s8, s0, 28
1269 ; GFX7-NEXT: s_bfe_u32 s21, s1, 0x40004
1270 ; GFX7-NEXT: s_lshr_b32 s15, s1, 28
1271 ; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40018
1272 ; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40014
1273 ; GFX7-NEXT: s_bfe_u32 s18, s1, 0x40010
1274 ; GFX7-NEXT: s_bfe_u32 s19, s1, 0x4000c
1275 ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40008
1276 ; GFX7-NEXT: s_and_b32 s1, s1, 15
1277 ; GFX7-NEXT: s_bfe_u32 s9, s0, 0x40018
1278 ; GFX7-NEXT: s_bfe_u32 s10, s0, 0x40014
1279 ; GFX7-NEXT: s_bfe_u32 s11, s0, 0x40010
1280 ; GFX7-NEXT: s_bfe_u32 s12, s0, 0x4000c
1281 ; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40008
1282 ; GFX7-NEXT: s_bfe_u32 s14, s0, 0x40004
1283 ; GFX7-NEXT: s_and_b32 s0, s0, 15
1284 ; GFX7-NEXT: v_mov_b32_e32 v0, s1
1285 ; GFX7-NEXT: v_mov_b32_e32 v1, s2
1286 ; GFX7-NEXT: v_mad_u32_u24 v1, s0, v0, v1
1287 ; GFX7-NEXT: v_mov_b32_e32 v2, s21
1288 ; GFX7-NEXT: v_mad_u32_u24 v0, s0, v0, v1
1289 ; GFX7-NEXT: v_mad_u32_u24 v1, s14, v2, v1
1290 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
1291 ; GFX7-NEXT: v_mad_u32_u24 v1, s13, v2, v1
1292 ; GFX7-NEXT: v_mov_b32_e32 v2, s19
1293 ; GFX7-NEXT: v_mad_u32_u24 v1, s12, v2, v1
1294 ; GFX7-NEXT: v_mov_b32_e32 v2, s18
1295 ; GFX7-NEXT: v_mad_u32_u24 v1, s11, v2, v1
1296 ; GFX7-NEXT: v_mov_b32_e32 v2, s17
1297 ; GFX7-NEXT: v_mad_u32_u24 v1, s10, v2, v1
1298 ; GFX7-NEXT: v_mov_b32_e32 v2, s16
1299 ; GFX7-NEXT: v_mad_u32_u24 v1, s9, v2, v1
1300 ; GFX7-NEXT: v_mov_b32_e32 v2, s15
1301 ; GFX7-NEXT: v_mad_u32_u24 v1, s8, v2, v1
1302 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v1, v0
1303 ; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
1304 ; GFX7-NEXT: s_endpgm
1306 ; GFX8-LABEL: udot8_multiuses_mul1:
1307 ; GFX8: ; %bb.0: ; %entry
1308 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1309 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1310 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1311 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
1312 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
1313 ; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
1314 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1315 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1316 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1317 ; GFX8-NEXT: s_lshr_b32 s0, s2, 28
1318 ; GFX8-NEXT: s_bfe_u32 s17, s4, 0x40004
1319 ; GFX8-NEXT: s_lshr_b32 s11, s4, 28
1320 ; GFX8-NEXT: s_bfe_u32 s12, s4, 0x40018
1321 ; GFX8-NEXT: s_bfe_u32 s13, s4, 0x40014
1322 ; GFX8-NEXT: s_bfe_u32 s14, s4, 0x40010
1323 ; GFX8-NEXT: s_bfe_u32 s15, s4, 0x4000c
1324 ; GFX8-NEXT: s_bfe_u32 s16, s4, 0x40008
1325 ; GFX8-NEXT: s_and_b32 s4, s4, 15
1326 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40018
1327 ; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40014
1328 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40010
1329 ; GFX8-NEXT: s_bfe_u32 s8, s2, 0x4000c
1330 ; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40008
1331 ; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40004
1332 ; GFX8-NEXT: s_and_b32 s2, s2, 15
1333 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
1334 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
1335 ; GFX8-NEXT: v_mad_u32_u24 v3, s2, v2, v3
1336 ; GFX8-NEXT: v_mov_b32_e32 v4, s17
1337 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v2, v3
1338 ; GFX8-NEXT: v_mad_u32_u24 v3, s10, v4, v3
1339 ; GFX8-NEXT: v_mov_b32_e32 v4, s16
1340 ; GFX8-NEXT: v_mad_u32_u24 v3, s9, v4, v3
1341 ; GFX8-NEXT: v_mov_b32_e32 v4, s15
1342 ; GFX8-NEXT: v_mad_u32_u24 v3, s8, v4, v3
1343 ; GFX8-NEXT: v_mov_b32_e32 v4, s14
1344 ; GFX8-NEXT: v_mad_u32_u24 v3, s7, v4, v3
1345 ; GFX8-NEXT: v_mov_b32_e32 v4, s13
1346 ; GFX8-NEXT: v_mad_u32_u24 v3, s6, v4, v3
1347 ; GFX8-NEXT: v_mov_b32_e32 v4, s12
1348 ; GFX8-NEXT: v_mad_u32_u24 v3, s1, v4, v3
1349 ; GFX8-NEXT: v_mov_b32_e32 v4, s11
1350 ; GFX8-NEXT: v_mad_u32_u24 v3, s0, v4, v3
1351 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2
1352 ; GFX8-NEXT: flat_store_dword v[0:1], v2
1353 ; GFX8-NEXT: s_endpgm
1355 ; GFX9-LABEL: udot8_multiuses_mul1:
1356 ; GFX9: ; %bb.0: ; %entry
1357 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1358 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1359 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1360 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
1361 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
1362 ; GFX9-NEXT: s_load_dword s5, s[0:1], 0x0
1363 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
1364 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
1365 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1366 ; GFX9-NEXT: s_lshr_b32 s0, s2, 28
1367 ; GFX9-NEXT: s_bfe_u32 s17, s4, 0x40004
1368 ; GFX9-NEXT: s_lshr_b32 s11, s4, 28
1369 ; GFX9-NEXT: s_bfe_u32 s12, s4, 0x40018
1370 ; GFX9-NEXT: s_bfe_u32 s13, s4, 0x40014
1371 ; GFX9-NEXT: s_bfe_u32 s14, s4, 0x40010
1372 ; GFX9-NEXT: s_bfe_u32 s15, s4, 0x4000c
1373 ; GFX9-NEXT: s_bfe_u32 s16, s4, 0x40008
1374 ; GFX9-NEXT: s_and_b32 s4, s4, 15
1375 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40018
1376 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40014
1377 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40010
1378 ; GFX9-NEXT: s_bfe_u32 s8, s2, 0x4000c
1379 ; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40008
1380 ; GFX9-NEXT: s_bfe_u32 s10, s2, 0x40004
1381 ; GFX9-NEXT: s_and_b32 s2, s2, 15
1382 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1383 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1384 ; GFX9-NEXT: v_mad_u32_u24 v3, s2, v2, v3
1385 ; GFX9-NEXT: v_mov_b32_e32 v4, s17
1386 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v2, v3
1387 ; GFX9-NEXT: v_mad_u32_u24 v3, s10, v4, v3
1388 ; GFX9-NEXT: v_mov_b32_e32 v4, s16
1389 ; GFX9-NEXT: v_mad_u32_u24 v3, s9, v4, v3
1390 ; GFX9-NEXT: v_mov_b32_e32 v4, s15
1391 ; GFX9-NEXT: v_mad_u32_u24 v3, s8, v4, v3
1392 ; GFX9-NEXT: v_mov_b32_e32 v4, s14
1393 ; GFX9-NEXT: v_mad_u32_u24 v3, s7, v4, v3
1394 ; GFX9-NEXT: v_mov_b32_e32 v4, s13
1395 ; GFX9-NEXT: v_mad_u32_u24 v3, s6, v4, v3
1396 ; GFX9-NEXT: v_mov_b32_e32 v4, s12
1397 ; GFX9-NEXT: v_mad_u32_u24 v3, s1, v4, v3
1398 ; GFX9-NEXT: v_mov_b32_e32 v4, s11
1399 ; GFX9-NEXT: v_mad_u32_u24 v3, s0, v4, v3
1400 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v3
1401 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
1402 ; GFX9-NEXT: s_endpgm
1404 ; GFX9-DL-LABEL: udot8_multiuses_mul1:
1405 ; GFX9-DL: ; %bb.0: ; %entry
1406 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1407 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1408 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1409 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
1410 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
1411 ; GFX9-DL-NEXT: s_load_dword s5, s[0:1], 0x0
1412 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
1413 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
1414 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1415 ; GFX9-DL-NEXT: s_lshr_b32 s0, s2, 28
1416 ; GFX9-DL-NEXT: s_bfe_u32 s17, s4, 0x40004
1417 ; GFX9-DL-NEXT: s_lshr_b32 s11, s4, 28
1418 ; GFX9-DL-NEXT: s_bfe_u32 s12, s4, 0x40018
1419 ; GFX9-DL-NEXT: s_bfe_u32 s13, s4, 0x40014
1420 ; GFX9-DL-NEXT: s_bfe_u32 s14, s4, 0x40010
1421 ; GFX9-DL-NEXT: s_bfe_u32 s15, s4, 0x4000c
1422 ; GFX9-DL-NEXT: s_bfe_u32 s16, s4, 0x40008
1423 ; GFX9-DL-NEXT: s_and_b32 s4, s4, 15
1424 ; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40018
1425 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x40014
1426 ; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x40010
1427 ; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x4000c
1428 ; GFX9-DL-NEXT: s_bfe_u32 s9, s2, 0x40008
1429 ; GFX9-DL-NEXT: s_bfe_u32 s10, s2, 0x40004
1430 ; GFX9-DL-NEXT: s_and_b32 s2, s2, 15
1431 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
1432 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5
1433 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, s2, v2, v3
1434 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s17
1435 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v2, v3
1436 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, s10, v4, v3
1437 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
1438 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, s9, v4, v3
1439 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s15
1440 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, s8, v4, v3
1441 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s14
1442 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, s7, v4, v3
1443 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s13
1444 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, s6, v4, v3
1445 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s12
1446 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, s1, v4, v3
1447 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s11
1448 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, s0, v4, v3
1449 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v3
1450 ; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off
1451 ; GFX9-DL-NEXT: s_endpgm
1452 <8 x i4> addrspace(1)* %src2,
1453 i32 addrspace(1)* nocapture %dst) {
1455 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1456 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1458 %v1e0 = extractelement <8 x i4> %vec1, i64 0
1459 %cv1e0 = zext i4 %v1e0 to i32
1460 %v2e0 = extractelement <8 x i4> %vec2, i64 0
1461 %cv2e0 = zext i4 %v2e0 to i32
1462 %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
1464 %v1e1 = extractelement <8 x i4> %vec1, i64 1
1465 %cv1e1 = zext i4 %v1e1 to i32
1466 %v2e1 = extractelement <8 x i4> %vec2, i64 1
1467 %cv2e1 = zext i4 %v2e1 to i32
1468 %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
1470 %v1e2 = extractelement <8 x i4> %vec1, i64 2
1471 %cv1e2 = zext i4 %v1e2 to i32
1472 %v2e2 = extractelement <8 x i4> %vec2, i64 2
1473 %cv2e2 = zext i4 %v2e2 to i32
1474 %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
1476 %v1e3 = extractelement <8 x i4> %vec1, i64 3
1477 %cv1e3 = zext i4 %v1e3 to i32
1478 %v2e3 = extractelement <8 x i4> %vec2, i64 3
1479 %cv2e3 = zext i4 %v2e3 to i32
1480 %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
1482 %v1e4 = extractelement <8 x i4> %vec1, i64 4
1483 %cv1e4 = zext i4 %v1e4 to i32
1484 %v2e4 = extractelement <8 x i4> %vec2, i64 4
1485 %cv2e4 = zext i4 %v2e4 to i32
1486 %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
1488 %v1e5 = extractelement <8 x i4> %vec1, i64 5
1489 %cv1e5 = zext i4 %v1e5 to i32
1490 %v2e5 = extractelement <8 x i4> %vec2, i64 5
1491 %cv2e5 = zext i4 %v2e5 to i32
1492 %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
1494 %v1e6 = extractelement <8 x i4> %vec1, i64 6
1495 %cv1e6 = zext i4 %v1e6 to i32
1496 %v2e6 = extractelement <8 x i4> %vec2, i64 6
1497 %cv2e6 = zext i4 %v2e6 to i32
1498 %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
1500 %v1e7 = extractelement <8 x i4> %vec1, i64 7
1501 %cv1e7 = zext i4 %v1e7 to i32
1502 %v2e7 = extractelement <8 x i4> %vec2, i64 7
1503 %cv2e7 = zext i4 %v2e7 to i32
1504 %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
1506 %acc = load i32, i32 addrspace(1)* %dst, align 4
1507 %add1 = add i32 %mul0, %acc
1508 %add = add i32 %mul0, %add1
1509 %add2 = add i32 %add1, %mul1
1510 %add3 = add i32 %add2, %mul2
1511 %add4 = add i32 %add3, %mul3
1512 %add5 = add i32 %add4, %mul4
1513 %add6 = add i32 %add5, %mul5
1514 %add7 = add i32 %add6, %mul6
1515 %add8 = add i32 %add7, %mul7
1517 %res = add i32 %add, %add8
1518 store i32 %res, i32 addrspace(1)* %dst, align 4
1522 define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
1523 ; GFX7-LABEL: udot8_acc32_vecMul:
1524 ; GFX7: ; %bb.0: ; %entry
1525 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
1526 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
1527 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1528 ; GFX7-NEXT: s_mov_b32 s6, -1
1529 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1530 ; GFX7-NEXT: s_load_dword s0, s[8:9], 0x0
1531 ; GFX7-NEXT: s_load_dword s1, s[10:11], 0x0
1532 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
1533 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1534 ; GFX7-NEXT: s_lshr_b32 s8, s0, 28
1535 ; GFX7-NEXT: s_lshr_b32 s15, s1, 28
1536 ; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40018
1537 ; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40014
1538 ; GFX7-NEXT: s_bfe_u32 s18, s1, 0x40010
1539 ; GFX7-NEXT: s_bfe_u32 s19, s1, 0x4000c
1540 ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40008
1541 ; GFX7-NEXT: s_bfe_u32 s21, s1, 0x40004
1542 ; GFX7-NEXT: s_and_b32 s1, s1, 15
1543 ; GFX7-NEXT: s_bfe_u32 s9, s0, 0x40018
1544 ; GFX7-NEXT: s_bfe_u32 s10, s0, 0x40014
1545 ; GFX7-NEXT: s_bfe_u32 s11, s0, 0x40010
1546 ; GFX7-NEXT: s_bfe_u32 s12, s0, 0x4000c
1547 ; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40008
1548 ; GFX7-NEXT: s_bfe_u32 s14, s0, 0x40004
1549 ; GFX7-NEXT: s_and_b32 s0, s0, 15
1550 ; GFX7-NEXT: v_mov_b32_e32 v0, s1
1551 ; GFX7-NEXT: v_mov_b32_e32 v1, s2
1552 ; GFX7-NEXT: v_mad_u32_u24 v0, s0, v0, v1
1553 ; GFX7-NEXT: v_mov_b32_e32 v1, s21
1554 ; GFX7-NEXT: v_mad_u32_u24 v0, s14, v1, v0
1555 ; GFX7-NEXT: v_mov_b32_e32 v1, s20
1556 ; GFX7-NEXT: v_mad_u32_u24 v0, s13, v1, v0
1557 ; GFX7-NEXT: v_mov_b32_e32 v1, s19
1558 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v1, v0
1559 ; GFX7-NEXT: v_mov_b32_e32 v1, s18
1560 ; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0
1561 ; GFX7-NEXT: v_mov_b32_e32 v1, s17
1562 ; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
1563 ; GFX7-NEXT: v_mov_b32_e32 v1, s16
1564 ; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0
1565 ; GFX7-NEXT: v_mov_b32_e32 v1, s15
1566 ; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0
1567 ; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
1568 ; GFX7-NEXT: s_endpgm
1570 ; GFX8-LABEL: udot8_acc32_vecMul:
1571 ; GFX8: ; %bb.0: ; %entry
1572 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1573 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1574 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1575 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
1576 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
1577 ; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
1578 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1579 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1580 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1581 ; GFX8-NEXT: s_lshr_b32 s0, s2, 28
1582 ; GFX8-NEXT: s_lshr_b32 s11, s4, 28
1583 ; GFX8-NEXT: s_bfe_u32 s12, s4, 0x40018
1584 ; GFX8-NEXT: s_bfe_u32 s13, s4, 0x40014
1585 ; GFX8-NEXT: s_bfe_u32 s14, s4, 0x40010
1586 ; GFX8-NEXT: s_bfe_u32 s15, s4, 0x4000c
1587 ; GFX8-NEXT: s_bfe_u32 s16, s4, 0x40008
1588 ; GFX8-NEXT: s_bfe_u32 s17, s4, 0x40004
1589 ; GFX8-NEXT: s_and_b32 s4, s4, 15
1590 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40018
1591 ; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40014
1592 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40010
1593 ; GFX8-NEXT: s_bfe_u32 s8, s2, 0x4000c
1594 ; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40008
1595 ; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40004
1596 ; GFX8-NEXT: s_and_b32 s2, s2, 15
1597 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
1598 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
1599 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v2, v3
1600 ; GFX8-NEXT: v_mov_b32_e32 v3, s17
1601 ; GFX8-NEXT: v_mad_u32_u24 v2, s10, v3, v2
1602 ; GFX8-NEXT: v_mov_b32_e32 v3, s16
1603 ; GFX8-NEXT: v_mad_u32_u24 v2, s9, v3, v2
1604 ; GFX8-NEXT: v_mov_b32_e32 v3, s15
1605 ; GFX8-NEXT: v_mad_u32_u24 v2, s8, v3, v2
1606 ; GFX8-NEXT: v_mov_b32_e32 v3, s14
1607 ; GFX8-NEXT: v_mad_u32_u24 v2, s7, v3, v2
1608 ; GFX8-NEXT: v_mov_b32_e32 v3, s13
1609 ; GFX8-NEXT: v_mad_u32_u24 v2, s6, v3, v2
1610 ; GFX8-NEXT: v_mov_b32_e32 v3, s12
1611 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2
1612 ; GFX8-NEXT: v_mov_b32_e32 v3, s11
1613 ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
1614 ; GFX8-NEXT: flat_store_dword v[0:1], v2
1615 ; GFX8-NEXT: s_endpgm
1617 ; GFX9-LABEL: udot8_acc32_vecMul:
1618 ; GFX9: ; %bb.0: ; %entry
1619 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1620 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1621 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1622 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
1623 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
1624 ; GFX9-NEXT: s_load_dword s5, s[0:1], 0x0
1625 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
1626 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
1627 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1628 ; GFX9-NEXT: s_lshr_b32 s0, s2, 28
1629 ; GFX9-NEXT: s_lshr_b32 s11, s4, 28
1630 ; GFX9-NEXT: s_bfe_u32 s12, s4, 0x40018
1631 ; GFX9-NEXT: s_bfe_u32 s13, s4, 0x40014
1632 ; GFX9-NEXT: s_bfe_u32 s14, s4, 0x40010
1633 ; GFX9-NEXT: s_bfe_u32 s15, s4, 0x4000c
1634 ; GFX9-NEXT: s_bfe_u32 s16, s4, 0x40008
1635 ; GFX9-NEXT: s_bfe_u32 s17, s4, 0x40004
1636 ; GFX9-NEXT: s_and_b32 s4, s4, 15
1637 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40018
1638 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40014
1639 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40010
1640 ; GFX9-NEXT: s_bfe_u32 s8, s2, 0x4000c
1641 ; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40008
1642 ; GFX9-NEXT: s_bfe_u32 s10, s2, 0x40004
1643 ; GFX9-NEXT: s_and_b32 s2, s2, 15
1644 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1645 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1646 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v2, v3
1647 ; GFX9-NEXT: v_mov_b32_e32 v3, s17
1648 ; GFX9-NEXT: v_mad_u32_u24 v2, s10, v3, v2
1649 ; GFX9-NEXT: v_mov_b32_e32 v3, s16
1650 ; GFX9-NEXT: v_mad_u32_u24 v2, s9, v3, v2
1651 ; GFX9-NEXT: v_mov_b32_e32 v3, s15
1652 ; GFX9-NEXT: v_mad_u32_u24 v2, s8, v3, v2
1653 ; GFX9-NEXT: v_mov_b32_e32 v3, s14
1654 ; GFX9-NEXT: v_mad_u32_u24 v2, s7, v3, v2
1655 ; GFX9-NEXT: v_mov_b32_e32 v3, s13
1656 ; GFX9-NEXT: v_mad_u32_u24 v2, s6, v3, v2
1657 ; GFX9-NEXT: v_mov_b32_e32 v3, s12
1658 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v3, v2
1659 ; GFX9-NEXT: v_mov_b32_e32 v3, s11
1660 ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2
1661 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
1662 ; GFX9-NEXT: s_endpgm
1664 ; GFX9-DL-LABEL: udot8_acc32_vecMul:
1665 ; GFX9-DL: ; %bb.0: ; %entry
1666 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1667 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1668 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1669 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
1670 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
1671 ; GFX9-DL-NEXT: s_load_dword s5, s[0:1], 0x0
1672 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
1673 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
1674 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1675 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
1676 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5
1677 ; GFX9-DL-NEXT: v_dot8_u32_u4 v2, s2, v2, v3
1678 ; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off
1679 ; GFX9-DL-NEXT: s_endpgm
1680 <8 x i4> addrspace(1)* %src2,
1681 i32 addrspace(1)* nocapture %dst) {
1683 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1684 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1686 %cvec1 = zext <8 x i4> %vec1 to <8 x i32>
1687 %cvec2 = zext <8 x i4> %vec2 to <8 x i32>
1689 %mul = mul <8 x i32> %cvec1, %cvec2
1690 %mul0 = extractelement <8 x i32> %mul, i64 0
1691 %mul1 = extractelement <8 x i32> %mul, i64 1
1692 %mul2 = extractelement <8 x i32> %mul, i64 2
1693 %mul3 = extractelement <8 x i32> %mul, i64 3
1694 %mul4 = extractelement <8 x i32> %mul, i64 4
1695 %mul5 = extractelement <8 x i32> %mul, i64 5
1696 %mul6 = extractelement <8 x i32> %mul, i64 6
1697 %mul7 = extractelement <8 x i32> %mul, i64 7
1699 %acc = load i32, i32 addrspace(1)* %dst, align 4
1700 %add1 = add i32 %mul0, %acc
1701 %add2 = add i32 %add1, %mul1
1702 %add3 = add i32 %add2, %mul2
1703 %add4 = add i32 %add3, %mul3
1704 %add5 = add i32 %add4, %mul4
1705 %add6 = add i32 %add5, %mul5
1706 %add7 = add i32 %add6, %mul6
1707 %add8 = add i32 %add7, %mul7
1709 store i32 %add8, i32 addrspace(1)* %dst, align 4
1713 ; TODO: Clean up the code(by default pk_mad_I16 should be generated), then
1714 ; support the pattern.
1715 define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
1716 ; GFX7-LABEL: udot8_acc16_vecMul:
1717 ; GFX7: ; %bb.0: ; %entry
1718 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
1719 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
1720 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1721 ; GFX7-NEXT: s_mov_b32 s6, -1
1722 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1723 ; GFX7-NEXT: s_load_dword s0, s[8:9], 0x0
1724 ; GFX7-NEXT: buffer_load_ushort v0, off, s[4:7], 0
1725 ; GFX7-NEXT: s_load_dword s1, s[10:11], 0x0
1726 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1727 ; GFX7-NEXT: s_bfe_u32 s11, s0, 0x40004
1728 ; GFX7-NEXT: s_bfe_u32 s13, s0, 0x4000c
1729 ; GFX7-NEXT: s_bfe_u32 s18, s1, 0x40004
1730 ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x4000c
1731 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
1732 ; GFX7-NEXT: v_mov_b32_e32 v4, s18
1733 ; GFX7-NEXT: s_bfe_u32 s14, s1, 0x40014
1734 ; GFX7-NEXT: s_bfe_u32 s15, s1, 0x40010
1735 ; GFX7-NEXT: s_lshr_b32 s16, s1, 28
1736 ; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40018
1737 ; GFX7-NEXT: s_and_b32 s19, s1, 15
1738 ; GFX7-NEXT: s_bfe_u32 s1, s1, 0x40008
1739 ; GFX7-NEXT: v_mul_u32_u24_e32 v2, s13, v2
1740 ; GFX7-NEXT: v_mul_u32_u24_e32 v4, s11, v4
1741 ; GFX7-NEXT: s_bfe_u32 s2, s0, 0x40014
1742 ; GFX7-NEXT: s_bfe_u32 s8, s0, 0x40010
1743 ; GFX7-NEXT: s_lshr_b32 s9, s0, 28
1744 ; GFX7-NEXT: v_mov_b32_e32 v6, s16
1745 ; GFX7-NEXT: s_bfe_u32 s10, s0, 0x40018
1746 ; GFX7-NEXT: s_and_b32 s12, s0, 15
1747 ; GFX7-NEXT: v_mov_b32_e32 v3, s19
1748 ; GFX7-NEXT: s_bfe_u32 s0, s0, 0x40008
1749 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1750 ; GFX7-NEXT: v_mov_b32_e32 v5, s17
1751 ; GFX7-NEXT: v_mul_u32_u24_e32 v6, s9, v6
1752 ; GFX7-NEXT: v_mul_u32_u24_e32 v1, s0, v1
1753 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
1754 ; GFX7-NEXT: v_mul_u32_u24_e32 v3, s12, v3
1755 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
1756 ; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
1757 ; GFX7-NEXT: v_or_b32_e32 v2, v3, v4
1758 ; GFX7-NEXT: v_mul_u32_u24_e32 v5, s10, v5
1759 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
1760 ; GFX7-NEXT: v_mov_b32_e32 v8, s14
1761 ; GFX7-NEXT: v_or_b32_e32 v3, v5, v6
1762 ; GFX7-NEXT: v_alignbit_b32 v5, v1, v2, 16
1763 ; GFX7-NEXT: v_mov_b32_e32 v7, s15
1764 ; GFX7-NEXT: v_mul_u32_u24_e32 v8, s2, v8
1765 ; GFX7-NEXT: v_mul_u32_u24_e32 v7, s8, v7
1766 ; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
1767 ; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v1
1768 ; GFX7-NEXT: v_or_b32_e32 v4, v7, v8
1769 ; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v4
1770 ; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v3
1771 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1772 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
1773 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v5, v0
1774 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
1775 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v6, v0
1776 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0
1777 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v7, v0
1778 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0
1779 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v8, v0
1780 ; GFX7-NEXT: buffer_store_short v0, off, s[4:7], 0
1781 ; GFX7-NEXT: s_endpgm
1783 ; GFX8-LABEL: udot8_acc16_vecMul:
1784 ; GFX8: ; %bb.0: ; %entry
1785 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1786 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1787 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1788 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
1789 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
1790 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1791 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1792 ; GFX8-NEXT: flat_load_ushort v2, v[0:1]
1793 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1794 ; GFX8-NEXT: s_and_b32 s0, s2, 15
1795 ; GFX8-NEXT: s_and_b32 s1, s4, 15
1796 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
1797 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004
1798 ; GFX8-NEXT: v_mov_b32_e32 v4, s5
1799 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004
1800 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40008
1801 ; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40010
1802 ; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40014
1803 ; GFX8-NEXT: s_bfe_u32 s12, s4, 0x40018
1804 ; GFX8-NEXT: s_lshr_b32 s14, s4, 28
1805 ; GFX8-NEXT: s_bfe_u32 s4, s4, 0x4000c
1806 ; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40008
1807 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
1808 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x4000c
1809 ; GFX8-NEXT: v_mov_b32_e32 v6, s4
1810 ; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40010
1811 ; GFX8-NEXT: v_mov_b32_e32 v7, s8
1812 ; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40014
1813 ; GFX8-NEXT: v_mov_b32_e32 v8, s10
1814 ; GFX8-NEXT: s_bfe_u32 s13, s2, 0x40018
1815 ; GFX8-NEXT: v_mov_b32_e32 v9, s12
1816 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28
1817 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1818 ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
1819 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v4, v2
1820 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v2
1821 ; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2
1822 ; GFX8-NEXT: v_mad_u32_u24 v2, s7, v6, v2
1823 ; GFX8-NEXT: v_mad_u32_u24 v2, s9, v7, v2
1824 ; GFX8-NEXT: v_mad_u32_u24 v2, s11, v8, v2
1825 ; GFX8-NEXT: v_mad_u32_u24 v2, s13, v9, v2
1826 ; GFX8-NEXT: v_mov_b32_e32 v3, s14
1827 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
1828 ; GFX8-NEXT: flat_store_short v[0:1], v2
1829 ; GFX8-NEXT: s_endpgm
1831 ; GFX9-LABEL: udot8_acc16_vecMul:
1832 ; GFX9: ; %bb.0: ; %entry
1833 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1834 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1835 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1836 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
1837 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
1838 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
1839 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
1840 ; GFX9-NEXT: global_load_ushort v2, v[0:1], off
1841 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1842 ; GFX9-NEXT: s_and_b32 s0, s2, 15
1843 ; GFX9-NEXT: s_and_b32 s1, s4, 15
1844 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004
1845 ; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s5
1846 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40004
1847 ; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s6
1848 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
1849 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40008
1850 ; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c
1851 ; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s7
1852 ; GFX9-NEXT: v_pk_mul_lo_u16 v3, s0, v3
1853 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40008
1854 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x4000c
1855 ; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s6
1856 ; GFX9-NEXT: v_mov_b32_e32 v4, s5
1857 ; GFX9-NEXT: s_bfe_u32 s0, s4, 0x40010
1858 ; GFX9-NEXT: s_bfe_u32 s7, s4, 0x40014
1859 ; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s7
1860 ; GFX9-NEXT: v_pk_mul_lo_u16 v4, s1, v4
1861 ; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40010
1862 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40014
1863 ; GFX9-NEXT: s_bfe_u32 s1, s4, 0x40018
1864 ; GFX9-NEXT: s_lshr_b32 s4, s4, 28
1865 ; GFX9-NEXT: v_mov_b32_e32 v5, s0
1866 ; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s6
1867 ; GFX9-NEXT: s_bfe_u32 s0, s2, 0x40018
1868 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28
1869 ; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
1870 ; GFX9-NEXT: v_pk_mul_lo_u16 v5, s5, v5
1871 ; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
1872 ; GFX9-NEXT: v_mov_b32_e32 v6, s1
1873 ; GFX9-NEXT: v_pk_mul_lo_u16 v6, s0, v6
1874 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1875 ; GFX9-NEXT: v_add_u32_e32 v2, v3, v2
1876 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1877 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
1878 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1879 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v5
1880 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1881 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v6
1882 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1883 ; GFX9-NEXT: global_store_short v[0:1], v2, off
1884 ; GFX9-NEXT: s_endpgm
1886 ; GFX9-DL-LABEL: udot8_acc16_vecMul:
1887 ; GFX9-DL: ; %bb.0: ; %entry
1888 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1889 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1890 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1891 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
1892 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
1893 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
1894 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
1895 ; GFX9-DL-NEXT: global_load_ushort v2, v[0:1], off
1896 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1897 ; GFX9-DL-NEXT: s_and_b32 s0, s2, 15
1898 ; GFX9-DL-NEXT: s_and_b32 s1, s4, 15
1899 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004
1900 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s1, s5
1901 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x40004
1902 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s6
1903 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
1904 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40008
1905 ; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x4000c
1906 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s5, s5, s7
1907 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, s0, v3
1908 ; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40008
1909 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x4000c
1910 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s1, s6
1911 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s5
1912 ; GFX9-DL-NEXT: s_bfe_u32 s0, s4, 0x40010
1913 ; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x40014
1914 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s7
1915 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, s1, v4
1916 ; GFX9-DL-NEXT: s_bfe_u32 s5, s2, 0x40010
1917 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x40014
1918 ; GFX9-DL-NEXT: s_bfe_u32 s1, s4, 0x40018
1919 ; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28
1920 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s0
1921 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s5, s5, s6
1922 ; GFX9-DL-NEXT: s_bfe_u32 s0, s2, 0x40018
1923 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28
1924 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s1, s4
1925 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v5, s5, v5
1926 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s2
1927 ; GFX9-DL-NEXT: v_mov_b32_e32 v6, s1
1928 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v6, s0, v6
1929 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
1930 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v3, v2
1931 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1932 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
1933 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1934 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v5
1935 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1936 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v6
1937 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1938 ; GFX9-DL-NEXT: global_store_short v[0:1], v2, off
1939 ; GFX9-DL-NEXT: s_endpgm
1940 <8 x i4> addrspace(1)* %src2,
1941 i16 addrspace(1)* nocapture %dst) {
1943 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1944 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1946 %cvec1 = zext <8 x i4> %vec1 to <8 x i16>
1947 %cvec2 = zext <8 x i4> %vec2 to <8 x i16>
1949 %mul = mul <8 x i16> %cvec1, %cvec2
1950 %mul0 = extractelement <8 x i16> %mul, i64 0
1951 %mul1 = extractelement <8 x i16> %mul, i64 1
1952 %mul2 = extractelement <8 x i16> %mul, i64 2
1953 %mul3 = extractelement <8 x i16> %mul, i64 3
1954 %mul4 = extractelement <8 x i16> %mul, i64 4
1955 %mul5 = extractelement <8 x i16> %mul, i64 5
1956 %mul6 = extractelement <8 x i16> %mul, i64 6
1957 %mul7 = extractelement <8 x i16> %mul, i64 7
1959 %acc = load i16, i16 addrspace(1)* %dst, align 4
1960 %add1 = add i16 %mul0, %acc
1961 %add2 = add i16 %add1, %mul1
1962 %add3 = add i16 %add2, %mul2
1963 %add4 = add i16 %add3, %mul3
1964 %add5 = add i16 %add4, %mul4
1965 %add6 = add i16 %add5, %mul5
1966 %add7 = add i16 %add6, %mul6
1967 %add8 = add i16 %add7, %mul7
1969 store i16 %add8, i16 addrspace(1)* %dst, align 4
1973 ; TODO: Cleanup the code to generate MAD; pattern should be recognized then.
1974 define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
1975 ; GFX7-LABEL: udot8_acc8_vecMul:
1976 ; GFX7: ; %bb.0: ; %entry
1977 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
1978 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
1979 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1980 ; GFX7-NEXT: s_mov_b32 s6, -1
1981 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1982 ; GFX7-NEXT: s_load_dword s0, s[8:9], 0x0
1983 ; GFX7-NEXT: buffer_load_ubyte v0, off, s[4:7], 0
1984 ; GFX7-NEXT: s_load_dword s1, s[10:11], 0x0
1985 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1986 ; GFX7-NEXT: s_bfe_u32 s2, s0, 0x4000c
1987 ; GFX7-NEXT: s_lshr_b32 s11, s0, 28
1988 ; GFX7-NEXT: s_bfe_u32 s14, s1, 0x4000c
1989 ; GFX7-NEXT: s_lshr_b32 s18, s1, 28
1990 ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40014
1991 ; GFX7-NEXT: v_mov_b32_e32 v4, s18
1992 ; GFX7-NEXT: v_mov_b32_e32 v8, s14
1993 ; GFX7-NEXT: s_bfe_u32 s15, s1, 0x40008
1994 ; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40004
1995 ; GFX7-NEXT: s_and_b32 s17, s1, 15
1996 ; GFX7-NEXT: s_bfe_u32 s19, s1, 0x40018
1997 ; GFX7-NEXT: s_bfe_u32 s1, s1, 0x40010
1998 ; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40014
1999 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
2000 ; GFX7-NEXT: v_mul_u32_u24_e32 v2, s13, v2
2001 ; GFX7-NEXT: v_mul_u32_u24_e32 v4, s11, v4
2002 ; GFX7-NEXT: v_mul_u32_u24_e32 v8, s2, v8
2003 ; GFX7-NEXT: s_bfe_u32 s8, s0, 0x40008
2004 ; GFX7-NEXT: v_mov_b32_e32 v7, s15
2005 ; GFX7-NEXT: s_bfe_u32 s9, s0, 0x40004
2006 ; GFX7-NEXT: v_mov_b32_e32 v6, s16
2007 ; GFX7-NEXT: s_and_b32 s10, s0, 15
2008 ; GFX7-NEXT: s_bfe_u32 s12, s0, 0x40018
2009 ; GFX7-NEXT: v_mov_b32_e32 v3, s19
2010 ; GFX7-NEXT: s_bfe_u32 s0, s0, 0x40010
2011 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2012 ; GFX7-NEXT: v_mov_b32_e32 v5, s17
2013 ; GFX7-NEXT: v_mul_u32_u24_e32 v6, s9, v6
2014 ; GFX7-NEXT: v_mul_u32_u24_e32 v1, s0, v1
2015 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
2016 ; GFX7-NEXT: v_mul_u32_u24_e32 v3, s12, v3
2017 ; GFX7-NEXT: v_mul_u32_u24_e32 v7, s8, v7
2018 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v4
2019 ; GFX7-NEXT: v_lshlrev_b32_e32 v8, 8, v8
2020 ; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
2021 ; GFX7-NEXT: v_or_b32_e32 v2, v3, v4
2022 ; GFX7-NEXT: v_or_b32_e32 v4, v7, v8
2023 ; GFX7-NEXT: v_mul_u32_u24_e32 v5, s10, v5
2024 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v6
2025 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
2026 ; GFX7-NEXT: v_or_b32_e32 v3, v5, v6
2027 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
2028 ; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
2029 ; GFX7-NEXT: v_or_b32_e32 v2, v3, v4
2030 ; GFX7-NEXT: v_alignbit_b32 v3, v1, v2, 8
2031 ; GFX7-NEXT: v_alignbit_b32 v4, v1, v2, 16
2032 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v2
2033 ; GFX7-NEXT: v_lshrrev_b32_e32 v6, 8, v1
2034 ; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v1
2035 ; GFX7-NEXT: v_lshrrev_b32_e32 v8, 24, v1
2036 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2037 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
2038 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0
2039 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0
2040 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v5, v0
2041 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
2042 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v6, v0
2043 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v7, v0
2044 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v8, v0
2045 ; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0
2046 ; GFX7-NEXT: s_endpgm
2048 ; GFX8-LABEL: udot8_acc8_vecMul:
2049 ; GFX8: ; %bb.0: ; %entry
2050 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2051 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2052 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2053 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
2054 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
2055 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
2056 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
2057 ; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
2058 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2059 ; GFX8-NEXT: s_bfe_u32 s0, s2, 0x40004
2060 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x4000c
2061 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004
2062 ; GFX8-NEXT: s_and_b32 s6, s4, 15
2063 ; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c
2064 ; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40008
2065 ; GFX8-NEXT: v_mov_b32_e32 v3, s8
2066 ; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40008
2067 ; GFX8-NEXT: v_mov_b32_e32 v4, s7
2068 ; GFX8-NEXT: v_mov_b32_e32 v5, s1
2069 ; GFX8-NEXT: v_mov_b32_e32 v6, s6
2070 ; GFX8-NEXT: v_mov_b32_e32 v7, s5
2071 ; GFX8-NEXT: v_mov_b32_e32 v8, s0
2072 ; GFX8-NEXT: s_and_b32 s9, s2, 15
2073 ; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2074 ; GFX8-NEXT: v_mul_u32_u24_e32 v3, s10, v3
2075 ; GFX8-NEXT: v_mul_u32_u24_e32 v5, s9, v6
2076 ; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2077 ; GFX8-NEXT: v_or_b32_e32 v5, v5, v6
2078 ; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2079 ; GFX8-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
2080 ; GFX8-NEXT: s_bfe_u32 s0, s2, 0x40014
2081 ; GFX8-NEXT: s_lshr_b32 s1, s2, 28
2082 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40014
2083 ; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40010
2084 ; GFX8-NEXT: s_lshr_b32 s7, s4, 28
2085 ; GFX8-NEXT: s_bfe_u32 s4, s4, 0x40018
2086 ; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40010
2087 ; GFX8-NEXT: s_bfe_u32 s2, s2, 0x40018
2088 ; GFX8-NEXT: v_mov_b32_e32 v6, s4
2089 ; GFX8-NEXT: v_mov_b32_e32 v7, s7
2090 ; GFX8-NEXT: v_mov_b32_e32 v8, s1
2091 ; GFX8-NEXT: v_mov_b32_e32 v9, s6
2092 ; GFX8-NEXT: v_mov_b32_e32 v10, s5
2093 ; GFX8-NEXT: v_mov_b32_e32 v11, s0
2094 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 8, v3
2095 ; GFX8-NEXT: v_mul_u32_u24_sdwa v7, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2096 ; GFX8-NEXT: v_mul_u32_u24_e32 v6, s2, v6
2097 ; GFX8-NEXT: v_mul_u32_u24_e32 v8, s8, v9
2098 ; GFX8-NEXT: v_mul_u32_u24_sdwa v9, v11, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2099 ; GFX8-NEXT: v_or_b32_e32 v8, v8, v9
2100 ; GFX8-NEXT: v_or_b32_sdwa v6, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2101 ; GFX8-NEXT: v_or_b32_sdwa v4, v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
2102 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v4
2103 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2104 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
2105 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v5, v2
2106 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2107 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2108 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4
2109 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v6
2110 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2111 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2112 ; GFX8-NEXT: flat_store_byte v[0:1], v2
2113 ; GFX8-NEXT: s_endpgm
2115 ; GFX9-LABEL: udot8_acc8_vecMul:
2116 ; GFX9: ; %bb.0: ; %entry
2117 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2118 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2119 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
2120 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
2121 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
2122 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
2123 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
2124 ; GFX9-NEXT: global_load_ubyte v2, v[0:1], off
2125 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
2126 ; GFX9-NEXT: s_and_b32 s0, s2, 15
2127 ; GFX9-NEXT: s_and_b32 s1, s4, 15
2128 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004
2129 ; GFX9-NEXT: s_bfe_u32 s6, s4, 0x40008
2130 ; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c
2131 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
2132 ; GFX9-NEXT: v_mov_b32_e32 v4, s5
2133 ; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40004
2134 ; GFX9-NEXT: v_mov_b32_e32 v5, s6
2135 ; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40008
2136 ; GFX9-NEXT: v_mov_b32_e32 v6, s7
2137 ; GFX9-NEXT: s_bfe_u32 s10, s2, 0x4000c
2138 ; GFX9-NEXT: v_mul_lo_u16_e32 v3, s0, v3
2139 ; GFX9-NEXT: v_mul_lo_u16_sdwa v4, s8, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2140 ; GFX9-NEXT: v_mul_lo_u16_e32 v5, s9, v5
2141 ; GFX9-NEXT: v_mul_lo_u16_sdwa v6, s10, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2142 ; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
2143 ; GFX9-NEXT: v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2144 ; GFX9-NEXT: s_bfe_u32 s1, s4, 0x40014
2145 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40018
2146 ; GFX9-NEXT: s_bfe_u32 s0, s4, 0x40010
2147 ; GFX9-NEXT: s_lshr_b32 s4, s4, 28
2148 ; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
2149 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40010
2150 ; GFX9-NEXT: v_mov_b32_e32 v4, s0
2151 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40014
2152 ; GFX9-NEXT: v_mov_b32_e32 v5, s1
2153 ; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40018
2154 ; GFX9-NEXT: v_mov_b32_e32 v6, s5
2155 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28
2156 ; GFX9-NEXT: v_mov_b32_e32 v7, s4
2157 ; GFX9-NEXT: v_mul_lo_u16_e32 v4, s6, v4
2158 ; GFX9-NEXT: v_mul_lo_u16_sdwa v5, s7, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2159 ; GFX9-NEXT: v_mul_lo_u16_e32 v6, s8, v6
2160 ; GFX9-NEXT: v_mul_lo_u16_sdwa v7, s2, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2161 ; GFX9-NEXT: v_or_b32_e32 v4, v4, v5
2162 ; GFX9-NEXT: v_or_b32_sdwa v5, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2163 ; GFX9-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
2164 ; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v3
2165 ; GFX9-NEXT: s_waitcnt vmcnt(0)
2166 ; GFX9-NEXT: v_add_u32_e32 v2, v3, v2
2167 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v5
2168 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2169 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2170 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v4
2171 ; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v4
2172 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v3
2173 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2174 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2175 ; GFX9-NEXT: global_store_byte v[0:1], v2, off
2176 ; GFX9-NEXT: s_endpgm
2178 ; GFX9-DL-LABEL: udot8_acc8_vecMul:
2179 ; GFX9-DL: ; %bb.0: ; %entry
2180 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2181 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2182 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2183 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
2184 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
2185 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
2186 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
2187 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off
2188 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2189 ; GFX9-DL-NEXT: s_and_b32 s0, s2, 15
2190 ; GFX9-DL-NEXT: s_and_b32 s1, s4, 15
2191 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004
2192 ; GFX9-DL-NEXT: s_bfe_u32 s6, s4, 0x40008
2193 ; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x4000c
2194 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
2195 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s5
2196 ; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x40004
2197 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s6
2198 ; GFX9-DL-NEXT: s_bfe_u32 s9, s2, 0x40008
2199 ; GFX9-DL-NEXT: v_mov_b32_e32 v6, s7
2200 ; GFX9-DL-NEXT: s_bfe_u32 s10, s2, 0x4000c
2201 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v3, s0, v3
2202 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v4, s8, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2203 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v5, s9, v5
2204 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, s10, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2205 ; GFX9-DL-NEXT: v_or_b32_e32 v3, v3, v4
2206 ; GFX9-DL-NEXT: v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2207 ; GFX9-DL-NEXT: s_bfe_u32 s1, s4, 0x40014
2208 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40018
2209 ; GFX9-DL-NEXT: s_bfe_u32 s0, s4, 0x40010
2210 ; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28
2211 ; GFX9-DL-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
2212 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x40010
2213 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s0
2214 ; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x40014
2215 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s1
2216 ; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x40018
2217 ; GFX9-DL-NEXT: v_mov_b32_e32 v6, s5
2218 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28
2219 ; GFX9-DL-NEXT: v_mov_b32_e32 v7, s4
2220 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v4, s6, v4
2221 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v5, s7, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2222 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v6, s8, v6
2223 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v7, s2, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2224 ; GFX9-DL-NEXT: v_or_b32_e32 v4, v4, v5
2225 ; GFX9-DL-NEXT: v_or_b32_sdwa v5, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2226 ; GFX9-DL-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
2227 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
2228 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
2229 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v3, v2
2230 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v5
2231 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2232 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2233 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v4
2234 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v4
2235 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v3
2236 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2237 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2238 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off
2239 ; GFX9-DL-NEXT: s_endpgm
2240 <8 x i4> addrspace(1)* %src2,
2241 i8 addrspace(1)* nocapture %dst) {
2243 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
2244 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
2246 %cvec1 = zext <8 x i4> %vec1 to <8 x i8>
2247 %cvec2 = zext <8 x i4> %vec2 to <8 x i8>
2249 %mul = mul <8 x i8> %cvec1, %cvec2
2250 %mul0 = extractelement <8 x i8> %mul, i64 0
2251 %mul1 = extractelement <8 x i8> %mul, i64 1
2252 %mul2 = extractelement <8 x i8> %mul, i64 2
2253 %mul3 = extractelement <8 x i8> %mul, i64 3
2254 %mul4 = extractelement <8 x i8> %mul, i64 4
2255 %mul5 = extractelement <8 x i8> %mul, i64 5
2256 %mul6 = extractelement <8 x i8> %mul, i64 6
2257 %mul7 = extractelement <8 x i8> %mul, i64 7
2259 %acc = load i8, i8 addrspace(1)* %dst, align 4
2260 %add1 = add i8 %mul0, %acc
2261 %add2 = add i8 %add1, %mul1
2262 %add3 = add i8 %add2, %mul2
2263 %add4 = add i8 %add3, %mul3
2264 %add5 = add i8 %add4, %mul4
2265 %add6 = add i8 %add5, %mul5
2266 %add7 = add i8 %add6, %mul6
2267 %add8 = add i8 %add7, %mul7
2269 store i8 %add8, i8 addrspace(1)* %dst, align 4
2273 ; TODO: Once the adictional "and+add" are removed, the pattern will be recognized.
2274 define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
2275 ; GFX7-LABEL: udot8_acc4_vecMul:
2276 ; GFX7: ; %bb.0: ; %entry
2277 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
2278 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd
2279 ; GFX7-NEXT: s_mov_b32 s11, 0xf000
2280 ; GFX7-NEXT: s_mov_b32 s10, -1
2281 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2282 ; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
2283 ; GFX7-NEXT: buffer_load_ubyte v0, off, s[8:11], 0
2284 ; GFX7-NEXT: s_load_dword s1, s[6:7], 0x0
2285 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2286 ; GFX7-NEXT: s_lshr_b32 s2, s0, 28
2287 ; GFX7-NEXT: s_bfe_u32 s4, s0, 0x40018
2288 ; GFX7-NEXT: s_bfe_u32 s15, s1, 0x40018
2289 ; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40014
2290 ; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40010
2291 ; GFX7-NEXT: s_bfe_u32 s18, s1, 0x4000c
2292 ; GFX7-NEXT: s_bfe_u32 s19, s1, 0x40008
2293 ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40004
2294 ; GFX7-NEXT: s_lshr_b32 s14, s1, 28
2295 ; GFX7-NEXT: s_and_b32 s1, s1, 15
2296 ; GFX7-NEXT: s_bfe_u32 s5, s0, 0x40014
2297 ; GFX7-NEXT: s_bfe_u32 s6, s0, 0x40010
2298 ; GFX7-NEXT: s_bfe_u32 s7, s0, 0x4000c
2299 ; GFX7-NEXT: s_bfe_u32 s12, s0, 0x40008
2300 ; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40004
2301 ; GFX7-NEXT: s_and_b32 s0, s0, 15
2302 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2303 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
2304 ; GFX7-NEXT: v_mov_b32_e32 v3, s19
2305 ; GFX7-NEXT: v_mov_b32_e32 v4, s18
2306 ; GFX7-NEXT: v_mov_b32_e32 v5, s17
2307 ; GFX7-NEXT: v_mov_b32_e32 v6, s16
2308 ; GFX7-NEXT: v_mov_b32_e32 v7, s15
2309 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2310 ; GFX7-NEXT: v_mad_u32_u24 v0, s0, v1, v0
2311 ; GFX7-NEXT: v_mad_u32_u24 v0, s13, v2, v0
2312 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v3, v0
2313 ; GFX7-NEXT: v_mad_u32_u24 v0, s7, v4, v0
2314 ; GFX7-NEXT: v_mad_u32_u24 v0, s6, v5, v0
2315 ; GFX7-NEXT: v_mad_u32_u24 v0, s5, v6, v0
2316 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v7, v0
2317 ; GFX7-NEXT: v_mov_b32_e32 v1, s14
2318 ; GFX7-NEXT: v_mad_u32_u24 v0, s2, v1, v0
2319 ; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
2320 ; GFX7-NEXT: buffer_store_byte v0, off, s[8:11], 0
2321 ; GFX7-NEXT: s_endpgm
2323 ; GFX8-LABEL: udot8_acc4_vecMul:
2324 ; GFX8: ; %bb.0: ; %entry
2325 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2326 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2327 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2328 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
2329 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
2330 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
2331 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
2332 ; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
2333 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2334 ; GFX8-NEXT: s_and_b32 s0, s2, 15
2335 ; GFX8-NEXT: s_and_b32 s1, s4, 15
2336 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
2337 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004
2338 ; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40008
2339 ; GFX8-NEXT: v_mov_b32_e32 v4, s6
2340 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40008
2341 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
2342 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004
2343 ; GFX8-NEXT: v_mul_u32_u24_e32 v4, s7, v4
2344 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x4000c
2345 ; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
2346 ; GFX8-NEXT: s_bfe_u32 s7, s4, 0x40010
2347 ; GFX8-NEXT: v_mov_b32_e32 v6, s5
2348 ; GFX8-NEXT: s_bfe_u32 s6, s2, 0x4000c
2349 ; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40014
2350 ; GFX8-NEXT: v_mov_b32_e32 v7, s7
2351 ; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40010
2352 ; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40018
2353 ; GFX8-NEXT: v_mov_b32_e32 v8, s8
2354 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40014
2355 ; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40018
2356 ; GFX8-NEXT: s_lshr_b32 s4, s4, 28
2357 ; GFX8-NEXT: v_mov_b32_e32 v9, s9
2358 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28
2359 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2360 ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
2361 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v5, v2
2362 ; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
2363 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v4, v2
2364 ; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2
2365 ; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2
2366 ; GFX8-NEXT: v_mad_u32_u24 v2, s7, v8, v2
2367 ; GFX8-NEXT: v_mad_u32_u24 v2, s8, v9, v2
2368 ; GFX8-NEXT: v_mov_b32_e32 v3, s4
2369 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
2370 ; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
2371 ; GFX8-NEXT: flat_store_byte v[0:1], v2
2372 ; GFX8-NEXT: s_endpgm
2374 ; GFX9-LABEL: udot8_acc4_vecMul:
2375 ; GFX9: ; %bb.0: ; %entry
2376 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2377 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2378 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
2379 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
2380 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
2381 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
2382 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
2383 ; GFX9-NEXT: global_load_ubyte v2, v[0:1], off
2384 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
2385 ; GFX9-NEXT: s_and_b32 s0, s2, 15
2386 ; GFX9-NEXT: s_and_b32 s1, s4, 15
2387 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
2388 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004
2389 ; GFX9-NEXT: s_bfe_u32 s6, s4, 0x40008
2390 ; GFX9-NEXT: v_mov_b32_e32 v4, s6
2391 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40008
2392 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
2393 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004
2394 ; GFX9-NEXT: v_mul_u32_u24_e32 v4, s7, v4
2395 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x4000c
2396 ; GFX9-NEXT: v_and_b32_e32 v4, 15, v4
2397 ; GFX9-NEXT: s_bfe_u32 s7, s4, 0x40010
2398 ; GFX9-NEXT: v_mov_b32_e32 v6, s5
2399 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x4000c
2400 ; GFX9-NEXT: s_bfe_u32 s8, s4, 0x40014
2401 ; GFX9-NEXT: v_mov_b32_e32 v7, s7
2402 ; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40010
2403 ; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40018
2404 ; GFX9-NEXT: v_mov_b32_e32 v8, s8
2405 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40014
2406 ; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40018
2407 ; GFX9-NEXT: s_lshr_b32 s4, s4, 28
2408 ; GFX9-NEXT: v_mov_b32_e32 v9, s9
2409 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28
2410 ; GFX9-NEXT: s_waitcnt vmcnt(0)
2411 ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2
2412 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v5, v2
2413 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
2414 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v4
2415 ; GFX9-NEXT: v_mad_u32_u24 v2, s6, v6, v2
2416 ; GFX9-NEXT: v_mad_u32_u24 v2, s5, v7, v2
2417 ; GFX9-NEXT: v_mad_u32_u24 v2, s7, v8, v2
2418 ; GFX9-NEXT: v_mad_u32_u24 v2, s8, v9, v2
2419 ; GFX9-NEXT: v_mov_b32_e32 v3, s4
2420 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v3, v2
2421 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
2422 ; GFX9-NEXT: global_store_byte v[0:1], v2, off
2423 ; GFX9-NEXT: s_endpgm
2425 ; GFX9-DL-LABEL: udot8_acc4_vecMul:
2426 ; GFX9-DL: ; %bb.0: ; %entry
2427 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2428 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2429 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2430 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
2431 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
2432 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
2433 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
2434 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off
2435 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2436 ; GFX9-DL-NEXT: s_and_b32 s0, s2, 15
2437 ; GFX9-DL-NEXT: s_and_b32 s1, s4, 15
2438 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
2439 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004
2440 ; GFX9-DL-NEXT: s_bfe_u32 s6, s4, 0x40008
2441 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s6
2442 ; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x40008
2443 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s5
2444 ; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004
2445 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, s7, v4
2446 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x4000c
2447 ; GFX9-DL-NEXT: v_and_b32_e32 v4, 15, v4
2448 ; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x40010
2449 ; GFX9-DL-NEXT: v_mov_b32_e32 v6, s5
2450 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x4000c
2451 ; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x40014
2452 ; GFX9-DL-NEXT: v_mov_b32_e32 v7, s7
2453 ; GFX9-DL-NEXT: s_bfe_u32 s5, s2, 0x40010
2454 ; GFX9-DL-NEXT: s_bfe_u32 s9, s4, 0x40018
2455 ; GFX9-DL-NEXT: v_mov_b32_e32 v8, s8
2456 ; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x40014
2457 ; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x40018
2458 ; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28
2459 ; GFX9-DL-NEXT: v_mov_b32_e32 v9, s9
2460 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28
2461 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
2462 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2
2463 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v5, v2
2464 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
2465 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v4
2466 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v6, v2
2467 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v7, v2
2468 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s7, v8, v2
2469 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s8, v9, v2
2470 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4
2471 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2
2472 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
2473 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off
2474 ; GFX9-DL-NEXT: s_endpgm
2475 <8 x i4> addrspace(1)* %src2,
2476 i4 addrspace(1)* nocapture %dst) {
2478 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
2479 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
2481 %mul = mul <8 x i4> %vec1, %vec2
2482 %mul0 = extractelement <8 x i4> %mul, i64 0
2483 %mul1 = extractelement <8 x i4> %mul, i64 1
2484 %mul2 = extractelement <8 x i4> %mul, i64 2
2485 %mul3 = extractelement <8 x i4> %mul, i64 3
2486 %mul4 = extractelement <8 x i4> %mul, i64 4
2487 %mul5 = extractelement <8 x i4> %mul, i64 5
2488 %mul6 = extractelement <8 x i4> %mul, i64 6
2489 %mul7 = extractelement <8 x i4> %mul, i64 7
2491 %acc = load i4, i4 addrspace(1)* %dst, align 4
2492 %add1 = add i4 %mul0, %acc
2493 %add2 = add i4 %add1, %mul1
2494 %add3 = add i4 %add2, %mul2
2495 %add4 = add i4 %add3, %mul3
2496 %add5 = add i4 %add4, %mul4
2497 %add6 = add i4 %add5, %mul5
2498 %add7 = add i4 %add6, %mul6
2499 %add8 = add i4 %add7, %mul7
2501 store i4 %add8, i4 addrspace(1)* %dst, align 4
2505 define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
2506 ; GFX7-LABEL: idot8_acc32:
2507 ; GFX7: ; %bb.0: ; %entry
2508 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
2509 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
2510 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2511 ; GFX7-NEXT: s_mov_b32 s6, -1
2512 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2513 ; GFX7-NEXT: s_load_dword s0, s[8:9], 0x0
2514 ; GFX7-NEXT: s_load_dword s1, s[10:11], 0x0
2515 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
2516 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2517 ; GFX7-NEXT: s_bfe_i32 s8, s0, 0x40000
2518 ; GFX7-NEXT: s_bfe_i32 s9, s1, 0x40000
2519 ; GFX7-NEXT: s_bfe_i32 s11, s1, 0x40004
2520 ; GFX7-NEXT: v_mov_b32_e32 v0, s9
2521 ; GFX7-NEXT: v_mov_b32_e32 v1, s2
2522 ; GFX7-NEXT: v_mad_i32_i24 v0, s8, v0, v1
2523 ; GFX7-NEXT: s_bfe_i32 s10, s0, 0x40004
2524 ; GFX7-NEXT: v_mov_b32_e32 v1, s11
2525 ; GFX7-NEXT: s_bfe_i32 s13, s1, 0x40008
2526 ; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
2527 ; GFX7-NEXT: s_bfe_i32 s12, s0, 0x40008
2528 ; GFX7-NEXT: v_mov_b32_e32 v1, s13
2529 ; GFX7-NEXT: s_bfe_i32 s15, s1, 0x4000c
2530 ; GFX7-NEXT: v_mad_i32_i24 v0, s12, v1, v0
2531 ; GFX7-NEXT: s_bfe_i32 s14, s0, 0x4000c
2532 ; GFX7-NEXT: v_mov_b32_e32 v1, s15
2533 ; GFX7-NEXT: s_bfe_i32 s17, s1, 0x40010
2534 ; GFX7-NEXT: v_mad_i32_i24 v0, s14, v1, v0
2535 ; GFX7-NEXT: s_bfe_i32 s16, s0, 0x40010
2536 ; GFX7-NEXT: v_mov_b32_e32 v1, s17
2537 ; GFX7-NEXT: s_bfe_i32 s19, s1, 0x40014
2538 ; GFX7-NEXT: s_bfe_i32 s21, s1, 0x40018
2539 ; GFX7-NEXT: v_mad_i32_i24 v0, s16, v1, v0
2540 ; GFX7-NEXT: s_bfe_i32 s18, s0, 0x40014
2541 ; GFX7-NEXT: v_mov_b32_e32 v1, s19
2542 ; GFX7-NEXT: s_bfe_i32 s20, s0, 0x40018
2543 ; GFX7-NEXT: v_mad_i32_i24 v0, s18, v1, v0
2544 ; GFX7-NEXT: v_mov_b32_e32 v1, s21
2545 ; GFX7-NEXT: s_ashr_i32 s1, s1, 28
2546 ; GFX7-NEXT: v_mad_i32_i24 v0, s20, v1, v0
2547 ; GFX7-NEXT: s_ashr_i32 s0, s0, 28
2548 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2549 ; GFX7-NEXT: v_mad_i32_i24 v0, s0, v1, v0
2550 ; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
2551 ; GFX7-NEXT: s_endpgm
2553 ; GFX8-LABEL: idot8_acc32:
2554 ; GFX8: ; %bb.0: ; %entry
2555 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2556 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2557 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2558 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
2559 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
2560 ; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
2561 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
2562 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
2563 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2564 ; GFX8-NEXT: s_bfe_i32 s0, s2, 0x40000
2565 ; GFX8-NEXT: s_bfe_i32 s1, s4, 0x40000
2566 ; GFX8-NEXT: s_bfe_i32 s7, s4, 0x40004
2567 ; GFX8-NEXT: v_mov_b32_e32 v2, s1
2568 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
2569 ; GFX8-NEXT: v_mad_i32_i24 v2, s0, v2, v3
2570 ; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40004
2571 ; GFX8-NEXT: v_mov_b32_e32 v3, s7
2572 ; GFX8-NEXT: s_bfe_i32 s9, s4, 0x40008
2573 ; GFX8-NEXT: v_mad_i32_i24 v2, s6, v3, v2
2574 ; GFX8-NEXT: s_bfe_i32 s8, s2, 0x40008
2575 ; GFX8-NEXT: v_mov_b32_e32 v3, s9
2576 ; GFX8-NEXT: s_bfe_i32 s11, s4, 0x4000c
2577 ; GFX8-NEXT: v_mad_i32_i24 v2, s8, v3, v2
2578 ; GFX8-NEXT: s_bfe_i32 s10, s2, 0x4000c
2579 ; GFX8-NEXT: v_mov_b32_e32 v3, s11
2580 ; GFX8-NEXT: s_bfe_i32 s13, s4, 0x40010
2581 ; GFX8-NEXT: v_mad_i32_i24 v2, s10, v3, v2
2582 ; GFX8-NEXT: s_bfe_i32 s12, s2, 0x40010
2583 ; GFX8-NEXT: v_mov_b32_e32 v3, s13
2584 ; GFX8-NEXT: s_bfe_i32 s15, s4, 0x40014
2585 ; GFX8-NEXT: s_bfe_i32 s17, s4, 0x40018
2586 ; GFX8-NEXT: v_mad_i32_i24 v2, s12, v3, v2
2587 ; GFX8-NEXT: s_bfe_i32 s14, s2, 0x40014
2588 ; GFX8-NEXT: v_mov_b32_e32 v3, s15
2589 ; GFX8-NEXT: s_bfe_i32 s16, s2, 0x40018
2590 ; GFX8-NEXT: v_mad_i32_i24 v2, s14, v3, v2
2591 ; GFX8-NEXT: v_mov_b32_e32 v3, s17
2592 ; GFX8-NEXT: s_ashr_i32 s4, s4, 28
2593 ; GFX8-NEXT: v_mad_i32_i24 v2, s16, v3, v2
2594 ; GFX8-NEXT: s_ashr_i32 s2, s2, 28
2595 ; GFX8-NEXT: v_mov_b32_e32 v3, s4
2596 ; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2
2597 ; GFX8-NEXT: flat_store_dword v[0:1], v2
2598 ; GFX8-NEXT: s_endpgm
2600 ; GFX9-LABEL: idot8_acc32:
2601 ; GFX9: ; %bb.0: ; %entry
2602 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2603 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2604 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
2605 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
2606 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
2607 ; GFX9-NEXT: s_load_dword s5, s[0:1], 0x0
2608 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
2609 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
2610 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
2611 ; GFX9-NEXT: s_bfe_i32 s0, s2, 0x40000
2612 ; GFX9-NEXT: s_bfe_i32 s1, s4, 0x40000
2613 ; GFX9-NEXT: s_bfe_i32 s7, s4, 0x40004
2614 ; GFX9-NEXT: v_mov_b32_e32 v2, s1
2615 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
2616 ; GFX9-NEXT: v_mad_i32_i24 v2, s0, v2, v3
2617 ; GFX9-NEXT: s_bfe_i32 s6, s2, 0x40004
2618 ; GFX9-NEXT: v_mov_b32_e32 v3, s7
2619 ; GFX9-NEXT: s_bfe_i32 s9, s4, 0x40008
2620 ; GFX9-NEXT: v_mad_i32_i24 v2, s6, v3, v2
2621 ; GFX9-NEXT: s_bfe_i32 s8, s2, 0x40008
2622 ; GFX9-NEXT: v_mov_b32_e32 v3, s9
2623 ; GFX9-NEXT: s_bfe_i32 s11, s4, 0x4000c
2624 ; GFX9-NEXT: v_mad_i32_i24 v2, s8, v3, v2
2625 ; GFX9-NEXT: s_bfe_i32 s10, s2, 0x4000c
2626 ; GFX9-NEXT: v_mov_b32_e32 v3, s11
2627 ; GFX9-NEXT: s_bfe_i32 s13, s4, 0x40010
2628 ; GFX9-NEXT: v_mad_i32_i24 v2, s10, v3, v2
2629 ; GFX9-NEXT: s_bfe_i32 s12, s2, 0x40010
2630 ; GFX9-NEXT: v_mov_b32_e32 v3, s13
2631 ; GFX9-NEXT: s_bfe_i32 s15, s4, 0x40014
2632 ; GFX9-NEXT: s_bfe_i32 s17, s4, 0x40018
2633 ; GFX9-NEXT: v_mad_i32_i24 v2, s12, v3, v2
2634 ; GFX9-NEXT: s_bfe_i32 s14, s2, 0x40014
2635 ; GFX9-NEXT: v_mov_b32_e32 v3, s15
2636 ; GFX9-NEXT: s_bfe_i32 s16, s2, 0x40018
2637 ; GFX9-NEXT: v_mad_i32_i24 v2, s14, v3, v2
2638 ; GFX9-NEXT: v_mov_b32_e32 v3, s17
2639 ; GFX9-NEXT: s_ashr_i32 s4, s4, 28
2640 ; GFX9-NEXT: v_mad_i32_i24 v2, s16, v3, v2
2641 ; GFX9-NEXT: s_ashr_i32 s2, s2, 28
2642 ; GFX9-NEXT: v_mov_b32_e32 v3, s4
2643 ; GFX9-NEXT: v_mad_i32_i24 v2, s2, v3, v2
2644 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
2645 ; GFX9-NEXT: s_endpgm
2647 ; GFX9-DL-LABEL: idot8_acc32:
2648 ; GFX9-DL: ; %bb.0: ; %entry
2649 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2650 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2651 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2652 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
2653 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
2654 ; GFX9-DL-NEXT: s_load_dword s5, s[0:1], 0x0
2655 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
2656 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
2657 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2658 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
2659 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5
2660 ; GFX9-DL-NEXT: v_dot8_i32_i4 v2, s2, v2, v3
2661 ; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off
2662 ; GFX9-DL-NEXT: s_endpgm
2663 <8 x i4> addrspace(1)* %src2,
2664 i32 addrspace(1)* nocapture %dst) {
2666 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
2667 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
2669 %v1e0 = extractelement <8 x i4> %vec1, i64 0
2670 %cv1e0 = sext i4 %v1e0 to i32
2671 %v2e0 = extractelement <8 x i4> %vec2, i64 0
2672 %cv2e0 = sext i4 %v2e0 to i32
2673 %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
2675 %v1e1 = extractelement <8 x i4> %vec1, i64 1
2676 %cv1e1 = sext i4 %v1e1 to i32
2677 %v2e1 = extractelement <8 x i4> %vec2, i64 1
2678 %cv2e1 = sext i4 %v2e1 to i32
2679 %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
2681 %v1e2 = extractelement <8 x i4> %vec1, i64 2
2682 %cv1e2 = sext i4 %v1e2 to i32
2683 %v2e2 = extractelement <8 x i4> %vec2, i64 2
2684 %cv2e2 = sext i4 %v2e2 to i32
2685 %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
2687 %v1e3 = extractelement <8 x i4> %vec1, i64 3
2688 %cv1e3 = sext i4 %v1e3 to i32
2689 %v2e3 = extractelement <8 x i4> %vec2, i64 3
2690 %cv2e3 = sext i4 %v2e3 to i32
2691 %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
2693 %v1e4 = extractelement <8 x i4> %vec1, i64 4
2694 %cv1e4 = sext i4 %v1e4 to i32
2695 %v2e4 = extractelement <8 x i4> %vec2, i64 4
2696 %cv2e4 = sext i4 %v2e4 to i32
2697 %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
2699 %v1e5 = extractelement <8 x i4> %vec1, i64 5
2700 %cv1e5 = sext i4 %v1e5 to i32
2701 %v2e5 = extractelement <8 x i4> %vec2, i64 5
2702 %cv2e5 = sext i4 %v2e5 to i32
2703 %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
2705 %v1e6 = extractelement <8 x i4> %vec1, i64 6
2706 %cv1e6 = sext i4 %v1e6 to i32
2707 %v2e6 = extractelement <8 x i4> %vec2, i64 6
2708 %cv2e6 = sext i4 %v2e6 to i32
2709 %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
2711 %v1e7 = extractelement <8 x i4> %vec1, i64 7
2712 %cv1e7 = sext i4 %v1e7 to i32
2713 %v2e7 = extractelement <8 x i4> %vec2, i64 7
2714 %cv2e7 = sext i4 %v2e7 to i32
2715 %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
2717 %acc = load i32, i32 addrspace(1)* %dst, align 4
2718 %add1 = add i32 %mul0, %acc
2719 %add2 = add i32 %add1, %mul1
2720 %add3 = add i32 %add2, %mul2
2721 %add4 = add i32 %add3, %mul3
2722 %add5 = add i32 %add4, %mul4
2723 %add6 = add i32 %add5, %mul5
2724 %add7 = add i32 %add6, %mul6
2725 %add8 = add i32 %add7, %mul7
2727 store i32 %add8, i32 addrspace(1)* %dst, align 4
2731 ; TODO: Once the unnecessary zero extentions of the elements are removed;
2732 ; pattern recognizer will kick in.
2733 define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
2734 ; GFX7-LABEL: idot8_acc16:
2735 ; GFX7: ; %bb.0: ; %entry
2736 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
2737 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
2738 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2739 ; GFX7-NEXT: s_mov_b32 s6, -1
2740 ; GFX7-NEXT: s_mov_b32 s0, 0xffff
2741 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2742 ; GFX7-NEXT: s_load_dword s1, s[8:9], 0x0
2743 ; GFX7-NEXT: buffer_load_ushort v0, off, s[4:7], 0
2744 ; GFX7-NEXT: s_load_dword s2, s[10:11], 0x0
2745 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2746 ; GFX7-NEXT: s_bfe_i32 s8, s1, 0x40000
2747 ; GFX7-NEXT: s_bfe_i32 s10, s1, 0x40004
2748 ; GFX7-NEXT: s_bfe_i32 s9, s2, 0x40000
2749 ; GFX7-NEXT: s_bfe_i32 s11, s2, 0x40004
2750 ; GFX7-NEXT: s_and_b32 s9, s9, s0
2751 ; GFX7-NEXT: s_bfe_i32 s13, s2, 0x40008
2752 ; GFX7-NEXT: s_and_b32 s11, s11, s0
2753 ; GFX7-NEXT: s_and_b32 s8, s8, s0
2754 ; GFX7-NEXT: v_mov_b32_e32 v1, s9
2755 ; GFX7-NEXT: s_bfe_i32 s12, s1, 0x40008
2756 ; GFX7-NEXT: s_bfe_i32 s15, s2, 0x4000c
2757 ; GFX7-NEXT: s_and_b32 s13, s13, s0
2758 ; GFX7-NEXT: s_and_b32 s10, s10, s0
2759 ; GFX7-NEXT: v_mov_b32_e32 v2, s11
2760 ; GFX7-NEXT: s_bfe_i32 s14, s1, 0x4000c
2761 ; GFX7-NEXT: s_bfe_i32 s17, s2, 0x40010
2762 ; GFX7-NEXT: s_and_b32 s15, s15, s0
2763 ; GFX7-NEXT: s_and_b32 s12, s12, s0
2764 ; GFX7-NEXT: v_mov_b32_e32 v3, s13
2765 ; GFX7-NEXT: s_bfe_i32 s16, s1, 0x40010
2766 ; GFX7-NEXT: s_bfe_i32 s19, s2, 0x40014
2767 ; GFX7-NEXT: s_and_b32 s17, s17, s0
2768 ; GFX7-NEXT: s_and_b32 s14, s14, s0
2769 ; GFX7-NEXT: v_mov_b32_e32 v4, s15
2770 ; GFX7-NEXT: s_bfe_i32 s21, s2, 0x40018
2771 ; GFX7-NEXT: s_bfe_i32 s18, s1, 0x40014
2772 ; GFX7-NEXT: s_and_b32 s19, s19, s0
2773 ; GFX7-NEXT: s_and_b32 s16, s16, s0
2774 ; GFX7-NEXT: v_mov_b32_e32 v5, s17
2775 ; GFX7-NEXT: s_bfe_i32 s20, s1, 0x40018
2776 ; GFX7-NEXT: s_ashr_i32 s2, s2, 28
2777 ; GFX7-NEXT: s_and_b32 s21, s21, s0
2778 ; GFX7-NEXT: s_and_b32 s18, s18, s0
2779 ; GFX7-NEXT: v_mov_b32_e32 v6, s19
2780 ; GFX7-NEXT: s_ashr_i32 s1, s1, 28
2781 ; GFX7-NEXT: s_and_b32 s20, s20, s0
2782 ; GFX7-NEXT: s_and_b32 s2, s2, s0
2783 ; GFX7-NEXT: v_mov_b32_e32 v7, s21
2784 ; GFX7-NEXT: s_and_b32 s0, s1, s0
2785 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2786 ; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0
2787 ; GFX7-NEXT: v_mad_u32_u24 v0, s10, v2, v0
2788 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v3, v0
2789 ; GFX7-NEXT: v_mad_u32_u24 v0, s14, v4, v0
2790 ; GFX7-NEXT: v_mad_u32_u24 v0, s16, v5, v0
2791 ; GFX7-NEXT: v_mad_u32_u24 v0, s18, v6, v0
2792 ; GFX7-NEXT: v_mad_u32_u24 v0, s20, v7, v0
2793 ; GFX7-NEXT: v_mov_b32_e32 v1, s2
2794 ; GFX7-NEXT: v_mad_u32_u24 v0, s0, v1, v0
2795 ; GFX7-NEXT: buffer_store_short v0, off, s[4:7], 0
2796 ; GFX7-NEXT: s_endpgm
2798 ; GFX8-LABEL: idot8_acc16:
2799 ; GFX8: ; %bb.0: ; %entry
2800 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2801 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2802 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2803 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
2804 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
2805 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
2806 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
2807 ; GFX8-NEXT: flat_load_ushort v2, v[0:1]
2808 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2809 ; GFX8-NEXT: s_lshr_b32 s0, s2, 4
2810 ; GFX8-NEXT: s_lshr_b32 s1, s4, 4
2811 ; GFX8-NEXT: s_bfe_i32 s5, s4, 0x40000
2812 ; GFX8-NEXT: v_lshlrev_b16_e64 v3, 12, s0
2813 ; GFX8-NEXT: v_lshlrev_b16_e64 v4, 12, s1
2814 ; GFX8-NEXT: s_bfe_i32 s0, s4, 0x40008
2815 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
2816 ; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40000
2817 ; GFX8-NEXT: s_lshr_b32 s1, s2, 12
2818 ; GFX8-NEXT: s_lshr_b32 s5, s4, 12
2819 ; GFX8-NEXT: v_mov_b32_e32 v6, s0
2820 ; GFX8-NEXT: s_bfe_i32 s7, s2, 0x40008
2821 ; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
2822 ; GFX8-NEXT: v_ashrrev_i16_e32 v4, 12, v4
2823 ; GFX8-NEXT: v_lshlrev_b16_e64 v7, 12, s1
2824 ; GFX8-NEXT: v_lshlrev_b16_e64 v8, 12, s5
2825 ; GFX8-NEXT: v_mul_i32_i24_e32 v6, s7, v6
2826 ; GFX8-NEXT: s_lshr_b32 s0, s2, 20
2827 ; GFX8-NEXT: s_lshr_b32 s1, s4, 20
2828 ; GFX8-NEXT: s_bfe_i32 s5, s4, 0x40010
2829 ; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
2830 ; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
2831 ; GFX8-NEXT: v_lshlrev_b16_e64 v9, 12, s0
2832 ; GFX8-NEXT: v_lshlrev_b16_e64 v10, 12, s1
2833 ; GFX8-NEXT: s_bfe_i32 s8, s2, 0x40010
2834 ; GFX8-NEXT: v_mov_b32_e32 v13, s5
2835 ; GFX8-NEXT: s_lshr_b32 s0, s2, 28
2836 ; GFX8-NEXT: s_lshr_b32 s9, s4, 28
2837 ; GFX8-NEXT: s_bfe_i32 s4, s4, 0x40018
2838 ; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
2839 ; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
2840 ; GFX8-NEXT: v_lshlrev_b16_e64 v11, 12, s0
2841 ; GFX8-NEXT: v_lshlrev_b16_e64 v12, 12, s9
2842 ; GFX8-NEXT: s_bfe_i32 s2, s2, 0x40018
2843 ; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v11
2844 ; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
2845 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2846 ; GFX8-NEXT: v_mad_i32_i24 v2, s6, v5, v2
2847 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v4, v2
2848 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
2849 ; GFX8-NEXT: v_mad_u32_u24 v2, v7, v8, v2
2850 ; GFX8-NEXT: v_mad_i32_i24 v2, s8, v13, v2
2851 ; GFX8-NEXT: v_mad_u32_u24 v2, v9, v10, v2
2852 ; GFX8-NEXT: v_mov_b32_e32 v3, s4
2853 ; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2
2854 ; GFX8-NEXT: v_mad_u32_u24 v2, v11, v12, v2
2855 ; GFX8-NEXT: flat_store_short v[0:1], v2
2856 ; GFX8-NEXT: s_endpgm
2858 ; GFX9-LABEL: idot8_acc16:
2859 ; GFX9: ; %bb.0: ; %entry
2860 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2861 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2862 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
2863 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
2864 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
2865 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
2866 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
2867 ; GFX9-NEXT: global_load_ushort v2, v[0:1], off
2868 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
2869 ; GFX9-NEXT: s_lshr_b32 s0, s2, 4
2870 ; GFX9-NEXT: s_lshr_b32 s1, s4, 4
2871 ; GFX9-NEXT: s_bfe_i32 s5, s4, 0x40000
2872 ; GFX9-NEXT: v_lshlrev_b16_e64 v3, 12, s0
2873 ; GFX9-NEXT: v_lshlrev_b16_e64 v4, 12, s1
2874 ; GFX9-NEXT: s_bfe_i32 s0, s4, 0x40008
2875 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
2876 ; GFX9-NEXT: s_bfe_i32 s6, s2, 0x40000
2877 ; GFX9-NEXT: s_lshr_b32 s1, s2, 12
2878 ; GFX9-NEXT: s_lshr_b32 s5, s4, 12
2879 ; GFX9-NEXT: v_mov_b32_e32 v6, s0
2880 ; GFX9-NEXT: s_bfe_i32 s7, s2, 0x40008
2881 ; GFX9-NEXT: v_ashrrev_i16_e32 v3, 12, v3
2882 ; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4
2883 ; GFX9-NEXT: v_lshlrev_b16_e64 v7, 12, s1
2884 ; GFX9-NEXT: v_lshlrev_b16_e64 v8, 12, s5
2885 ; GFX9-NEXT: v_mul_i32_i24_e32 v6, s7, v6
2886 ; GFX9-NEXT: s_lshr_b32 s0, s2, 20
2887 ; GFX9-NEXT: s_lshr_b32 s1, s4, 20
2888 ; GFX9-NEXT: s_bfe_i32 s5, s4, 0x40010
2889 ; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
2890 ; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
2891 ; GFX9-NEXT: v_lshlrev_b16_e64 v9, 12, s0
2892 ; GFX9-NEXT: v_lshlrev_b16_e64 v10, 12, s1
2893 ; GFX9-NEXT: s_bfe_i32 s8, s2, 0x40010
2894 ; GFX9-NEXT: v_mov_b32_e32 v13, s5
2895 ; GFX9-NEXT: s_lshr_b32 s0, s2, 28
2896 ; GFX9-NEXT: s_lshr_b32 s9, s4, 28
2897 ; GFX9-NEXT: s_bfe_i32 s4, s4, 0x40018
2898 ; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v9
2899 ; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v10
2900 ; GFX9-NEXT: v_lshlrev_b16_e64 v11, 12, s0
2901 ; GFX9-NEXT: v_lshlrev_b16_e64 v12, 12, s9
2902 ; GFX9-NEXT: s_bfe_i32 s2, s2, 0x40018
2903 ; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
2904 ; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
2905 ; GFX9-NEXT: s_waitcnt vmcnt(0)
2906 ; GFX9-NEXT: v_mad_i32_i24 v2, s6, v5, v2
2907 ; GFX9-NEXT: v_mad_u32_u24 v2, v3, v4, v2
2908 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
2909 ; GFX9-NEXT: v_mad_u32_u24 v2, v7, v8, v2
2910 ; GFX9-NEXT: v_mad_i32_i24 v2, s8, v13, v2
2911 ; GFX9-NEXT: v_mad_u32_u24 v2, v9, v10, v2
2912 ; GFX9-NEXT: v_mov_b32_e32 v3, s4
2913 ; GFX9-NEXT: v_mad_i32_i24 v2, s2, v3, v2
2914 ; GFX9-NEXT: v_mad_u32_u24 v2, v11, v12, v2
2915 ; GFX9-NEXT: global_store_short v[0:1], v2, off
2916 ; GFX9-NEXT: s_endpgm
2918 ; GFX9-DL-LABEL: idot8_acc16:
2919 ; GFX9-DL: ; %bb.0: ; %entry
2920 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2921 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2922 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2923 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
2924 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
2925 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
2926 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
2927 ; GFX9-DL-NEXT: global_load_ushort v2, v[0:1], off
2928 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2929 ; GFX9-DL-NEXT: s_lshr_b32 s0, s2, 4
2930 ; GFX9-DL-NEXT: s_lshr_b32 s1, s4, 4
2931 ; GFX9-DL-NEXT: s_bfe_i32 s5, s4, 0x40000
2932 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v3, 12, s0
2933 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v4, 12, s1
2934 ; GFX9-DL-NEXT: s_bfe_i32 s0, s4, 0x40008
2935 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s5
2936 ; GFX9-DL-NEXT: s_bfe_i32 s6, s2, 0x40000
2937 ; GFX9-DL-NEXT: s_lshr_b32 s1, s2, 12
2938 ; GFX9-DL-NEXT: s_lshr_b32 s5, s4, 12
2939 ; GFX9-DL-NEXT: v_mov_b32_e32 v6, s0
2940 ; GFX9-DL-NEXT: s_bfe_i32 s7, s2, 0x40008
2941 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v3, 12, v3
2942 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4
2943 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v7, 12, s1
2944 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v8, 12, s5
2945 ; GFX9-DL-NEXT: v_mul_i32_i24_e32 v6, s7, v6
2946 ; GFX9-DL-NEXT: s_lshr_b32 s0, s2, 20
2947 ; GFX9-DL-NEXT: s_lshr_b32 s1, s4, 20
2948 ; GFX9-DL-NEXT: s_bfe_i32 s5, s4, 0x40010
2949 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
2950 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
2951 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v9, 12, s0
2952 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v10, 12, s1
2953 ; GFX9-DL-NEXT: s_bfe_i32 s8, s2, 0x40010
2954 ; GFX9-DL-NEXT: v_mov_b32_e32 v13, s5
2955 ; GFX9-DL-NEXT: s_lshr_b32 s0, s2, 28
2956 ; GFX9-DL-NEXT: s_lshr_b32 s9, s4, 28
2957 ; GFX9-DL-NEXT: s_bfe_i32 s4, s4, 0x40018
2958 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v9
2959 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v10
2960 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v11, 12, s0
2961 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v12, 12, s9
2962 ; GFX9-DL-NEXT: s_bfe_i32 s2, s2, 0x40018
2963 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
2964 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
2965 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
2966 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s6, v5, v2
2967 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v3, v4, v2
2968 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
2969 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v7, v8, v2
2970 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s8, v13, v2
2971 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v9, v10, v2
2972 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4
2973 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s2, v3, v2
2974 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v11, v12, v2
2975 ; GFX9-DL-NEXT: global_store_short v[0:1], v2, off
2976 ; GFX9-DL-NEXT: s_endpgm
2977 <8 x i4> addrspace(1)* %src2,
2978 i16 addrspace(1)* nocapture %dst) {
2980 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
2981 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
2983 %v1e0 = extractelement <8 x i4> %vec1, i64 0
2984 %cv1e0 = sext i4 %v1e0 to i16
2985 %v2e0 = extractelement <8 x i4> %vec2, i64 0
2986 %cv2e0 = sext i4 %v2e0 to i16
2987 %mul0 = mul nuw nsw i16 %cv1e0, %cv2e0
2989 %v1e1 = extractelement <8 x i4> %vec1, i64 1
2990 %cv1e1 = sext i4 %v1e1 to i16
2991 %v2e1 = extractelement <8 x i4> %vec2, i64 1
2992 %cv2e1 = sext i4 %v2e1 to i16
2993 %mul1 = mul nuw nsw i16 %cv1e1, %cv2e1
2995 %v1e2 = extractelement <8 x i4> %vec1, i64 2
2996 %cv1e2 = sext i4 %v1e2 to i16
2997 %v2e2 = extractelement <8 x i4> %vec2, i64 2
2998 %cv2e2 = sext i4 %v2e2 to i16
2999 %mul2 = mul nuw nsw i16 %cv1e2, %cv2e2
3001 %v1e3 = extractelement <8 x i4> %vec1, i64 3
3002 %cv1e3 = sext i4 %v1e3 to i16
3003 %v2e3 = extractelement <8 x i4> %vec2, i64 3
3004 %cv2e3 = sext i4 %v2e3 to i16
3005 %mul3 = mul nuw nsw i16 %cv1e3, %cv2e3
3007 %v1e4 = extractelement <8 x i4> %vec1, i64 4
3008 %cv1e4 = sext i4 %v1e4 to i16
3009 %v2e4 = extractelement <8 x i4> %vec2, i64 4
3010 %cv2e4 = sext i4 %v2e4 to i16
3011 %mul4 = mul nuw nsw i16 %cv1e4, %cv2e4
3013 %v1e5 = extractelement <8 x i4> %vec1, i64 5
3014 %cv1e5 = sext i4 %v1e5 to i16
3015 %v2e5 = extractelement <8 x i4> %vec2, i64 5
3016 %cv2e5 = sext i4 %v2e5 to i16
3017 %mul5 = mul nuw nsw i16 %cv1e5, %cv2e5
3019 %v1e6 = extractelement <8 x i4> %vec1, i64 6
3020 %cv1e6 = sext i4 %v1e6 to i16
3021 %v2e6 = extractelement <8 x i4> %vec2, i64 6
3022 %cv2e6 = sext i4 %v2e6 to i16
3023 %mul6 = mul nuw nsw i16 %cv1e6, %cv2e6
3025 %v1e7 = extractelement <8 x i4> %vec1, i64 7
3026 %cv1e7 = sext i4 %v1e7 to i16
3027 %v2e7 = extractelement <8 x i4> %vec2, i64 7
3028 %cv2e7 = sext i4 %v2e7 to i16
3029 %mul7 = mul nuw nsw i16 %cv1e7, %cv2e7
3031 %acc = load i16, i16 addrspace(1)* %dst, align 4
3032 %add1 = add i16 %mul0, %acc
3033 %add2 = add i16 %add1, %mul1
3034 %add3 = add i16 %add2, %mul2
3035 %add4 = add i16 %add3, %mul3
3036 %add5 = add i16 %add4, %mul4
3037 %add6 = add i16 %add5, %mul5
3038 %add7 = add i16 %add6, %mul6
3039 %add8 = add i16 %add7, %mul7
3041 store i16 %add8, i16 addrspace(1)* %dst, align 4
3045 ; TODO: Support this pattern.
3046 define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
3047 ; GFX7-LABEL: idot8_acc8:
3048 ; GFX7: ; %bb.0: ; %entry
3049 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
3050 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
3051 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3052 ; GFX7-NEXT: s_mov_b32 s6, -1
3053 ; GFX7-NEXT: s_movk_i32 s0, 0xff
3054 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3055 ; GFX7-NEXT: s_load_dword s1, s[8:9], 0x0
3056 ; GFX7-NEXT: buffer_load_ubyte v0, off, s[4:7], 0
3057 ; GFX7-NEXT: s_load_dword s2, s[10:11], 0x0
3058 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3059 ; GFX7-NEXT: s_bfe_i32 s8, s1, 0x40000
3060 ; GFX7-NEXT: s_bfe_i32 s10, s1, 0x40004
3061 ; GFX7-NEXT: s_bfe_i32 s9, s2, 0x40000
3062 ; GFX7-NEXT: s_bfe_i32 s11, s2, 0x40004
3063 ; GFX7-NEXT: s_and_b32 s9, s9, s0
3064 ; GFX7-NEXT: s_bfe_i32 s13, s2, 0x40008
3065 ; GFX7-NEXT: s_and_b32 s11, s11, s0
3066 ; GFX7-NEXT: s_and_b32 s8, s8, s0
3067 ; GFX7-NEXT: v_mov_b32_e32 v1, s9
3068 ; GFX7-NEXT: s_bfe_i32 s12, s1, 0x40008
3069 ; GFX7-NEXT: s_bfe_i32 s15, s2, 0x4000c
3070 ; GFX7-NEXT: s_and_b32 s13, s13, s0
3071 ; GFX7-NEXT: s_and_b32 s10, s10, s0
3072 ; GFX7-NEXT: v_mov_b32_e32 v2, s11
3073 ; GFX7-NEXT: s_bfe_i32 s14, s1, 0x4000c
3074 ; GFX7-NEXT: s_bfe_i32 s17, s2, 0x40010
3075 ; GFX7-NEXT: s_and_b32 s15, s15, s0
3076 ; GFX7-NEXT: s_and_b32 s12, s12, s0
3077 ; GFX7-NEXT: v_mov_b32_e32 v3, s13
3078 ; GFX7-NEXT: s_bfe_i32 s16, s1, 0x40010
3079 ; GFX7-NEXT: s_bfe_i32 s19, s2, 0x40014
3080 ; GFX7-NEXT: s_and_b32 s17, s17, s0
3081 ; GFX7-NEXT: s_and_b32 s14, s14, s0
3082 ; GFX7-NEXT: v_mov_b32_e32 v4, s15
3083 ; GFX7-NEXT: s_bfe_i32 s21, s2, 0x40018
3084 ; GFX7-NEXT: s_bfe_i32 s18, s1, 0x40014
3085 ; GFX7-NEXT: s_and_b32 s19, s19, s0
3086 ; GFX7-NEXT: s_and_b32 s16, s16, s0
3087 ; GFX7-NEXT: v_mov_b32_e32 v5, s17
3088 ; GFX7-NEXT: s_bfe_i32 s20, s1, 0x40018
3089 ; GFX7-NEXT: s_ashr_i32 s2, s2, 28
3090 ; GFX7-NEXT: s_and_b32 s21, s21, s0
3091 ; GFX7-NEXT: s_and_b32 s18, s18, s0
3092 ; GFX7-NEXT: v_mov_b32_e32 v6, s19
3093 ; GFX7-NEXT: s_ashr_i32 s1, s1, 28
3094 ; GFX7-NEXT: s_and_b32 s20, s20, s0
3095 ; GFX7-NEXT: s_and_b32 s2, s2, s0
3096 ; GFX7-NEXT: v_mov_b32_e32 v7, s21
3097 ; GFX7-NEXT: s_and_b32 s0, s1, s0
3098 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3099 ; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0
3100 ; GFX7-NEXT: v_mad_u32_u24 v0, s10, v2, v0
3101 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v3, v0
3102 ; GFX7-NEXT: v_mad_u32_u24 v0, s14, v4, v0
3103 ; GFX7-NEXT: v_mad_u32_u24 v0, s16, v5, v0
3104 ; GFX7-NEXT: v_mad_u32_u24 v0, s18, v6, v0
3105 ; GFX7-NEXT: v_mad_u32_u24 v0, s20, v7, v0
3106 ; GFX7-NEXT: v_mov_b32_e32 v1, s2
3107 ; GFX7-NEXT: v_mad_u32_u24 v0, s0, v1, v0
3108 ; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0
3109 ; GFX7-NEXT: s_endpgm
3111 ; GFX8-LABEL: idot8_acc8:
3112 ; GFX8: ; %bb.0: ; %entry
3113 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3114 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3115 ; GFX8-NEXT: s_movk_i32 s2, 0xff
3116 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3117 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
3118 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
3119 ; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
3120 ; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
3121 ; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
3122 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3123 ; GFX8-NEXT: s_lshr_b32 s7, s0, 4
3124 ; GFX8-NEXT: s_lshr_b32 s11, s1, 4
3125 ; GFX8-NEXT: v_lshlrev_b16_e64 v3, 12, s7
3126 ; GFX8-NEXT: v_lshlrev_b16_e64 v4, 12, s11
3127 ; GFX8-NEXT: s_bfe_i32 s13, s1, 0x40000
3128 ; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
3129 ; GFX8-NEXT: v_ashrrev_i16_e32 v4, 12, v4
3130 ; GFX8-NEXT: s_lshr_b32 s6, s0, 12
3131 ; GFX8-NEXT: s_lshr_b32 s10, s1, 12
3132 ; GFX8-NEXT: s_bfe_i32 s15, s1, 0x40008
3133 ; GFX8-NEXT: s_bfe_i32 s12, s0, 0x40000
3134 ; GFX8-NEXT: v_mov_b32_e32 v12, s13
3135 ; GFX8-NEXT: v_lshlrev_b16_e64 v6, 12, s6
3136 ; GFX8-NEXT: v_lshlrev_b16_e64 v7, 12, s10
3137 ; GFX8-NEXT: s_bfe_i32 s14, s0, 0x40008
3138 ; GFX8-NEXT: v_mov_b32_e32 v5, s15
3139 ; GFX8-NEXT: v_and_b32_e32 v3, s2, v3
3140 ; GFX8-NEXT: v_and_b32_e32 v4, s2, v4
3141 ; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
3142 ; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
3143 ; GFX8-NEXT: s_lshr_b32 s5, s0, 20
3144 ; GFX8-NEXT: s_lshr_b32 s9, s1, 20
3145 ; GFX8-NEXT: v_mul_i32_i24_e32 v5, s14, v5
3146 ; GFX8-NEXT: v_lshlrev_b16_e64 v8, 12, s5
3147 ; GFX8-NEXT: v_lshlrev_b16_e64 v9, 12, s9
3148 ; GFX8-NEXT: s_bfe_i32 s17, s1, 0x40010
3149 ; GFX8-NEXT: v_and_b32_e32 v6, s2, v6
3150 ; GFX8-NEXT: v_and_b32_e32 v7, s2, v7
3151 ; GFX8-NEXT: s_lshr_b32 s8, s1, 28
3152 ; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
3153 ; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
3154 ; GFX8-NEXT: s_lshr_b32 s4, s0, 28
3155 ; GFX8-NEXT: s_bfe_i32 s16, s0, 0x40010
3156 ; GFX8-NEXT: v_mov_b32_e32 v13, s17
3157 ; GFX8-NEXT: v_lshlrev_b16_e64 v10, 12, s4
3158 ; GFX8-NEXT: v_lshlrev_b16_e64 v11, 12, s8
3159 ; GFX8-NEXT: s_bfe_i32 s1, s1, 0x40018
3160 ; GFX8-NEXT: v_and_b32_e32 v8, s2, v8
3161 ; GFX8-NEXT: v_and_b32_e32 v9, s2, v9
3162 ; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
3163 ; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v11
3164 ; GFX8-NEXT: s_bfe_i32 s0, s0, 0x40018
3165 ; GFX8-NEXT: v_and_b32_e32 v10, s2, v10
3166 ; GFX8-NEXT: v_and_b32_e32 v11, s2, v11
3167 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3168 ; GFX8-NEXT: v_mad_i32_i24 v2, s12, v12, v2
3169 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v4, v2
3170 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
3171 ; GFX8-NEXT: v_mad_u32_u24 v2, v6, v7, v2
3172 ; GFX8-NEXT: v_mad_i32_i24 v2, s16, v13, v2
3173 ; GFX8-NEXT: v_mad_u32_u24 v2, v8, v9, v2
3174 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
3175 ; GFX8-NEXT: v_mad_i32_i24 v2, s0, v3, v2
3176 ; GFX8-NEXT: v_mad_u32_u24 v2, v10, v11, v2
3177 ; GFX8-NEXT: flat_store_byte v[0:1], v2
3178 ; GFX8-NEXT: s_endpgm
3180 ; GFX9-LABEL: idot8_acc8:
3181 ; GFX9: ; %bb.0: ; %entry
3182 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3183 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3184 ; GFX9-NEXT: s_movk_i32 s2, 0xff
3185 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
3186 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
3187 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
3188 ; GFX9-NEXT: global_load_ubyte v2, v[0:1], off
3189 ; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
3190 ; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
3191 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
3192 ; GFX9-NEXT: s_lshr_b32 s7, s0, 4
3193 ; GFX9-NEXT: s_lshr_b32 s11, s1, 4
3194 ; GFX9-NEXT: v_lshlrev_b16_e64 v3, 12, s7
3195 ; GFX9-NEXT: v_lshlrev_b16_e64 v4, 12, s11
3196 ; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40000
3197 ; GFX9-NEXT: v_ashrrev_i16_e32 v3, 12, v3
3198 ; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4
3199 ; GFX9-NEXT: s_lshr_b32 s6, s0, 12
3200 ; GFX9-NEXT: s_lshr_b32 s10, s1, 12
3201 ; GFX9-NEXT: s_bfe_i32 s15, s1, 0x40008
3202 ; GFX9-NEXT: s_bfe_i32 s12, s0, 0x40000
3203 ; GFX9-NEXT: v_mov_b32_e32 v12, s13
3204 ; GFX9-NEXT: v_lshlrev_b16_e64 v6, 12, s6
3205 ; GFX9-NEXT: v_lshlrev_b16_e64 v7, 12, s10
3206 ; GFX9-NEXT: s_bfe_i32 s14, s0, 0x40008
3207 ; GFX9-NEXT: v_mov_b32_e32 v5, s15
3208 ; GFX9-NEXT: v_and_b32_e32 v3, s2, v3
3209 ; GFX9-NEXT: v_and_b32_e32 v4, s2, v4
3210 ; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
3211 ; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
3212 ; GFX9-NEXT: s_lshr_b32 s5, s0, 20
3213 ; GFX9-NEXT: s_lshr_b32 s9, s1, 20
3214 ; GFX9-NEXT: v_mul_i32_i24_e32 v5, s14, v5
3215 ; GFX9-NEXT: v_lshlrev_b16_e64 v8, 12, s5
3216 ; GFX9-NEXT: v_lshlrev_b16_e64 v9, 12, s9
3217 ; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40010
3218 ; GFX9-NEXT: v_and_b32_e32 v6, s2, v6
3219 ; GFX9-NEXT: v_and_b32_e32 v7, s2, v7
3220 ; GFX9-NEXT: s_lshr_b32 s8, s1, 28
3221 ; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
3222 ; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v9
3223 ; GFX9-NEXT: s_lshr_b32 s4, s0, 28
3224 ; GFX9-NEXT: s_bfe_i32 s16, s0, 0x40010
3225 ; GFX9-NEXT: v_mov_b32_e32 v13, s17
3226 ; GFX9-NEXT: v_lshlrev_b16_e64 v10, 12, s4
3227 ; GFX9-NEXT: v_lshlrev_b16_e64 v11, 12, s8
3228 ; GFX9-NEXT: s_bfe_i32 s1, s1, 0x40018
3229 ; GFX9-NEXT: v_and_b32_e32 v8, s2, v8
3230 ; GFX9-NEXT: v_and_b32_e32 v9, s2, v9
3231 ; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v10
3232 ; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
3233 ; GFX9-NEXT: s_bfe_i32 s0, s0, 0x40018
3234 ; GFX9-NEXT: v_and_b32_e32 v10, s2, v10
3235 ; GFX9-NEXT: v_and_b32_e32 v11, s2, v11
3236 ; GFX9-NEXT: s_waitcnt vmcnt(0)
3237 ; GFX9-NEXT: v_mad_i32_i24 v2, s12, v12, v2
3238 ; GFX9-NEXT: v_mad_u32_u24 v2, v3, v4, v2
3239 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
3240 ; GFX9-NEXT: v_mad_u32_u24 v2, v6, v7, v2
3241 ; GFX9-NEXT: v_mad_i32_i24 v2, s16, v13, v2
3242 ; GFX9-NEXT: v_mad_u32_u24 v2, v8, v9, v2
3243 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
3244 ; GFX9-NEXT: v_mad_i32_i24 v2, s0, v3, v2
3245 ; GFX9-NEXT: v_mad_u32_u24 v2, v10, v11, v2
3246 ; GFX9-NEXT: global_store_byte v[0:1], v2, off
3247 ; GFX9-NEXT: s_endpgm
3249 ; GFX9-DL-LABEL: idot8_acc8:
3250 ; GFX9-DL: ; %bb.0: ; %entry
3251 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3252 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3253 ; GFX9-DL-NEXT: s_movk_i32 s2, 0xff
3254 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3255 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
3256 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
3257 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off
3258 ; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
3259 ; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
3260 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3261 ; GFX9-DL-NEXT: s_lshr_b32 s7, s0, 4
3262 ; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 4
3263 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v3, 12, s7
3264 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v4, 12, s11
3265 ; GFX9-DL-NEXT: s_bfe_i32 s13, s1, 0x40000
3266 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v3, 12, v3
3267 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4
3268 ; GFX9-DL-NEXT: s_lshr_b32 s6, s0, 12
3269 ; GFX9-DL-NEXT: s_lshr_b32 s10, s1, 12
3270 ; GFX9-DL-NEXT: s_bfe_i32 s15, s1, 0x40008
3271 ; GFX9-DL-NEXT: s_bfe_i32 s12, s0, 0x40000
3272 ; GFX9-DL-NEXT: v_mov_b32_e32 v12, s13
3273 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v6, 12, s6
3274 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v7, 12, s10
3275 ; GFX9-DL-NEXT: s_bfe_i32 s14, s0, 0x40008
3276 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
3277 ; GFX9-DL-NEXT: v_and_b32_e32 v3, s2, v3
3278 ; GFX9-DL-NEXT: v_and_b32_e32 v4, s2, v4
3279 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
3280 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
3281 ; GFX9-DL-NEXT: s_lshr_b32 s5, s0, 20
3282 ; GFX9-DL-NEXT: s_lshr_b32 s9, s1, 20
3283 ; GFX9-DL-NEXT: v_mul_i32_i24_e32 v5, s14, v5
3284 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v8, 12, s5
3285 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v9, 12, s9
3286 ; GFX9-DL-NEXT: s_bfe_i32 s17, s1, 0x40010
3287 ; GFX9-DL-NEXT: v_and_b32_e32 v6, s2, v6
3288 ; GFX9-DL-NEXT: v_and_b32_e32 v7, s2, v7
3289 ; GFX9-DL-NEXT: s_lshr_b32 s8, s1, 28
3290 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
3291 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v9
3292 ; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
3293 ; GFX9-DL-NEXT: s_bfe_i32 s16, s0, 0x40010
3294 ; GFX9-DL-NEXT: v_mov_b32_e32 v13, s17
3295 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v10, 12, s4
3296 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v11, 12, s8
3297 ; GFX9-DL-NEXT: s_bfe_i32 s1, s1, 0x40018
3298 ; GFX9-DL-NEXT: v_and_b32_e32 v8, s2, v8
3299 ; GFX9-DL-NEXT: v_and_b32_e32 v9, s2, v9
3300 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v10
3301 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
3302 ; GFX9-DL-NEXT: s_bfe_i32 s0, s0, 0x40018
3303 ; GFX9-DL-NEXT: v_and_b32_e32 v10, s2, v10
3304 ; GFX9-DL-NEXT: v_and_b32_e32 v11, s2, v11
3305 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
3306 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s12, v12, v2
3307 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v3, v4, v2
3308 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
3309 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v6, v7, v2
3310 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s16, v13, v2
3311 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v8, v9, v2
3312 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
3313 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s0, v3, v2
3314 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v10, v11, v2
3315 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off
3316 ; GFX9-DL-NEXT: s_endpgm
3317 <8 x i4> addrspace(1)* %src2,
3318 i8 addrspace(1)* nocapture %dst) {
3320 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
3321 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
3323 %v1e0 = extractelement <8 x i4> %vec1, i64 0
3324 %cv1e0 = sext i4 %v1e0 to i8
3325 %v2e0 = extractelement <8 x i4> %vec2, i64 0
3326 %cv2e0 = sext i4 %v2e0 to i8
3327 %mul0 = mul nuw nsw i8 %cv1e0, %cv2e0
3329 %v1e1 = extractelement <8 x i4> %vec1, i64 1
3330 %cv1e1 = sext i4 %v1e1 to i8
3331 %v2e1 = extractelement <8 x i4> %vec2, i64 1
3332 %cv2e1 = sext i4 %v2e1 to i8
3333 %mul1 = mul nuw nsw i8 %cv1e1, %cv2e1
3335 %v1e2 = extractelement <8 x i4> %vec1, i64 2
3336 %cv1e2 = sext i4 %v1e2 to i8
3337 %v2e2 = extractelement <8 x i4> %vec2, i64 2
3338 %cv2e2 = sext i4 %v2e2 to i8
3339 %mul2 = mul nuw nsw i8 %cv1e2, %cv2e2
3341 %v1e3 = extractelement <8 x i4> %vec1, i64 3
3342 %cv1e3 = sext i4 %v1e3 to i8
3343 %v2e3 = extractelement <8 x i4> %vec2, i64 3
3344 %cv2e3 = sext i4 %v2e3 to i8
3345 %mul3 = mul nuw nsw i8 %cv1e3, %cv2e3
3347 %v1e4 = extractelement <8 x i4> %vec1, i64 4
3348 %cv1e4 = sext i4 %v1e4 to i8
3349 %v2e4 = extractelement <8 x i4> %vec2, i64 4
3350 %cv2e4 = sext i4 %v2e4 to i8
3351 %mul4 = mul nuw nsw i8 %cv1e4, %cv2e4
3353 %v1e5 = extractelement <8 x i4> %vec1, i64 5
3354 %cv1e5 = sext i4 %v1e5 to i8
3355 %v2e5 = extractelement <8 x i4> %vec2, i64 5
3356 %cv2e5 = sext i4 %v2e5 to i8
3357 %mul5 = mul nuw nsw i8 %cv1e5, %cv2e5
3359 %v1e6 = extractelement <8 x i4> %vec1, i64 6
3360 %cv1e6 = sext i4 %v1e6 to i8
3361 %v2e6 = extractelement <8 x i4> %vec2, i64 6
3362 %cv2e6 = sext i4 %v2e6 to i8
3363 %mul6 = mul nuw nsw i8 %cv1e6, %cv2e6
3365 %v1e7 = extractelement <8 x i4> %vec1, i64 7
3366 %cv1e7 = sext i4 %v1e7 to i8
3367 %v2e7 = extractelement <8 x i4> %vec2, i64 7
3368 %cv2e7 = sext i4 %v2e7 to i8
3369 %mul7 = mul nuw nsw i8 %cv1e7, %cv2e7
3371 %acc = load i8, i8 addrspace(1)* %dst, align 4
3372 %add1 = add i8 %mul0, %acc
3373 %add2 = add i8 %add1, %mul1
3374 %add3 = add i8 %add2, %mul2
3375 %add4 = add i8 %add3, %mul3
3376 %add5 = add i8 %add4, %mul4
3377 %add6 = add i8 %add5, %mul5
3378 %add7 = add i8 %add6, %mul6
3379 %add8 = add i8 %add7, %mul7
3381 store i8 %add8, i8 addrspace(1)* %dst, align 4
3385 ; Make sure the pattern is not recognized if there are multiple uses of the
3386 ; intermediate multiplications.
3387 define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
3388 ; GFX7-LABEL: idot8_multiuses_mul1:
3389 ; GFX7: ; %bb.0: ; %entry
3390 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
3391 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
3392 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3393 ; GFX7-NEXT: s_mov_b32 s6, -1
3394 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3395 ; GFX7-NEXT: s_load_dword s0, s[8:9], 0x0
3396 ; GFX7-NEXT: s_load_dword s1, s[10:11], 0x0
3397 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
3398 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3399 ; GFX7-NEXT: s_bfe_i32 s8, s0, 0x40000
3400 ; GFX7-NEXT: s_bfe_i32 s9, s1, 0x40000
3401 ; GFX7-NEXT: v_mov_b32_e32 v0, s9
3402 ; GFX7-NEXT: v_mov_b32_e32 v1, s2
3403 ; GFX7-NEXT: v_mad_i32_i24 v1, s8, v0, v1
3404 ; GFX7-NEXT: s_bfe_i32 s11, s1, 0x40004
3405 ; GFX7-NEXT: s_bfe_i32 s10, s0, 0x40004
3406 ; GFX7-NEXT: s_bfe_i32 s13, s1, 0x40008
3407 ; GFX7-NEXT: v_mad_i32_i24 v0, s8, v0, v1
3408 ; GFX7-NEXT: v_mov_b32_e32 v2, s11
3409 ; GFX7-NEXT: v_mad_i32_i24 v0, s10, v2, v0
3410 ; GFX7-NEXT: s_bfe_i32 s12, s0, 0x40008
3411 ; GFX7-NEXT: v_mov_b32_e32 v2, s13
3412 ; GFX7-NEXT: s_bfe_i32 s15, s1, 0x4000c
3413 ; GFX7-NEXT: v_mad_i32_i24 v0, s12, v2, v0
3414 ; GFX7-NEXT: s_bfe_i32 s14, s0, 0x4000c
3415 ; GFX7-NEXT: v_mov_b32_e32 v2, s15
3416 ; GFX7-NEXT: s_bfe_i32 s17, s1, 0x40010
3417 ; GFX7-NEXT: v_mad_i32_i24 v0, s14, v2, v0
3418 ; GFX7-NEXT: s_bfe_i32 s16, s0, 0x40010
3419 ; GFX7-NEXT: v_mov_b32_e32 v2, s17
3420 ; GFX7-NEXT: s_bfe_i32 s19, s1, 0x40014
3421 ; GFX7-NEXT: s_bfe_i32 s21, s1, 0x40018
3422 ; GFX7-NEXT: v_mad_i32_i24 v0, s16, v2, v0
3423 ; GFX7-NEXT: s_bfe_i32 s18, s0, 0x40014
3424 ; GFX7-NEXT: v_mov_b32_e32 v2, s19
3425 ; GFX7-NEXT: s_bfe_i32 s20, s0, 0x40018
3426 ; GFX7-NEXT: v_mad_i32_i24 v0, s18, v2, v0
3427 ; GFX7-NEXT: v_mov_b32_e32 v2, s21
3428 ; GFX7-NEXT: s_ashr_i32 s1, s1, 28
3429 ; GFX7-NEXT: v_mad_i32_i24 v0, s20, v2, v0
3430 ; GFX7-NEXT: s_ashr_i32 s0, s0, 28
3431 ; GFX7-NEXT: v_mov_b32_e32 v2, s1
3432 ; GFX7-NEXT: v_mad_i32_i24 v0, s0, v2, v0
3433 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
3434 ; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
3435 ; GFX7-NEXT: s_endpgm
3437 ; GFX8-LABEL: idot8_multiuses_mul1:
3438 ; GFX8: ; %bb.0: ; %entry
3439 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3440 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3441 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3442 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
3443 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
3444 ; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0
3445 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
3446 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
3447 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3448 ; GFX8-NEXT: s_bfe_i32 s0, s2, 0x40000
3449 ; GFX8-NEXT: s_bfe_i32 s1, s4, 0x40000
3450 ; GFX8-NEXT: v_mov_b32_e32 v2, s1
3451 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
3452 ; GFX8-NEXT: v_mad_i32_i24 v3, s0, v2, v3
3453 ; GFX8-NEXT: s_bfe_i32 s7, s4, 0x40004
3454 ; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40004
3455 ; GFX8-NEXT: s_bfe_i32 s9, s4, 0x40008
3456 ; GFX8-NEXT: v_mad_i32_i24 v2, s0, v2, v3
3457 ; GFX8-NEXT: v_mov_b32_e32 v4, s7
3458 ; GFX8-NEXT: v_mad_i32_i24 v2, s6, v4, v2
3459 ; GFX8-NEXT: s_bfe_i32 s8, s2, 0x40008
3460 ; GFX8-NEXT: v_mov_b32_e32 v4, s9
3461 ; GFX8-NEXT: s_bfe_i32 s11, s4, 0x4000c
3462 ; GFX8-NEXT: v_mad_i32_i24 v2, s8, v4, v2
3463 ; GFX8-NEXT: s_bfe_i32 s10, s2, 0x4000c
3464 ; GFX8-NEXT: v_mov_b32_e32 v4, s11
3465 ; GFX8-NEXT: s_bfe_i32 s13, s4, 0x40010
3466 ; GFX8-NEXT: v_mad_i32_i24 v2, s10, v4, v2
3467 ; GFX8-NEXT: s_bfe_i32 s12, s2, 0x40010
3468 ; GFX8-NEXT: v_mov_b32_e32 v4, s13
3469 ; GFX8-NEXT: s_bfe_i32 s15, s4, 0x40014
3470 ; GFX8-NEXT: s_bfe_i32 s17, s4, 0x40018
3471 ; GFX8-NEXT: v_mad_i32_i24 v2, s12, v4, v2
3472 ; GFX8-NEXT: s_bfe_i32 s14, s2, 0x40014
3473 ; GFX8-NEXT: v_mov_b32_e32 v4, s15
3474 ; GFX8-NEXT: s_bfe_i32 s16, s2, 0x40018
3475 ; GFX8-NEXT: v_mad_i32_i24 v2, s14, v4, v2
3476 ; GFX8-NEXT: v_mov_b32_e32 v4, s17
3477 ; GFX8-NEXT: s_ashr_i32 s4, s4, 28
3478 ; GFX8-NEXT: v_mad_i32_i24 v2, s16, v4, v2
3479 ; GFX8-NEXT: s_ashr_i32 s2, s2, 28
3480 ; GFX8-NEXT: v_mov_b32_e32 v4, s4
3481 ; GFX8-NEXT: v_mad_i32_i24 v2, s2, v4, v2
3482 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
3483 ; GFX8-NEXT: flat_store_dword v[0:1], v2
3484 ; GFX8-NEXT: s_endpgm
3486 ; GFX9-LABEL: idot8_multiuses_mul1:
3487 ; GFX9: ; %bb.0: ; %entry
3488 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3489 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3490 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
3491 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
3492 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
3493 ; GFX9-NEXT: s_load_dword s5, s[0:1], 0x0
3494 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
3495 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
3496 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
3497 ; GFX9-NEXT: s_bfe_i32 s0, s2, 0x40000
3498 ; GFX9-NEXT: s_bfe_i32 s1, s4, 0x40000
3499 ; GFX9-NEXT: v_mov_b32_e32 v2, s1
3500 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
3501 ; GFX9-NEXT: v_mad_i32_i24 v3, s0, v2, v3
3502 ; GFX9-NEXT: s_bfe_i32 s7, s4, 0x40004
3503 ; GFX9-NEXT: s_bfe_i32 s6, s2, 0x40004
3504 ; GFX9-NEXT: s_bfe_i32 s9, s4, 0x40008
3505 ; GFX9-NEXT: v_mad_i32_i24 v2, s0, v2, v3
3506 ; GFX9-NEXT: v_mov_b32_e32 v4, s7
3507 ; GFX9-NEXT: v_mad_i32_i24 v2, s6, v4, v2
3508 ; GFX9-NEXT: s_bfe_i32 s8, s2, 0x40008
3509 ; GFX9-NEXT: v_mov_b32_e32 v4, s9
3510 ; GFX9-NEXT: s_bfe_i32 s11, s4, 0x4000c
3511 ; GFX9-NEXT: v_mad_i32_i24 v2, s8, v4, v2
3512 ; GFX9-NEXT: s_bfe_i32 s10, s2, 0x4000c
3513 ; GFX9-NEXT: v_mov_b32_e32 v4, s11
3514 ; GFX9-NEXT: s_bfe_i32 s13, s4, 0x40010
3515 ; GFX9-NEXT: v_mad_i32_i24 v2, s10, v4, v2
3516 ; GFX9-NEXT: s_bfe_i32 s12, s2, 0x40010
3517 ; GFX9-NEXT: v_mov_b32_e32 v4, s13
3518 ; GFX9-NEXT: s_bfe_i32 s15, s4, 0x40014
3519 ; GFX9-NEXT: s_bfe_i32 s17, s4, 0x40018
3520 ; GFX9-NEXT: v_mad_i32_i24 v2, s12, v4, v2
3521 ; GFX9-NEXT: s_bfe_i32 s14, s2, 0x40014
3522 ; GFX9-NEXT: v_mov_b32_e32 v4, s15
3523 ; GFX9-NEXT: s_bfe_i32 s16, s2, 0x40018
3524 ; GFX9-NEXT: v_mad_i32_i24 v2, s14, v4, v2
3525 ; GFX9-NEXT: v_mov_b32_e32 v4, s17
3526 ; GFX9-NEXT: s_ashr_i32 s4, s4, 28
3527 ; GFX9-NEXT: v_mad_i32_i24 v2, s16, v4, v2
3528 ; GFX9-NEXT: s_ashr_i32 s2, s2, 28
3529 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
3530 ; GFX9-NEXT: v_mad_i32_i24 v2, s2, v4, v2
3531 ; GFX9-NEXT: v_add_u32_e32 v2, v3, v2
3532 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
3533 ; GFX9-NEXT: s_endpgm
3535 ; GFX9-DL-LABEL: idot8_multiuses_mul1:
3536 ; GFX9-DL: ; %bb.0: ; %entry
3537 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3538 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3539 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3540 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
3541 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
3542 ; GFX9-DL-NEXT: s_load_dword s5, s[0:1], 0x0
3543 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
3544 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
3545 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3546 ; GFX9-DL-NEXT: s_bfe_i32 s0, s2, 0x40000
3547 ; GFX9-DL-NEXT: s_bfe_i32 s1, s4, 0x40000
3548 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
3549 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5
3550 ; GFX9-DL-NEXT: v_mad_i32_i24 v3, s0, v2, v3
3551 ; GFX9-DL-NEXT: s_bfe_i32 s7, s4, 0x40004
3552 ; GFX9-DL-NEXT: s_bfe_i32 s6, s2, 0x40004
3553 ; GFX9-DL-NEXT: s_bfe_i32 s9, s4, 0x40008
3554 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s0, v2, v3
3555 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s7
3556 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s6, v4, v2
3557 ; GFX9-DL-NEXT: s_bfe_i32 s8, s2, 0x40008
3558 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s9
3559 ; GFX9-DL-NEXT: s_bfe_i32 s11, s4, 0x4000c
3560 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s8, v4, v2
3561 ; GFX9-DL-NEXT: s_bfe_i32 s10, s2, 0x4000c
3562 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s11
3563 ; GFX9-DL-NEXT: s_bfe_i32 s13, s4, 0x40010
3564 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s10, v4, v2
3565 ; GFX9-DL-NEXT: s_bfe_i32 s12, s2, 0x40010
3566 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s13
3567 ; GFX9-DL-NEXT: s_bfe_i32 s15, s4, 0x40014
3568 ; GFX9-DL-NEXT: s_bfe_i32 s17, s4, 0x40018
3569 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s12, v4, v2
3570 ; GFX9-DL-NEXT: s_bfe_i32 s14, s2, 0x40014
3571 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s15
3572 ; GFX9-DL-NEXT: s_bfe_i32 s16, s2, 0x40018
3573 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s14, v4, v2
3574 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s17
3575 ; GFX9-DL-NEXT: s_ashr_i32 s4, s4, 28
3576 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s16, v4, v2
3577 ; GFX9-DL-NEXT: s_ashr_i32 s2, s2, 28
3578 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s4
3579 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s2, v4, v2
3580 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v3, v2
3581 ; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off
3582 ; GFX9-DL-NEXT: s_endpgm
3583 <8 x i4> addrspace(1)* %src2,
3584 i32 addrspace(1)* nocapture %dst) {
3586 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
3587 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
3589 %v1e0 = extractelement <8 x i4> %vec1, i64 0
3590 %cv1e0 = sext i4 %v1e0 to i32
3591 %v2e0 = extractelement <8 x i4> %vec2, i64 0
3592 %cv2e0 = sext i4 %v2e0 to i32
3593 %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
3595 %v1e1 = extractelement <8 x i4> %vec1, i64 1
3596 %cv1e1 = sext i4 %v1e1 to i32
3597 %v2e1 = extractelement <8 x i4> %vec2, i64 1
3598 %cv2e1 = sext i4 %v2e1 to i32
3599 %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
3601 %v1e2 = extractelement <8 x i4> %vec1, i64 2
3602 %cv1e2 = sext i4 %v1e2 to i32
3603 %v2e2 = extractelement <8 x i4> %vec2, i64 2
3604 %cv2e2 = sext i4 %v2e2 to i32
3605 %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
3607 %v1e3 = extractelement <8 x i4> %vec1, i64 3
3608 %cv1e3 = sext i4 %v1e3 to i32
3609 %v2e3 = extractelement <8 x i4> %vec2, i64 3
3610 %cv2e3 = sext i4 %v2e3 to i32
3611 %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
3613 %v1e4 = extractelement <8 x i4> %vec1, i64 4
3614 %cv1e4 = sext i4 %v1e4 to i32
3615 %v2e4 = extractelement <8 x i4> %vec2, i64 4
3616 %cv2e4 = sext i4 %v2e4 to i32
3617 %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
3619 %v1e5 = extractelement <8 x i4> %vec1, i64 5
3620 %cv1e5 = sext i4 %v1e5 to i32
3621 %v2e5 = extractelement <8 x i4> %vec2, i64 5
3622 %cv2e5 = sext i4 %v2e5 to i32
3623 %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
3625 %v1e6 = extractelement <8 x i4> %vec1, i64 6
3626 %cv1e6 = sext i4 %v1e6 to i32
3627 %v2e6 = extractelement <8 x i4> %vec2, i64 6
3628 %cv2e6 = sext i4 %v2e6 to i32
3629 %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
3631 %v1e7 = extractelement <8 x i4> %vec1, i64 7
3632 %cv1e7 = sext i4 %v1e7 to i32
3633 %v2e7 = extractelement <8 x i4> %vec2, i64 7
3634 %cv2e7 = sext i4 %v2e7 to i32
3635 %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
3637 %acc = load i32, i32 addrspace(1)* %dst, align 4
3638 %add = add i32 %mul0, %acc
3639 %add1 = add i32 %mul0, %add
3640 %add2 = add i32 %add1, %mul1
3641 %add3 = add i32 %add2, %mul2
3642 %add4 = add i32 %add3, %mul3
3643 %add5 = add i32 %add4, %mul4
3644 %add6 = add i32 %add5, %mul5
3645 %add7 = add i32 %add6, %mul6
3646 %add8 = add i32 %add7, %mul7
3648 %res = add i32 %add, %add8
3649 store i32 %res, i32 addrspace(1)* %dst, align 4
3653 ; TODO: Support this pattern.
3654 define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
3655 ; GFX7-LABEL: idot8_acc32_vecMul:
3656 ; GFX7: ; %bb.0: ; %entry
3657 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
3658 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
3659 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3660 ; GFX7-NEXT: s_mov_b32 s6, -1
3661 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3662 ; GFX7-NEXT: s_load_dword s1, s[8:9], 0x0
3663 ; GFX7-NEXT: s_load_dword s9, s[10:11], 0x0
3664 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
3665 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3666 ; GFX7-NEXT: s_ashr_i64 s[10:11], s[0:1], 60
3667 ; GFX7-NEXT: s_lshl_b32 s11, s1, 4
3668 ; GFX7-NEXT: s_ashr_i64 s[14:15], s[10:11], 60
3669 ; GFX7-NEXT: s_lshl_b32 s11, s1, 12
3670 ; GFX7-NEXT: s_ashr_i64 s[16:17], s[10:11], 60
3671 ; GFX7-NEXT: s_lshl_b32 s11, s1, 16
3672 ; GFX7-NEXT: s_ashr_i64 s[18:19], s[10:11], 60
3673 ; GFX7-NEXT: s_lshl_b32 s11, s1, 20
3674 ; GFX7-NEXT: s_lshl_b32 s13, s1, 8
3675 ; GFX7-NEXT: s_ashr_i64 s[20:21], s[10:11], 60
3676 ; GFX7-NEXT: s_lshl_b32 s11, s1, 24
3677 ; GFX7-NEXT: s_lshl_b32 s1, s1, 28
3678 ; GFX7-NEXT: s_ashr_i64 s[0:1], s[0:1], 60
3679 ; GFX7-NEXT: s_lshl_b32 s1, s9, 4
3680 ; GFX7-NEXT: s_ashr_i64 s[26:27], s[0:1], 60
3681 ; GFX7-NEXT: s_lshl_b32 s1, s9, 8
3682 ; GFX7-NEXT: s_ashr_i64 s[28:29], s[0:1], 60
3683 ; GFX7-NEXT: s_lshl_b32 s1, s9, 12
3684 ; GFX7-NEXT: s_ashr_i64 s[30:31], s[0:1], 60
3685 ; GFX7-NEXT: s_lshl_b32 s1, s9, 16
3686 ; GFX7-NEXT: s_ashr_i64 s[32:33], s[0:1], 60
3687 ; GFX7-NEXT: s_lshl_b32 s1, s9, 20
3688 ; GFX7-NEXT: s_ashr_i64 s[34:35], s[0:1], 60
3689 ; GFX7-NEXT: s_lshl_b32 s1, s9, 24
3690 ; GFX7-NEXT: s_ashr_i64 s[36:37], s[0:1], 60
3691 ; GFX7-NEXT: s_lshl_b32 s1, s9, 28
3692 ; GFX7-NEXT: s_ashr_i64 s[24:25], s[8:9], 60
3693 ; GFX7-NEXT: s_ashr_i64 s[8:9], s[0:1], 60
3694 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
3695 ; GFX7-NEXT: v_mov_b32_e32 v1, s2
3696 ; GFX7-NEXT: v_mad_i32_i24 v0, s0, v0, v1
3697 ; GFX7-NEXT: s_ashr_i64 s[22:23], s[10:11], 60
3698 ; GFX7-NEXT: v_mov_b32_e32 v1, s36
3699 ; GFX7-NEXT: v_mad_i32_i24 v0, s22, v1, v0
3700 ; GFX7-NEXT: v_mov_b32_e32 v1, s34
3701 ; GFX7-NEXT: v_mad_i32_i24 v0, s20, v1, v0
3702 ; GFX7-NEXT: v_mov_b32_e32 v1, s32
3703 ; GFX7-NEXT: v_mad_i32_i24 v0, s18, v1, v0
3704 ; GFX7-NEXT: v_mov_b32_e32 v1, s30
3705 ; GFX7-NEXT: v_mad_i32_i24 v0, s16, v1, v0
3706 ; GFX7-NEXT: s_ashr_i64 s[12:13], s[12:13], 60
3707 ; GFX7-NEXT: v_mov_b32_e32 v1, s28
3708 ; GFX7-NEXT: v_mad_i32_i24 v0, s12, v1, v0
3709 ; GFX7-NEXT: v_mov_b32_e32 v1, s26
3710 ; GFX7-NEXT: v_mad_i32_i24 v0, s14, v1, v0
3711 ; GFX7-NEXT: v_mov_b32_e32 v1, s24
3712 ; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
3713 ; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
3714 ; GFX7-NEXT: s_endpgm
3716 ; GFX8-LABEL: idot8_acc32_vecMul:
3717 ; GFX8: ; %bb.0: ; %entry
3718 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3719 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3720 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3721 ; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
3722 ; GFX8-NEXT: s_load_dword s7, s[6:7], 0x0
3723 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
3724 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
3725 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
3726 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3727 ; GFX8-NEXT: s_ashr_i64 s[0:1], s[4:5], 60
3728 ; GFX8-NEXT: s_lshl_b32 s1, s5, 4
3729 ; GFX8-NEXT: s_ashr_i64 s[12:13], s[0:1], 60
3730 ; GFX8-NEXT: s_lshl_b32 s1, s5, 16
3731 ; GFX8-NEXT: s_ashr_i64 s[14:15], s[0:1], 60
3732 ; GFX8-NEXT: s_lshl_b32 s1, s5, 20
3733 ; GFX8-NEXT: s_ashr_i64 s[16:17], s[0:1], 60
3734 ; GFX8-NEXT: s_lshl_b32 s1, s5, 24
3735 ; GFX8-NEXT: s_ashr_i64 s[18:19], s[0:1], 60
3736 ; GFX8-NEXT: s_lshl_b32 s1, s5, 28
3737 ; GFX8-NEXT: s_lshl_b32 s9, s5, 8
3738 ; GFX8-NEXT: s_lshl_b32 s11, s5, 12
3739 ; GFX8-NEXT: s_ashr_i64 s[4:5], s[0:1], 60
3740 ; GFX8-NEXT: s_lshl_b32 s1, s7, 4
3741 ; GFX8-NEXT: s_ashr_i64 s[22:23], s[0:1], 60
3742 ; GFX8-NEXT: s_lshl_b32 s1, s7, 8
3743 ; GFX8-NEXT: s_ashr_i64 s[24:25], s[0:1], 60
3744 ; GFX8-NEXT: s_lshl_b32 s1, s7, 12
3745 ; GFX8-NEXT: s_ashr_i64 s[26:27], s[0:1], 60
3746 ; GFX8-NEXT: s_lshl_b32 s1, s7, 16
3747 ; GFX8-NEXT: s_ashr_i64 s[28:29], s[0:1], 60
3748 ; GFX8-NEXT: s_lshl_b32 s1, s7, 20
3749 ; GFX8-NEXT: s_ashr_i64 s[30:31], s[0:1], 60
3750 ; GFX8-NEXT: s_lshl_b32 s1, s7, 24
3751 ; GFX8-NEXT: s_ashr_i64 s[32:33], s[0:1], 60
3752 ; GFX8-NEXT: s_lshl_b32 s1, s7, 28
3753 ; GFX8-NEXT: s_ashr_i64 s[20:21], s[6:7], 60
3754 ; GFX8-NEXT: s_ashr_i64 s[6:7], s[0:1], 60
3755 ; GFX8-NEXT: v_mov_b32_e32 v2, s6
3756 ; GFX8-NEXT: v_mov_b32_e32 v3, s2
3757 ; GFX8-NEXT: v_mad_i32_i24 v2, s4, v2, v3
3758 ; GFX8-NEXT: v_mov_b32_e32 v3, s32
3759 ; GFX8-NEXT: v_mad_i32_i24 v2, s18, v3, v2
3760 ; GFX8-NEXT: v_mov_b32_e32 v3, s30
3761 ; GFX8-NEXT: v_mad_i32_i24 v2, s16, v3, v2
3762 ; GFX8-NEXT: v_mov_b32_e32 v3, s28
3763 ; GFX8-NEXT: v_mad_i32_i24 v2, s14, v3, v2
3764 ; GFX8-NEXT: s_ashr_i64 s[10:11], s[10:11], 60
3765 ; GFX8-NEXT: v_mov_b32_e32 v3, s26
3766 ; GFX8-NEXT: v_mad_i32_i24 v2, s10, v3, v2
3767 ; GFX8-NEXT: s_ashr_i64 s[8:9], s[8:9], 60
3768 ; GFX8-NEXT: v_mov_b32_e32 v3, s24
3769 ; GFX8-NEXT: v_mad_i32_i24 v2, s8, v3, v2
3770 ; GFX8-NEXT: v_mov_b32_e32 v3, s22
3771 ; GFX8-NEXT: v_mad_i32_i24 v2, s12, v3, v2
3772 ; GFX8-NEXT: v_mov_b32_e32 v3, s20
3773 ; GFX8-NEXT: v_mad_i32_i24 v2, s0, v3, v2
3774 ; GFX8-NEXT: flat_store_dword v[0:1], v2
3775 ; GFX8-NEXT: s_endpgm
3777 ; GFX9-LABEL: idot8_acc32_vecMul:
3778 ; GFX9: ; %bb.0: ; %entry
3779 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3780 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3781 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
3782 ; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
3783 ; GFX9-NEXT: s_load_dword s7, s[6:7], 0x0
3784 ; GFX9-NEXT: s_load_dword s2, s[0:1], 0x0
3785 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
3786 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
3787 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
3788 ; GFX9-NEXT: s_ashr_i64 s[0:1], s[4:5], 60
3789 ; GFX9-NEXT: s_lshl_b32 s1, s5, 4
3790 ; GFX9-NEXT: s_ashr_i64 s[12:13], s[0:1], 60
3791 ; GFX9-NEXT: s_lshl_b32 s1, s5, 16
3792 ; GFX9-NEXT: s_ashr_i64 s[14:15], s[0:1], 60
3793 ; GFX9-NEXT: s_lshl_b32 s1, s5, 20
3794 ; GFX9-NEXT: s_ashr_i64 s[16:17], s[0:1], 60
3795 ; GFX9-NEXT: s_lshl_b32 s1, s5, 24
3796 ; GFX9-NEXT: s_ashr_i64 s[18:19], s[0:1], 60
3797 ; GFX9-NEXT: s_lshl_b32 s1, s5, 28
3798 ; GFX9-NEXT: s_lshl_b32 s9, s5, 8
3799 ; GFX9-NEXT: s_lshl_b32 s11, s5, 12
3800 ; GFX9-NEXT: s_ashr_i64 s[4:5], s[0:1], 60
3801 ; GFX9-NEXT: s_lshl_b32 s1, s7, 4
3802 ; GFX9-NEXT: s_ashr_i64 s[22:23], s[0:1], 60
3803 ; GFX9-NEXT: s_lshl_b32 s1, s7, 8
3804 ; GFX9-NEXT: s_ashr_i64 s[24:25], s[0:1], 60
3805 ; GFX9-NEXT: s_lshl_b32 s1, s7, 12
3806 ; GFX9-NEXT: s_ashr_i64 s[26:27], s[0:1], 60
3807 ; GFX9-NEXT: s_lshl_b32 s1, s7, 16
3808 ; GFX9-NEXT: s_ashr_i64 s[28:29], s[0:1], 60
3809 ; GFX9-NEXT: s_lshl_b32 s1, s7, 20
3810 ; GFX9-NEXT: s_ashr_i64 s[30:31], s[0:1], 60
3811 ; GFX9-NEXT: s_lshl_b32 s1, s7, 24
3812 ; GFX9-NEXT: s_ashr_i64 s[32:33], s[0:1], 60
3813 ; GFX9-NEXT: s_lshl_b32 s1, s7, 28
3814 ; GFX9-NEXT: s_ashr_i64 s[20:21], s[6:7], 60
3815 ; GFX9-NEXT: s_ashr_i64 s[6:7], s[0:1], 60
3816 ; GFX9-NEXT: v_mov_b32_e32 v2, s6
3817 ; GFX9-NEXT: v_mov_b32_e32 v3, s2
3818 ; GFX9-NEXT: v_mad_i32_i24 v2, s4, v2, v3
3819 ; GFX9-NEXT: v_mov_b32_e32 v3, s32
3820 ; GFX9-NEXT: v_mad_i32_i24 v2, s18, v3, v2
3821 ; GFX9-NEXT: v_mov_b32_e32 v3, s30
3822 ; GFX9-NEXT: v_mad_i32_i24 v2, s16, v3, v2
3823 ; GFX9-NEXT: v_mov_b32_e32 v3, s28
3824 ; GFX9-NEXT: v_mad_i32_i24 v2, s14, v3, v2
3825 ; GFX9-NEXT: s_ashr_i64 s[10:11], s[10:11], 60
3826 ; GFX9-NEXT: v_mov_b32_e32 v3, s26
3827 ; GFX9-NEXT: v_mad_i32_i24 v2, s10, v3, v2
3828 ; GFX9-NEXT: s_ashr_i64 s[8:9], s[8:9], 60
3829 ; GFX9-NEXT: v_mov_b32_e32 v3, s24
3830 ; GFX9-NEXT: v_mad_i32_i24 v2, s8, v3, v2
3831 ; GFX9-NEXT: v_mov_b32_e32 v3, s22
3832 ; GFX9-NEXT: v_mad_i32_i24 v2, s12, v3, v2
3833 ; GFX9-NEXT: v_mov_b32_e32 v3, s20
3834 ; GFX9-NEXT: v_mad_i32_i24 v2, s0, v3, v2
3835 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
3836 ; GFX9-NEXT: s_endpgm
3838 ; GFX9-DL-LABEL: idot8_acc32_vecMul:
3839 ; GFX9-DL: ; %bb.0: ; %entry
3840 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3841 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3842 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3843 ; GFX9-DL-NEXT: s_load_dword s5, s[4:5], 0x0
3844 ; GFX9-DL-NEXT: s_load_dword s7, s[6:7], 0x0
3845 ; GFX9-DL-NEXT: s_load_dword s2, s[0:1], 0x0
3846 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
3847 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
3848 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3849 ; GFX9-DL-NEXT: s_ashr_i64 s[0:1], s[4:5], 60
3850 ; GFX9-DL-NEXT: s_lshl_b32 s1, s5, 4
3851 ; GFX9-DL-NEXT: s_ashr_i64 s[12:13], s[0:1], 60
3852 ; GFX9-DL-NEXT: s_lshl_b32 s1, s5, 16
3853 ; GFX9-DL-NEXT: s_ashr_i64 s[14:15], s[0:1], 60
3854 ; GFX9-DL-NEXT: s_lshl_b32 s1, s5, 20
3855 ; GFX9-DL-NEXT: s_ashr_i64 s[16:17], s[0:1], 60
3856 ; GFX9-DL-NEXT: s_lshl_b32 s1, s5, 24
3857 ; GFX9-DL-NEXT: s_ashr_i64 s[18:19], s[0:1], 60
3858 ; GFX9-DL-NEXT: s_lshl_b32 s1, s5, 28
3859 ; GFX9-DL-NEXT: s_lshl_b32 s9, s5, 8
3860 ; GFX9-DL-NEXT: s_lshl_b32 s11, s5, 12
3861 ; GFX9-DL-NEXT: s_ashr_i64 s[4:5], s[0:1], 60
3862 ; GFX9-DL-NEXT: s_lshl_b32 s1, s7, 4
3863 ; GFX9-DL-NEXT: s_ashr_i64 s[22:23], s[0:1], 60
3864 ; GFX9-DL-NEXT: s_lshl_b32 s1, s7, 8
3865 ; GFX9-DL-NEXT: s_ashr_i64 s[24:25], s[0:1], 60
3866 ; GFX9-DL-NEXT: s_lshl_b32 s1, s7, 12
3867 ; GFX9-DL-NEXT: s_ashr_i64 s[26:27], s[0:1], 60
3868 ; GFX9-DL-NEXT: s_lshl_b32 s1, s7, 16
3869 ; GFX9-DL-NEXT: s_ashr_i64 s[28:29], s[0:1], 60
3870 ; GFX9-DL-NEXT: s_lshl_b32 s1, s7, 20
3871 ; GFX9-DL-NEXT: s_ashr_i64 s[30:31], s[0:1], 60
3872 ; GFX9-DL-NEXT: s_lshl_b32 s1, s7, 24
3873 ; GFX9-DL-NEXT: s_ashr_i64 s[32:33], s[0:1], 60
3874 ; GFX9-DL-NEXT: s_lshl_b32 s1, s7, 28
3875 ; GFX9-DL-NEXT: s_ashr_i64 s[20:21], s[6:7], 60
3876 ; GFX9-DL-NEXT: s_ashr_i64 s[6:7], s[0:1], 60
3877 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, s6
3878 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s2
3879 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s4, v2, v3
3880 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s32
3881 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s18, v3, v2
3882 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s30
3883 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s16, v3, v2
3884 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s28
3885 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s14, v3, v2
3886 ; GFX9-DL-NEXT: s_ashr_i64 s[10:11], s[10:11], 60
3887 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s26
3888 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s10, v3, v2
3889 ; GFX9-DL-NEXT: s_ashr_i64 s[8:9], s[8:9], 60
3890 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s24
3891 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s8, v3, v2
3892 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s22
3893 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s12, v3, v2
3894 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s20
3895 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, s0, v3, v2
3896 ; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off
3897 ; GFX9-DL-NEXT: s_endpgm
3898 <8 x i4> addrspace(1)* %src2,
3899 i32 addrspace(1)* nocapture %dst) {
3901 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
3902 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
3904 %cvec1 = sext <8 x i4> %vec1 to <8 x i32>
3905 %cvec2 = sext <8 x i4> %vec2 to <8 x i32>
3907 %mul = mul <8 x i32> %cvec1, %cvec2
3908 %mul0 = extractelement <8 x i32> %mul, i64 0
3909 %mul1 = extractelement <8 x i32> %mul, i64 1
3910 %mul2 = extractelement <8 x i32> %mul, i64 2
3911 %mul3 = extractelement <8 x i32> %mul, i64 3
3912 %mul4 = extractelement <8 x i32> %mul, i64 4
3913 %mul5 = extractelement <8 x i32> %mul, i64 5
3914 %mul6 = extractelement <8 x i32> %mul, i64 6
3915 %mul7 = extractelement <8 x i32> %mul, i64 7
3917 %acc = load i32, i32 addrspace(1)* %dst, align 4
3918 %add1 = add i32 %mul0, %acc
3919 %add2 = add i32 %add1, %mul1
3920 %add3 = add i32 %add2, %mul2
3921 %add4 = add i32 %add3, %mul3
3922 %add5 = add i32 %add4, %mul4
3923 %add6 = add i32 %add5, %mul5
3924 %add7 = add i32 %add6, %mul6
3925 %add8 = add i32 %add7, %mul7
3927 store i32 %add8, i32 addrspace(1)* %dst, align 4
3931 ; TODO: Support this pattern.
3932 define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
3933 ; GFX7-LABEL: idot8_acc16_vecMul:
3934 ; GFX7: ; %bb.0: ; %entry
3935 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
3936 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
3937 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3938 ; GFX7-NEXT: s_mov_b32 s6, -1
3939 ; GFX7-NEXT: s_mov_b32 s0, 0xffff
3940 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3941 ; GFX7-NEXT: s_load_dword s1, s[8:9], 0x0
3942 ; GFX7-NEXT: buffer_load_ushort v0, off, s[4:7], 0
3943 ; GFX7-NEXT: s_load_dword s2, s[10:11], 0x0
3944 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3945 ; GFX7-NEXT: s_bfe_i32 s8, s1, 0x40010
3946 ; GFX7-NEXT: s_bfe_i32 s9, s1, 0x40014
3947 ; GFX7-NEXT: s_bfe_i32 s15, s2, 0x40010
3948 ; GFX7-NEXT: s_bfe_i32 s16, s2, 0x40014
3949 ; GFX7-NEXT: s_bfe_i32 s17, s2, 0x40018
3950 ; GFX7-NEXT: s_ashr_i32 s18, s2, 28
3951 ; GFX7-NEXT: s_bfe_i32 s19, s2, 0x40000
3952 ; GFX7-NEXT: s_bfe_i32 s20, s2, 0x40004
3953 ; GFX7-NEXT: s_bfe_i32 s21, s2, 0x40008
3954 ; GFX7-NEXT: s_bfe_i32 s2, s2, 0x4000c
3955 ; GFX7-NEXT: s_bfe_i32 s10, s1, 0x40018
3956 ; GFX7-NEXT: s_ashr_i32 s11, s1, 28
3957 ; GFX7-NEXT: s_bfe_i32 s12, s1, 0x40000
3958 ; GFX7-NEXT: v_mov_b32_e32 v4, s19
3959 ; GFX7-NEXT: s_bfe_i32 s13, s1, 0x40004
3960 ; GFX7-NEXT: v_mov_b32_e32 v3, s20
3961 ; GFX7-NEXT: s_bfe_i32 s14, s1, 0x40008
3962 ; GFX7-NEXT: v_mov_b32_e32 v2, s21
3963 ; GFX7-NEXT: s_bfe_i32 s1, s1, 0x4000c
3964 ; GFX7-NEXT: v_mov_b32_e32 v1, s2
3965 ; GFX7-NEXT: v_mov_b32_e32 v5, s18
3966 ; GFX7-NEXT: v_mov_b32_e32 v6, s17
3967 ; GFX7-NEXT: v_mul_i32_i24_e32 v1, s1, v1
3968 ; GFX7-NEXT: v_mul_i32_i24_e32 v2, s14, v2
3969 ; GFX7-NEXT: v_mul_i32_i24_e32 v3, s13, v3
3970 ; GFX7-NEXT: v_mul_i32_i24_e32 v4, s12, v4
3971 ; GFX7-NEXT: v_mul_i32_i24_e32 v5, s11, v5
3972 ; GFX7-NEXT: v_mul_i32_i24_e32 v6, s10, v6
3973 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
3974 ; GFX7-NEXT: v_and_b32_e32 v2, s0, v2
3975 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
3976 ; GFX7-NEXT: v_and_b32_e32 v4, s0, v4
3977 ; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
3978 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v3
3979 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
3980 ; GFX7-NEXT: v_and_b32_e32 v6, s0, v6
3981 ; GFX7-NEXT: v_mov_b32_e32 v7, s16
3982 ; GFX7-NEXT: v_mov_b32_e32 v8, s15
3983 ; GFX7-NEXT: v_or_b32_e32 v3, v6, v5
3984 ; GFX7-NEXT: v_alignbit_b32 v5, v1, v2, 16
3985 ; GFX7-NEXT: v_mul_i32_i24_e32 v7, s9, v7
3986 ; GFX7-NEXT: v_mul_i32_i24_e32 v8, s8, v8
3987 ; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
3988 ; GFX7-NEXT: v_and_b32_e32 v8, s0, v8
3989 ; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v1
3990 ; GFX7-NEXT: v_or_b32_e32 v4, v8, v7
3991 ; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v4
3992 ; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v3
3993 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3994 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
3995 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v5, v0
3996 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
3997 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v6, v0
3998 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0
3999 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v7, v0
4000 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0
4001 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v8, v0
4002 ; GFX7-NEXT: buffer_store_short v0, off, s[4:7], 0
4003 ; GFX7-NEXT: s_endpgm
4005 ; GFX8-LABEL: idot8_acc16_vecMul:
4006 ; GFX8: ; %bb.0: ; %entry
4007 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4008 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4009 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4010 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
4011 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
4012 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
4013 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
4014 ; GFX8-NEXT: flat_load_ushort v2, v[0:1]
4015 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4016 ; GFX8-NEXT: v_lshlrev_b16_e64 v3, 12, s2
4017 ; GFX8-NEXT: v_lshlrev_b16_e64 v4, 12, s4
4018 ; GFX8-NEXT: s_lshr_b32 s0, s2, 4
4019 ; GFX8-NEXT: s_lshr_b32 s1, s2, 8
4020 ; GFX8-NEXT: s_lshr_b32 s5, s4, 4
4021 ; GFX8-NEXT: s_lshr_b32 s6, s4, 8
4022 ; GFX8-NEXT: v_lshlrev_b16_e64 v5, 12, s1
4023 ; GFX8-NEXT: v_lshlrev_b16_e64 v6, 12, s0
4024 ; GFX8-NEXT: v_lshlrev_b16_e64 v7, 12, s6
4025 ; GFX8-NEXT: v_lshlrev_b16_e64 v8, 12, s5
4026 ; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
4027 ; GFX8-NEXT: v_ashrrev_i16_e32 v4, 12, v4
4028 ; GFX8-NEXT: s_lshr_b32 s0, s2, 12
4029 ; GFX8-NEXT: s_lshr_b32 s1, s4, 12
4030 ; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
4031 ; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v5
4032 ; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
4033 ; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
4034 ; GFX8-NEXT: v_lshlrev_b16_e64 v9, 12, s0
4035 ; GFX8-NEXT: v_lshlrev_b16_e64 v10, 12, s1
4036 ; GFX8-NEXT: s_lshr_b32 s5, s2, 16
4037 ; GFX8-NEXT: s_lshr_b32 s6, s4, 16
4038 ; GFX8-NEXT: v_mul_u32_u24_e32 v5, v5, v7
4039 ; GFX8-NEXT: v_lshlrev_b16_e64 v11, 12, s5
4040 ; GFX8-NEXT: v_lshlrev_b16_e64 v12, 12, s6
4041 ; GFX8-NEXT: s_lshr_b32 s0, s2, 20
4042 ; GFX8-NEXT: s_lshr_b32 s1, s4, 20
4043 ; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
4044 ; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
4045 ; GFX8-NEXT: v_lshlrev_b16_e64 v13, 12, s0
4046 ; GFX8-NEXT: v_lshlrev_b16_e64 v14, 12, s1
4047 ; GFX8-NEXT: s_lshr_b32 s5, s2, 24
4048 ; GFX8-NEXT: s_lshr_b32 s6, s4, 24
4049 ; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v11
4050 ; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
4051 ; GFX8-NEXT: v_lshlrev_b16_e64 v15, 12, s5
4052 ; GFX8-NEXT: v_lshlrev_b16_e64 v17, 12, s6
4053 ; GFX8-NEXT: s_lshr_b32 s0, s2, 28
4054 ; GFX8-NEXT: s_lshr_b32 s1, s4, 28
4055 ; GFX8-NEXT: v_ashrrev_i16_e32 v13, 12, v13
4056 ; GFX8-NEXT: v_ashrrev_i16_e32 v14, 12, v14
4057 ; GFX8-NEXT: v_lshlrev_b16_e64 v16, 12, s0
4058 ; GFX8-NEXT: v_lshlrev_b16_e64 v18, 12, s1
4059 ; GFX8-NEXT: v_ashrrev_i16_e32 v15, 12, v15
4060 ; GFX8-NEXT: v_ashrrev_i16_e32 v17, 12, v17
4061 ; GFX8-NEXT: v_ashrrev_i16_e32 v16, 12, v16
4062 ; GFX8-NEXT: v_ashrrev_i16_e32 v18, 12, v18
4063 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4064 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v4, v2
4065 ; GFX8-NEXT: v_mad_u32_u24 v2, v6, v8, v2
4066 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
4067 ; GFX8-NEXT: v_mad_u32_u24 v2, v9, v10, v2
4068 ; GFX8-NEXT: v_mad_u32_u24 v2, v11, v12, v2
4069 ; GFX8-NEXT: v_mad_u32_u24 v2, v13, v14, v2
4070 ; GFX8-NEXT: v_mad_u32_u24 v2, v15, v17, v2
4071 ; GFX8-NEXT: v_mad_u32_u24 v2, v16, v18, v2
4072 ; GFX8-NEXT: flat_store_short v[0:1], v2
4073 ; GFX8-NEXT: s_endpgm
4075 ; GFX9-LABEL: idot8_acc16_vecMul:
4076 ; GFX9: ; %bb.0: ; %entry
4077 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4078 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4079 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
4080 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
4081 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
4082 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
4083 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
4084 ; GFX9-NEXT: global_load_ushort v2, v[0:1], off
4085 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
4086 ; GFX9-NEXT: s_and_b32 s0, s2, 15
4087 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004
4088 ; GFX9-NEXT: s_and_b32 s5, s4, 15
4089 ; GFX9-NEXT: s_bfe_u32 s6, s4, 0x40004
4090 ; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
4091 ; GFX9-NEXT: s_pack_ll_b32_b16 s1, s5, s6
4092 ; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40008
4093 ; GFX9-NEXT: s_bfe_u32 s6, s2, 0x4000c
4094 ; GFX9-NEXT: v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
4095 ; GFX9-NEXT: s_pack_ll_b32_b16 s0, s5, s6
4096 ; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40010
4097 ; GFX9-NEXT: s_bfe_u32 s10, s2, 0x40014
4098 ; GFX9-NEXT: v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
4099 ; GFX9-NEXT: s_bfe_u32 s13, s2, 0x40018
4100 ; GFX9-NEXT: s_pack_ll_b32_b16 s0, s9, s10
4101 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28
4102 ; GFX9-NEXT: v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
4103 ; GFX9-NEXT: v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
4104 ; GFX9-NEXT: s_pack_ll_b32_b16 s0, s13, s2
4105 ; GFX9-NEXT: s_bfe_u32 s7, s4, 0x40008
4106 ; GFX9-NEXT: s_bfe_u32 s8, s4, 0x4000c
4107 ; GFX9-NEXT: v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
4108 ; GFX9-NEXT: s_pack_ll_b32_b16 s0, s7, s8
4109 ; GFX9-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
4110 ; GFX9-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
4111 ; GFX9-NEXT: v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
4112 ; GFX9-NEXT: v_pk_mul_lo_u16 v3, v3, v7
4113 ; GFX9-NEXT: s_bfe_u32 s11, s4, 0x40010
4114 ; GFX9-NEXT: s_bfe_u32 s12, s4, 0x40014
4115 ; GFX9-NEXT: s_pack_ll_b32_b16 s0, s11, s12
4116 ; GFX9-NEXT: v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
4117 ; GFX9-NEXT: v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
4118 ; GFX9-NEXT: v_pk_lshlrev_b16 v9, 12, s0 op_sel_hi:[0,1]
4119 ; GFX9-NEXT: s_bfe_u32 s14, s4, 0x40018
4120 ; GFX9-NEXT: s_lshr_b32 s4, s4, 28
4121 ; GFX9-NEXT: v_pk_mul_lo_u16 v4, v4, v8
4122 ; GFX9-NEXT: s_pack_ll_b32_b16 s0, s14, s4
4123 ; GFX9-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
4124 ; GFX9-NEXT: v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
4125 ; GFX9-NEXT: v_pk_lshlrev_b16 v10, 12, s0 op_sel_hi:[0,1]
4126 ; GFX9-NEXT: v_pk_mul_lo_u16 v5, v5, v9
4127 ; GFX9-NEXT: v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
4128 ; GFX9-NEXT: v_pk_ashrrev_i16 v10, 12, v10 op_sel_hi:[0,1]
4129 ; GFX9-NEXT: v_pk_mul_lo_u16 v6, v6, v10
4130 ; GFX9-NEXT: s_waitcnt vmcnt(0)
4131 ; GFX9-NEXT: v_add_u32_e32 v2, v3, v2
4132 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4133 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
4134 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4135 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v5
4136 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4137 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v6
4138 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4139 ; GFX9-NEXT: global_store_short v[0:1], v2, off
4140 ; GFX9-NEXT: s_endpgm
4142 ; GFX9-DL-LABEL: idot8_acc16_vecMul:
4143 ; GFX9-DL: ; %bb.0: ; %entry
4144 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4145 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4146 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4147 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
4148 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
4149 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
4150 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
4151 ; GFX9-DL-NEXT: global_load_ushort v2, v[0:1], off
4152 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4153 ; GFX9-DL-NEXT: s_and_b32 s0, s2, 15
4154 ; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004
4155 ; GFX9-DL-NEXT: s_and_b32 s5, s4, 15
4156 ; GFX9-DL-NEXT: s_bfe_u32 s6, s4, 0x40004
4157 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s1
4158 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s5, s6
4159 ; GFX9-DL-NEXT: s_bfe_u32 s5, s2, 0x40008
4160 ; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x4000c
4161 ; GFX9-DL-NEXT: v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
4162 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s5, s6
4163 ; GFX9-DL-NEXT: s_bfe_u32 s9, s2, 0x40010
4164 ; GFX9-DL-NEXT: s_bfe_u32 s10, s2, 0x40014
4165 ; GFX9-DL-NEXT: v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
4166 ; GFX9-DL-NEXT: s_bfe_u32 s13, s2, 0x40018
4167 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s9, s10
4168 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28
4169 ; GFX9-DL-NEXT: v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
4170 ; GFX9-DL-NEXT: v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
4171 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s13, s2
4172 ; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x40008
4173 ; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x4000c
4174 ; GFX9-DL-NEXT: v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
4175 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s7, s8
4176 ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
4177 ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
4178 ; GFX9-DL-NEXT: v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
4179 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v7
4180 ; GFX9-DL-NEXT: s_bfe_u32 s11, s4, 0x40010
4181 ; GFX9-DL-NEXT: s_bfe_u32 s12, s4, 0x40014
4182 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s11, s12
4183 ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
4184 ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
4185 ; GFX9-DL-NEXT: v_pk_lshlrev_b16 v9, 12, s0 op_sel_hi:[0,1]
4186 ; GFX9-DL-NEXT: s_bfe_u32 s14, s4, 0x40018
4187 ; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28
4188 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v8
4189 ; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s14, s4
4190 ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
4191 ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
4192 ; GFX9-DL-NEXT: v_pk_lshlrev_b16 v10, 12, s0 op_sel_hi:[0,1]
4193 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v5, v5, v9
4194 ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
4195 ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v10, 12, v10 op_sel_hi:[0,1]
4196 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v6, v6, v10
4197 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
4198 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v3, v2
4199 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4200 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
4201 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4202 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v5
4203 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4204 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v6
4205 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4206 ; GFX9-DL-NEXT: global_store_short v[0:1], v2, off
4207 ; GFX9-DL-NEXT: s_endpgm
4208 <8 x i4> addrspace(1)* %src2,
4209 i16 addrspace(1)* nocapture %dst) {
4211 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
4212 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
4214 %cvec1 = sext <8 x i4> %vec1 to <8 x i16>
4215 %cvec2 = sext <8 x i4> %vec2 to <8 x i16>
4217 %mul = mul <8 x i16> %cvec1, %cvec2
4218 %mul0 = extractelement <8 x i16> %mul, i64 0
4219 %mul1 = extractelement <8 x i16> %mul, i64 1
4220 %mul2 = extractelement <8 x i16> %mul, i64 2
4221 %mul3 = extractelement <8 x i16> %mul, i64 3
4222 %mul4 = extractelement <8 x i16> %mul, i64 4
4223 %mul5 = extractelement <8 x i16> %mul, i64 5
4224 %mul6 = extractelement <8 x i16> %mul, i64 6
4225 %mul7 = extractelement <8 x i16> %mul, i64 7
4227 %acc = load i16, i16 addrspace(1)* %dst, align 4
4228 %add1 = add i16 %mul0, %acc
4229 %add2 = add i16 %add1, %mul1
4230 %add3 = add i16 %add2, %mul2
4231 %add4 = add i16 %add3, %mul3
4232 %add5 = add i16 %add4, %mul4
4233 %add6 = add i16 %add5, %mul5
4234 %add7 = add i16 %add6, %mul6
4235 %add8 = add i16 %add7, %mul7
4237 store i16 %add8, i16 addrspace(1)* %dst, align 4
4241 ; TODO: Support this pattern.
4242 define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
4243 ; GFX7-LABEL: idot8_acc8_vecMul:
4244 ; GFX7: ; %bb.0: ; %entry
4245 ; GFX7-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9
4246 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
4247 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4248 ; GFX7-NEXT: s_mov_b32 s6, -1
4249 ; GFX7-NEXT: s_movk_i32 s0, 0xff
4250 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4251 ; GFX7-NEXT: s_load_dword s2, s[8:9], 0x0
4252 ; GFX7-NEXT: buffer_load_ubyte v0, off, s[4:7], 0
4253 ; GFX7-NEXT: s_load_dword s8, s[10:11], 0x0
4254 ; GFX7-NEXT: s_mov_b32 s1, 0xffff
4255 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4256 ; GFX7-NEXT: s_bfe_i32 s9, s2, 0x40000
4257 ; GFX7-NEXT: s_bfe_i32 s10, s2, 0x40004
4258 ; GFX7-NEXT: s_bfe_i32 s16, s8, 0x40000
4259 ; GFX7-NEXT: s_bfe_i32 s17, s8, 0x40004
4260 ; GFX7-NEXT: s_bfe_i32 s18, s8, 0x40008
4261 ; GFX7-NEXT: s_bfe_i32 s19, s8, 0x4000c
4262 ; GFX7-NEXT: s_bfe_i32 s20, s8, 0x40010
4263 ; GFX7-NEXT: s_bfe_i32 s21, s8, 0x40014
4264 ; GFX7-NEXT: s_bfe_i32 s22, s8, 0x40018
4265 ; GFX7-NEXT: s_ashr_i32 s8, s8, 28
4266 ; GFX7-NEXT: v_mov_b32_e32 v7, s17
4267 ; GFX7-NEXT: v_mov_b32_e32 v8, s16
4268 ; GFX7-NEXT: s_bfe_i32 s11, s2, 0x40008
4269 ; GFX7-NEXT: v_mov_b32_e32 v6, s18
4270 ; GFX7-NEXT: s_bfe_i32 s12, s2, 0x4000c
4271 ; GFX7-NEXT: v_mov_b32_e32 v5, s19
4272 ; GFX7-NEXT: s_bfe_i32 s13, s2, 0x40010
4273 ; GFX7-NEXT: v_mov_b32_e32 v4, s20
4274 ; GFX7-NEXT: s_bfe_i32 s14, s2, 0x40014
4275 ; GFX7-NEXT: v_mov_b32_e32 v3, s21
4276 ; GFX7-NEXT: s_bfe_i32 s15, s2, 0x40018
4277 ; GFX7-NEXT: v_mov_b32_e32 v2, s22
4278 ; GFX7-NEXT: s_ashr_i32 s2, s2, 28
4279 ; GFX7-NEXT: v_mov_b32_e32 v1, s8
4280 ; GFX7-NEXT: v_mul_i32_i24_e32 v1, s2, v1
4281 ; GFX7-NEXT: v_mul_i32_i24_e32 v2, s15, v2
4282 ; GFX7-NEXT: v_mul_i32_i24_e32 v3, s14, v3
4283 ; GFX7-NEXT: v_mul_i32_i24_e32 v4, s13, v4
4284 ; GFX7-NEXT: v_mul_i32_i24_e32 v5, s12, v5
4285 ; GFX7-NEXT: v_mul_i32_i24_e32 v6, s11, v6
4286 ; GFX7-NEXT: v_mul_i32_i24_e32 v7, s10, v7
4287 ; GFX7-NEXT: v_mul_i32_i24_e32 v8, s9, v8
4288 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
4289 ; GFX7-NEXT: v_and_b32_e32 v2, s0, v2
4290 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
4291 ; GFX7-NEXT: v_and_b32_e32 v4, s0, v4
4292 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v5
4293 ; GFX7-NEXT: v_and_b32_e32 v6, s0, v6
4294 ; GFX7-NEXT: v_lshlrev_b32_e32 v7, 8, v7
4295 ; GFX7-NEXT: v_and_b32_e32 v8, s0, v8
4296 ; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
4297 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v3
4298 ; GFX7-NEXT: v_or_b32_e32 v3, v6, v5
4299 ; GFX7-NEXT: v_or_b32_e32 v4, v8, v7
4300 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
4301 ; GFX7-NEXT: v_and_b32_e32 v2, s1, v2
4302 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
4303 ; GFX7-NEXT: v_and_b32_e32 v4, s1, v4
4304 ; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
4305 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v3
4306 ; GFX7-NEXT: v_alignbit_b32 v3, v1, v2, 8
4307 ; GFX7-NEXT: v_alignbit_b32 v4, v1, v2, 16
4308 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v2
4309 ; GFX7-NEXT: v_lshrrev_b32_e32 v6, 8, v1
4310 ; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v1
4311 ; GFX7-NEXT: v_lshrrev_b32_e32 v8, 24, v1
4312 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4313 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
4314 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0
4315 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0
4316 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v5, v0
4317 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
4318 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v6, v0
4319 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v7, v0
4320 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v8, v0
4321 ; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0
4322 ; GFX7-NEXT: s_endpgm
4324 ; GFX8-LABEL: idot8_acc8_vecMul:
4325 ; GFX8: ; %bb.0: ; %entry
4326 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4327 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4328 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4329 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
4330 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0
4331 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
4332 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
4333 ; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
4334 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4335 ; GFX8-NEXT: s_lshr_b32 s0, s2, 4
4336 ; GFX8-NEXT: s_lshr_b32 s1, s2, 12
4337 ; GFX8-NEXT: s_lshr_b32 s5, s2, 8
4338 ; GFX8-NEXT: s_lshr_b32 s6, s4, 4
4339 ; GFX8-NEXT: s_lshr_b32 s7, s4, 12
4340 ; GFX8-NEXT: s_lshr_b32 s8, s4, 8
4341 ; GFX8-NEXT: v_lshlrev_b16_e64 v3, 12, s5
4342 ; GFX8-NEXT: v_lshlrev_b16_e64 v4, 12, s1
4343 ; GFX8-NEXT: v_lshlrev_b16_e64 v5, 12, s0
4344 ; GFX8-NEXT: v_lshlrev_b16_e64 v7, 12, s8
4345 ; GFX8-NEXT: v_lshlrev_b16_e64 v8, 12, s7
4346 ; GFX8-NEXT: v_lshlrev_b16_e64 v9, 12, s6
4347 ; GFX8-NEXT: v_lshlrev_b16_e64 v6, 12, s2
4348 ; GFX8-NEXT: v_lshlrev_b16_e64 v10, 12, s4
4349 ; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
4350 ; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
4351 ; GFX8-NEXT: v_ashrrev_i16_e32 v4, 12, v4
4352 ; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
4353 ; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v5
4354 ; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
4355 ; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
4356 ; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
4357 ; GFX8-NEXT: v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
4358 ; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
4359 ; GFX8-NEXT: v_mul_u32_u24_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
4360 ; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v6, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
4361 ; GFX8-NEXT: s_lshr_b32 s0, s2, 20
4362 ; GFX8-NEXT: s_lshr_b32 s1, s2, 16
4363 ; GFX8-NEXT: s_lshr_b32 s5, s2, 28
4364 ; GFX8-NEXT: s_lshr_b32 s2, s2, 24
4365 ; GFX8-NEXT: s_lshr_b32 s6, s4, 20
4366 ; GFX8-NEXT: s_lshr_b32 s7, s4, 16
4367 ; GFX8-NEXT: s_lshr_b32 s8, s4, 28
4368 ; GFX8-NEXT: s_lshr_b32 s4, s4, 24
4369 ; GFX8-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4370 ; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4371 ; GFX8-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
4372 ; GFX8-NEXT: v_lshlrev_b16_e64 v7, 12, s2
4373 ; GFX8-NEXT: v_lshlrev_b16_e64 v8, 12, s5
4374 ; GFX8-NEXT: v_lshlrev_b16_e64 v9, 12, s1
4375 ; GFX8-NEXT: v_lshlrev_b16_e64 v10, 12, s0
4376 ; GFX8-NEXT: v_lshlrev_b16_e64 v11, 12, s4
4377 ; GFX8-NEXT: v_lshlrev_b16_e64 v12, 12, s8
4378 ; GFX8-NEXT: v_lshlrev_b16_e64 v13, 12, s7
4379 ; GFX8-NEXT: v_lshlrev_b16_e64 v14, 12, s6
4380 ; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
4381 ; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v11
4382 ; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
4383 ; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
4384 ; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
4385 ; GFX8-NEXT: v_ashrrev_i16_e32 v13, 12, v13
4386 ; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
4387 ; GFX8-NEXT: v_ashrrev_i16_e32 v14, 12, v14
4388 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 8, v3
4389 ; GFX8-NEXT: v_mul_u32_u24_sdwa v7, v7, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
4390 ; GFX8-NEXT: v_mul_u32_u24_sdwa v8, v8, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
4391 ; GFX8-NEXT: v_mul_u32_u24_sdwa v9, v9, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
4392 ; GFX8-NEXT: v_mul_u32_u24_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
4393 ; GFX8-NEXT: v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4394 ; GFX8-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4395 ; GFX8-NEXT: v_or_b32_sdwa v4, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
4396 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v4
4397 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4398 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
4399 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v5, v2
4400 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
4401 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
4402 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4
4403 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v6
4404 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4405 ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
4406 ; GFX8-NEXT: flat_store_byte v[0:1], v2
4407 ; GFX8-NEXT: s_endpgm
4409 ; GFX9-LABEL: idot8_acc8_vecMul:
4410 ; GFX9: ; %bb.0: ; %entry
4411 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4412 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4413 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
4414 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
4415 ; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
4416 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
4417 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
4418 ; GFX9-NEXT: global_load_ubyte v2, v[0:1], off
4419 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
4420 ; GFX9-NEXT: s_lshr_b32 s0, s2, 4
4421 ; GFX9-NEXT: s_lshr_b32 s1, s2, 12
4422 ; GFX9-NEXT: s_lshr_b32 s5, s2, 8
4423 ; GFX9-NEXT: s_lshr_b32 s6, s4, 4
4424 ; GFX9-NEXT: s_lshr_b32 s7, s4, 12
4425 ; GFX9-NEXT: s_lshr_b32 s8, s4, 8
4426 ; GFX9-NEXT: v_lshlrev_b16_e64 v3, 12, s5
4427 ; GFX9-NEXT: v_lshlrev_b16_e64 v4, 12, s1
4428 ; GFX9-NEXT: v_lshlrev_b16_e64 v5, 12, s0
4429 ; GFX9-NEXT: v_lshlrev_b16_e64 v7, 12, s8
4430 ; GFX9-NEXT: v_lshlrev_b16_e64 v8, 12, s7
4431 ; GFX9-NEXT: v_lshlrev_b16_e64 v9, 12, s6
4432 ; GFX9-NEXT: v_lshlrev_b16_e64 v6, 12, s2
4433 ; GFX9-NEXT: v_lshlrev_b16_e64 v10, 12, s4
4434 ; GFX9-NEXT: v_ashrrev_i16_e32 v3, 12, v3
4435 ; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
4436 ; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4
4437 ; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
4438 ; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5
4439 ; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v9
4440 ; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
4441 ; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v10
4442 ; GFX9-NEXT: v_mul_lo_u16_e32 v6, v6, v10
4443 ; GFX9-NEXT: v_mul_lo_u16_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4444 ; GFX9-NEXT: v_mul_lo_u16_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4445 ; GFX9-NEXT: v_mul_lo_u16_e32 v3, v3, v7
4446 ; GFX9-NEXT: s_lshr_b32 s0, s2, 20
4447 ; GFX9-NEXT: s_lshr_b32 s1, s2, 16
4448 ; GFX9-NEXT: s_lshr_b32 s5, s2, 28
4449 ; GFX9-NEXT: s_lshr_b32 s2, s2, 24
4450 ; GFX9-NEXT: s_lshr_b32 s6, s4, 20
4451 ; GFX9-NEXT: s_lshr_b32 s7, s4, 16
4452 ; GFX9-NEXT: s_lshr_b32 s8, s4, 28
4453 ; GFX9-NEXT: s_lshr_b32 s4, s4, 24
4454 ; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4455 ; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4456 ; GFX9-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
4457 ; GFX9-NEXT: v_lshlrev_b16_e64 v9, 12, s2
4458 ; GFX9-NEXT: v_lshlrev_b16_e64 v10, 12, s5
4459 ; GFX9-NEXT: v_lshlrev_b16_e64 v11, 12, s1
4460 ; GFX9-NEXT: v_lshlrev_b16_e64 v12, 12, s0
4461 ; GFX9-NEXT: v_lshlrev_b16_e64 v13, 12, s4
4462 ; GFX9-NEXT: v_lshlrev_b16_e64 v14, 12, s8
4463 ; GFX9-NEXT: v_lshlrev_b16_e64 v15, 12, s7
4464 ; GFX9-NEXT: v_lshlrev_b16_e64 v16, 12, s6
4465 ; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v9
4466 ; GFX9-NEXT: v_ashrrev_i16_e32 v13, 12, v13
4467 ; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v10
4468 ; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v14
4469 ; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
4470 ; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v15
4471 ; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
4472 ; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v16
4473 ; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v3
4474 ; GFX9-NEXT: v_mul_lo_u16_sdwa v12, v12, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4475 ; GFX9-NEXT: v_mul_lo_u16_e32 v11, v11, v15
4476 ; GFX9-NEXT: v_mul_lo_u16_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4477 ; GFX9-NEXT: v_mul_lo_u16_e32 v9, v9, v13
4478 ; GFX9-NEXT: v_or_b32_sdwa v7, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4479 ; GFX9-NEXT: v_or_b32_sdwa v8, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4480 ; GFX9-NEXT: v_or_b32_sdwa v4, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
4481 ; GFX9-NEXT: s_waitcnt vmcnt(0)
4482 ; GFX9-NEXT: v_add_u32_e32 v2, v3, v2
4483 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v5
4484 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
4485 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
4486 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v4
4487 ; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v4
4488 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v3
4489 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4490 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
4491 ; GFX9-NEXT: global_store_byte v[0:1], v2, off
4492 ; GFX9-NEXT: s_endpgm
4494 ; GFX9-DL-LABEL: idot8_acc8_vecMul:
4495 ; GFX9-DL: ; %bb.0: ; %entry
4496 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4497 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4498 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4499 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
4500 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0
4501 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
4502 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
4503 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off
4504 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4505 ; GFX9-DL-NEXT: s_lshr_b32 s0, s2, 4
4506 ; GFX9-DL-NEXT: s_lshr_b32 s1, s2, 12
4507 ; GFX9-DL-NEXT: s_lshr_b32 s5, s2, 8
4508 ; GFX9-DL-NEXT: s_lshr_b32 s6, s4, 4
4509 ; GFX9-DL-NEXT: s_lshr_b32 s7, s4, 12
4510 ; GFX9-DL-NEXT: s_lshr_b32 s8, s4, 8
4511 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v3, 12, s5
4512 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v4, 12, s1
4513 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v5, 12, s0
4514 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v7, 12, s8
4515 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v8, 12, s7
4516 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v9, 12, s6
4517 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v6, 12, s2
4518 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v10, 12, s4
4519 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v3, 12, v3
4520 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
4521 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4
4522 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
4523 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5
4524 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v9
4525 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
4526 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v10
4527 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v6, v6, v10
4528 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4529 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4530 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v3, v3, v7
4531 ; GFX9-DL-NEXT: s_lshr_b32 s0, s2, 20
4532 ; GFX9-DL-NEXT: s_lshr_b32 s1, s2, 16
4533 ; GFX9-DL-NEXT: s_lshr_b32 s5, s2, 28
4534 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 24
4535 ; GFX9-DL-NEXT: s_lshr_b32 s6, s4, 20
4536 ; GFX9-DL-NEXT: s_lshr_b32 s7, s4, 16
4537 ; GFX9-DL-NEXT: s_lshr_b32 s8, s4, 28
4538 ; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 24
4539 ; GFX9-DL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4540 ; GFX9-DL-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4541 ; GFX9-DL-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
4542 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v9, 12, s2
4543 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v10, 12, s5
4544 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v11, 12, s1
4545 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v12, 12, s0
4546 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v13, 12, s4
4547 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v14, 12, s8
4548 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v15, 12, s7
4549 ; GFX9-DL-NEXT: v_lshlrev_b16_e64 v16, 12, s6
4550 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v9
4551 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v13, 12, v13
4552 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v10
4553 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v14
4554 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
4555 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v15
4556 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
4557 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v16
4558 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
4559 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v12, v12, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4560 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v11, v11, v15
4561 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4562 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v9, v9, v13
4563 ; GFX9-DL-NEXT: v_or_b32_sdwa v7, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4564 ; GFX9-DL-NEXT: v_or_b32_sdwa v8, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4565 ; GFX9-DL-NEXT: v_or_b32_sdwa v4, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
4566 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
4567 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v3, v2
4568 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v5
4569 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
4570 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
4571 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v4
4572 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v4
4573 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v3
4574 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4575 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
4576 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off
4577 ; GFX9-DL-NEXT: s_endpgm
4578 <8 x i4> addrspace(1)* %src2,
4579 i8 addrspace(1)* nocapture %dst) {
4581 %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
4582 %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
4584 %cvec1 = sext <8 x i4> %vec1 to <8 x i8>
4585 %cvec2 = sext <8 x i4> %vec2 to <8 x i8>
4587 %mul = mul <8 x i8> %cvec1, %cvec2
4588 %mul0 = extractelement <8 x i8> %mul, i64 0
4589 %mul1 = extractelement <8 x i8> %mul, i64 1
4590 %mul2 = extractelement <8 x i8> %mul, i64 2
4591 %mul3 = extractelement <8 x i8> %mul, i64 3
4592 %mul4 = extractelement <8 x i8> %mul, i64 4
4593 %mul5 = extractelement <8 x i8> %mul, i64 5
4594 %mul6 = extractelement <8 x i8> %mul, i64 6
4595 %mul7 = extractelement <8 x i8> %mul, i64 7
4597 %acc = load i8, i8 addrspace(1)* %dst, align 4
4598 %add1 = add i8 %mul0, %acc
4599 %add2 = add i8 %add1, %mul1
4600 %add3 = add i8 %add2, %mul2
4601 %add4 = add i8 %add3, %mul3
4602 %add5 = add i8 %add4, %mul4
4603 %add6 = add i8 %add5, %mul5
4604 %add7 = add i8 %add6, %mul6
4605 %add8 = add i8 %add7, %mul7
4607 store i8 %add8, i8 addrspace(1)* %dst, align 4
4611 define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
4612 ; GFX7-LABEL: udot8_variant1:
4613 ; GFX7: ; %bb.0: ; %entry
4614 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
4615 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
4616 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
4617 ; GFX7-NEXT: s_mov_b32 s2, -1
4618 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4619 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
4620 ; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
4621 ; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
4622 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4623 ; GFX7-NEXT: s_and_b32 s7, s4, 15
4624 ; GFX7-NEXT: s_and_b32 s8, s5, 15
4625 ; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40004
4626 ; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008
4627 ; GFX7-NEXT: s_bfe_u32 s13, s4, 0x4000c
4628 ; GFX7-NEXT: s_bfe_u32 s15, s4, 0x40010
4629 ; GFX7-NEXT: s_bfe_u32 s17, s4, 0x40014
4630 ; GFX7-NEXT: s_bfe_u32 s19, s4, 0x40018
4631 ; GFX7-NEXT: s_lshr_b32 s4, s4, 28
4632 ; GFX7-NEXT: v_mov_b32_e32 v0, s7
4633 ; GFX7-NEXT: v_mov_b32_e32 v1, s6
4634 ; GFX7-NEXT: v_mad_u32_u24 v0, s8, v0, v1
4635 ; GFX7-NEXT: s_bfe_u32 s10, s5, 0x40004
4636 ; GFX7-NEXT: s_bfe_u32 s12, s5, 0x40008
4637 ; GFX7-NEXT: s_bfe_u32 s14, s5, 0x4000c
4638 ; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010
4639 ; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40014
4640 ; GFX7-NEXT: s_bfe_u32 s20, s5, 0x40018
4641 ; GFX7-NEXT: s_lshr_b32 s5, s5, 28
4642 ; GFX7-NEXT: v_mov_b32_e32 v1, s4
4643 ; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
4644 ; GFX7-NEXT: v_mov_b32_e32 v1, s9
4645 ; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
4646 ; GFX7-NEXT: v_mov_b32_e32 v1, s11
4647 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v1, v0
4648 ; GFX7-NEXT: v_mov_b32_e32 v1, s13
4649 ; GFX7-NEXT: v_mad_u32_u24 v0, s14, v1, v0
4650 ; GFX7-NEXT: v_mov_b32_e32 v1, s15
4651 ; GFX7-NEXT: v_mad_u32_u24 v0, s16, v1, v0
4652 ; GFX7-NEXT: v_mov_b32_e32 v1, s17
4653 ; GFX7-NEXT: v_mad_u32_u24 v0, s18, v1, v0
4654 ; GFX7-NEXT: v_mov_b32_e32 v1, s19
4655 ; GFX7-NEXT: v_mad_u32_u24 v0, s20, v1, v0
4656 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
4657 ; GFX7-NEXT: s_endpgm
4659 ; GFX8-LABEL: udot8_variant1:
4660 ; GFX8: ; %bb.0: ; %entry
4661 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4662 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4663 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4664 ; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
4665 ; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
4666 ; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
4667 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
4668 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
4669 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4670 ; GFX8-NEXT: s_and_b32 s0, s2, 15
4671 ; GFX8-NEXT: s_and_b32 s1, s3, 15
4672 ; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40004
4673 ; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40008
4674 ; GFX8-NEXT: s_bfe_u32 s9, s2, 0x4000c
4675 ; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40010
4676 ; GFX8-NEXT: s_bfe_u32 s13, s2, 0x40014
4677 ; GFX8-NEXT: s_bfe_u32 s15, s2, 0x40018
4678 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28
4679 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
4680 ; GFX8-NEXT: v_mov_b32_e32 v3, s4
4681 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v2, v3
4682 ; GFX8-NEXT: s_bfe_u32 s6, s3, 0x40004
4683 ; GFX8-NEXT: s_bfe_u32 s8, s3, 0x40008
4684 ; GFX8-NEXT: s_bfe_u32 s10, s3, 0x4000c
4685 ; GFX8-NEXT: s_bfe_u32 s12, s3, 0x40010
4686 ; GFX8-NEXT: s_bfe_u32 s14, s3, 0x40014
4687 ; GFX8-NEXT: s_bfe_u32 s16, s3, 0x40018
4688 ; GFX8-NEXT: s_lshr_b32 s3, s3, 28
4689 ; GFX8-NEXT: v_mov_b32_e32 v3, s2
4690 ; GFX8-NEXT: v_mad_u32_u24 v2, s3, v3, v2
4691 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
4692 ; GFX8-NEXT: v_mad_u32_u24 v2, s6, v3, v2
4693 ; GFX8-NEXT: v_mov_b32_e32 v3, s7
4694 ; GFX8-NEXT: v_mad_u32_u24 v2, s8, v3, v2
4695 ; GFX8-NEXT: v_mov_b32_e32 v3, s9
4696 ; GFX8-NEXT: v_mad_u32_u24 v2, s10, v3, v2
4697 ; GFX8-NEXT: v_mov_b32_e32 v3, s11
4698 ; GFX8-NEXT: v_mad_u32_u24 v2, s12, v3, v2
4699 ; GFX8-NEXT: v_mov_b32_e32 v3, s13
4700 ; GFX8-NEXT: v_mad_u32_u24 v2, s14, v3, v2
4701 ; GFX8-NEXT: v_mov_b32_e32 v3, s15
4702 ; GFX8-NEXT: v_mad_u32_u24 v2, s16, v3, v2
4703 ; GFX8-NEXT: flat_store_dword v[0:1], v2
4704 ; GFX8-NEXT: s_endpgm
4706 ; GFX9-LABEL: udot8_variant1:
4707 ; GFX9: ; %bb.0: ; %entry
4708 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4709 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4710 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
4711 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x0
4712 ; GFX9-NEXT: s_load_dword s3, s[6:7], 0x0
4713 ; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
4714 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
4715 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
4716 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
4717 ; GFX9-NEXT: s_and_b32 s0, s2, 15
4718 ; GFX9-NEXT: s_and_b32 s1, s3, 15
4719 ; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40004
4720 ; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40008
4721 ; GFX9-NEXT: s_bfe_u32 s9, s2, 0x4000c
4722 ; GFX9-NEXT: s_bfe_u32 s11, s2, 0x40010
4723 ; GFX9-NEXT: s_bfe_u32 s13, s2, 0x40014
4724 ; GFX9-NEXT: s_bfe_u32 s15, s2, 0x40018
4725 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28
4726 ; GFX9-NEXT: v_mov_b32_e32 v2, s0
4727 ; GFX9-NEXT: v_mov_b32_e32 v3, s4
4728 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v2, v3
4729 ; GFX9-NEXT: s_bfe_u32 s6, s3, 0x40004
4730 ; GFX9-NEXT: s_bfe_u32 s8, s3, 0x40008
4731 ; GFX9-NEXT: s_bfe_u32 s10, s3, 0x4000c
4732 ; GFX9-NEXT: s_bfe_u32 s12, s3, 0x40010
4733 ; GFX9-NEXT: s_bfe_u32 s14, s3, 0x40014
4734 ; GFX9-NEXT: s_bfe_u32 s16, s3, 0x40018
4735 ; GFX9-NEXT: s_lshr_b32 s3, s3, 28
4736 ; GFX9-NEXT: v_mov_b32_e32 v3, s2
4737 ; GFX9-NEXT: v_mad_u32_u24 v2, s3, v3, v2
4738 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
4739 ; GFX9-NEXT: v_mad_u32_u24 v2, s6, v3, v2
4740 ; GFX9-NEXT: v_mov_b32_e32 v3, s7
4741 ; GFX9-NEXT: v_mad_u32_u24 v2, s8, v3, v2
4742 ; GFX9-NEXT: v_mov_b32_e32 v3, s9
4743 ; GFX9-NEXT: v_mad_u32_u24 v2, s10, v3, v2
4744 ; GFX9-NEXT: v_mov_b32_e32 v3, s11
4745 ; GFX9-NEXT: v_mad_u32_u24 v2, s12, v3, v2
4746 ; GFX9-NEXT: v_mov_b32_e32 v3, s13
4747 ; GFX9-NEXT: v_mad_u32_u24 v2, s14, v3, v2
4748 ; GFX9-NEXT: v_mov_b32_e32 v3, s15
4749 ; GFX9-NEXT: v_mad_u32_u24 v2, s16, v3, v2
4750 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
4751 ; GFX9-NEXT: s_endpgm
4753 ; GFX9-DL-LABEL: udot8_variant1:
4754 ; GFX9-DL: ; %bb.0: ; %entry
4755 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4756 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4757 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4758 ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
4759 ; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0
4760 ; GFX9-DL-NEXT: s_load_dword s4, s[0:1], 0x0
4761 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
4762 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
4763 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4764 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, s2
4765 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4
4766 ; GFX9-DL-NEXT: v_dot8_u32_u4 v2, s3, v2, v3
4767 ; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off
4768 ; GFX9-DL-NEXT: s_endpgm
4769 i32 addrspace(1)* %v2addr,
4770 i32 addrspace(1)* %dst) {
4772 %v1 = load i32, i32 addrspace(1)* %v1addr, align 4
4773 %v2 = load i32, i32 addrspace(1)* %v2addr, align 4
4774 %and = and i32 %v1, 15
4775 %and1 = and i32 %v2, 15
4776 %mul1 = mul nuw nsw i32 %and1, %and
4778 %shr = lshr i32 %v1, 4
4779 %and2 = and i32 %shr, 15
4780 %shr3 = lshr i32 %v2, 4
4781 %and4 = and i32 %shr3, 15
4782 %mul2 = mul nuw nsw i32 %and4, %and2
4784 %shr6 = lshr i32 %v1, 8
4785 %and7 = and i32 %shr6, 15
4786 %shr8 = lshr i32 %v2, 8
4787 %and9 = and i32 %shr8, 15
4788 %mul3 = mul nuw nsw i32 %and9, %and7
4790 %shr12 = lshr i32 %v1, 12
4791 %and13 = and i32 %shr12, 15
4792 %shr14 = lshr i32 %v2, 12
4793 %and15 = and i32 %shr14, 15
4794 %mul4 = mul nuw nsw i32 %and15, %and13
4796 %shr18 = lshr i32 %v1, 16
4797 %and19 = and i32 %shr18, 15
4798 %shr20 = lshr i32 %v2, 16
4799 %and21 = and i32 %shr20, 15
4800 %mul5 = mul nuw nsw i32 %and21, %and19
4802 %shr24 = lshr i32 %v1, 20
4803 %and25 = and i32 %shr24, 15
4804 %shr26 = lshr i32 %v2, 20
4805 %and27 = and i32 %shr26, 15
4806 %mul6 = mul nuw nsw i32 %and27, %and25
4808 %shr30 = lshr i32 %v1, 24
4809 %and31 = and i32 %shr30, 15
4810 %shr32 = lshr i32 %v2, 24
4811 %and33 = and i32 %shr32, 15
4812 %mul7 = mul nuw nsw i32 %and33, %and31
4814 %shr36 = lshr i32 %v1, 28
4815 %shr37 = lshr i32 %v2, 28
4816 %mul8 = mul nuw nsw i32 %shr37, %shr36
4817 %acc = load i32, i32 addrspace(1)* %dst, align 4
4819 %add1 = add i32 %mul1, %acc
4820 %add2 = add i32 %add1, %mul8
4821 %add3 = add i32 %add2, %mul2
4822 %add4 = add i32 %add3, %mul3
4823 %add5 = add i32 %add4, %mul4
4824 %add6 = add i32 %add5, %mul5
4825 %add7 = add i32 %add6, %mul6
4826 %add8 = add i32 %add7, %mul7
4827 store i32 %add8, i32 addrspace(1)* %dst, align 4
projects / FreeBSD / FreeBSD.git / blob