2 * Implement fast Fletcher4 using superscalar pipelines.
4 * Use regular C code to compute
5 * Fletcher4 in four incremental 64-bit parallel accumulator streams,
6 * and then combine the streams to form the final four checksum words.
7 * This implementation is a derivative of the AVX SIMD implementation by
8 * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c).
10 * Copyright (C) 2016 Romain Dolbeau.
13 * Romain Dolbeau <romain.dolbeau@atos.net>
15 * This software is available to you under a choice of one of two
16 * licenses. You may choose to be licensed under the terms of the GNU
17 * General Public License (GPL) Version 2, available from the file
18 * COPYING in the main directory of this source tree, or the
19 * OpenIB.org BSD license below:
21 * Redistribution and use in source and binary forms, with or
22 * without modification, are permitted provided that the following
25 * - Redistributions of source code must retain the above
26 * copyright notice, this list of conditions and the following
29 * - Redistributions in binary form must reproduce the above
30 * copyright notice, this list of conditions and the following
31 * disclaimer in the documentation and/or other materials
32 * provided with the distribution.
34 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
35 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
36 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
37 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
38 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
39 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
40 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
44 #include <sys/param.h>
45 #include <sys/byteorder.h>
46 #include <sys/spa_checksum.h>
47 #include <sys/string.h>
48 #include <zfs_fletcher.h>
50 ZFS_NO_SANITIZE_UNDEFINED
52 fletcher_4_superscalar4_init(fletcher_4_ctx_t *ctx)
54 memset(ctx->superscalar, 0, 4 * sizeof (zfs_fletcher_superscalar_t));
57 ZFS_NO_SANITIZE_UNDEFINED
59 fletcher_4_superscalar4_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
63 A = ctx->superscalar[0].v[0] + ctx->superscalar[0].v[1] +
64 ctx->superscalar[0].v[2] + ctx->superscalar[0].v[3];
65 B = 0 - ctx->superscalar[0].v[1] - 2 * ctx->superscalar[0].v[2] -
66 3 * ctx->superscalar[0].v[3] + 4 * ctx->superscalar[1].v[0] +
67 4 * ctx->superscalar[1].v[1] + 4 * ctx->superscalar[1].v[2] +
68 4 * ctx->superscalar[1].v[3];
70 C = ctx->superscalar[0].v[2] + 3 * ctx->superscalar[0].v[3] -
71 6 * ctx->superscalar[1].v[0] - 10 * ctx->superscalar[1].v[1] -
72 14 * ctx->superscalar[1].v[2] - 18 * ctx->superscalar[1].v[3] +
73 16 * ctx->superscalar[2].v[0] + 16 * ctx->superscalar[2].v[1] +
74 16 * ctx->superscalar[2].v[2] + 16 * ctx->superscalar[2].v[3];
76 D = 0 - ctx->superscalar[0].v[3] + 4 * ctx->superscalar[1].v[0] +
77 10 * ctx->superscalar[1].v[1] + 20 * ctx->superscalar[1].v[2] +
78 34 * ctx->superscalar[1].v[3] - 48 * ctx->superscalar[2].v[0] -
79 64 * ctx->superscalar[2].v[1] - 80 * ctx->superscalar[2].v[2] -
80 96 * ctx->superscalar[2].v[3] + 64 * ctx->superscalar[3].v[0] +
81 64 * ctx->superscalar[3].v[1] + 64 * ctx->superscalar[3].v[2] +
82 64 * ctx->superscalar[3].v[3];
84 ZIO_SET_CHECKSUM(zcp, A, B, C, D);
87 ZFS_NO_SANITIZE_UNDEFINED
89 fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx,
90 const void *buf, uint64_t size)
92 const uint32_t *ip = buf;
93 const uint32_t *ipend = ip + (size / sizeof (uint32_t));
95 uint64_t a2, b2, c2, d2;
96 uint64_t a3, b3, c3, d3;
97 uint64_t a4, b4, c4, d4;
99 a = ctx->superscalar[0].v[0];
100 b = ctx->superscalar[1].v[0];
101 c = ctx->superscalar[2].v[0];
102 d = ctx->superscalar[3].v[0];
103 a2 = ctx->superscalar[0].v[1];
104 b2 = ctx->superscalar[1].v[1];
105 c2 = ctx->superscalar[2].v[1];
106 d2 = ctx->superscalar[3].v[1];
107 a3 = ctx->superscalar[0].v[2];
108 b3 = ctx->superscalar[1].v[2];
109 c3 = ctx->superscalar[2].v[2];
110 d3 = ctx->superscalar[3].v[2];
111 a4 = ctx->superscalar[0].v[3];
112 b4 = ctx->superscalar[1].v[3];
113 c4 = ctx->superscalar[2].v[3];
114 d4 = ctx->superscalar[3].v[3];
116 for (; ip < ipend; ip += 4) {
135 ctx->superscalar[0].v[0] = a;
136 ctx->superscalar[1].v[0] = b;
137 ctx->superscalar[2].v[0] = c;
138 ctx->superscalar[3].v[0] = d;
139 ctx->superscalar[0].v[1] = a2;
140 ctx->superscalar[1].v[1] = b2;
141 ctx->superscalar[2].v[1] = c2;
142 ctx->superscalar[3].v[1] = d2;
143 ctx->superscalar[0].v[2] = a3;
144 ctx->superscalar[1].v[2] = b3;
145 ctx->superscalar[2].v[2] = c3;
146 ctx->superscalar[3].v[2] = d3;
147 ctx->superscalar[0].v[3] = a4;
148 ctx->superscalar[1].v[3] = b4;
149 ctx->superscalar[2].v[3] = c4;
150 ctx->superscalar[3].v[3] = d4;
153 ZFS_NO_SANITIZE_UNDEFINED
155 fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t *ctx,
156 const void *buf, uint64_t size)
158 const uint32_t *ip = buf;
159 const uint32_t *ipend = ip + (size / sizeof (uint32_t));
161 uint64_t a2, b2, c2, d2;
162 uint64_t a3, b3, c3, d3;
163 uint64_t a4, b4, c4, d4;
165 a = ctx->superscalar[0].v[0];
166 b = ctx->superscalar[1].v[0];
167 c = ctx->superscalar[2].v[0];
168 d = ctx->superscalar[3].v[0];
169 a2 = ctx->superscalar[0].v[1];
170 b2 = ctx->superscalar[1].v[1];
171 c2 = ctx->superscalar[2].v[1];
172 d2 = ctx->superscalar[3].v[1];
173 a3 = ctx->superscalar[0].v[2];
174 b3 = ctx->superscalar[1].v[2];
175 c3 = ctx->superscalar[2].v[2];
176 d3 = ctx->superscalar[3].v[2];
177 a4 = ctx->superscalar[0].v[3];
178 b4 = ctx->superscalar[1].v[3];
179 c4 = ctx->superscalar[2].v[3];
180 d4 = ctx->superscalar[3].v[3];
182 for (; ip < ipend; ip += 4) {
183 a += BSWAP_32(ip[0]);
184 a2 += BSWAP_32(ip[1]);
185 a3 += BSWAP_32(ip[2]);
186 a4 += BSWAP_32(ip[3]);
201 ctx->superscalar[0].v[0] = a;
202 ctx->superscalar[1].v[0] = b;
203 ctx->superscalar[2].v[0] = c;
204 ctx->superscalar[3].v[0] = d;
205 ctx->superscalar[0].v[1] = a2;
206 ctx->superscalar[1].v[1] = b2;
207 ctx->superscalar[2].v[1] = c2;
208 ctx->superscalar[3].v[1] = d2;
209 ctx->superscalar[0].v[2] = a3;
210 ctx->superscalar[1].v[2] = b3;
211 ctx->superscalar[2].v[2] = c3;
212 ctx->superscalar[3].v[2] = d3;
213 ctx->superscalar[0].v[3] = a4;
214 ctx->superscalar[1].v[3] = b4;
215 ctx->superscalar[2].v[3] = c4;
216 ctx->superscalar[3].v[3] = d4;
219 static boolean_t fletcher_4_superscalar4_valid(void)
224 const fletcher_4_ops_t fletcher_4_superscalar4_ops = {
225 .init_native = fletcher_4_superscalar4_init,
226 .compute_native = fletcher_4_superscalar4_native,
227 .fini_native = fletcher_4_superscalar4_fini,
228 .init_byteswap = fletcher_4_superscalar4_init,
229 .compute_byteswap = fletcher_4_superscalar4_byteswap,
230 .fini_byteswap = fletcher_4_superscalar4_fini,
231 .valid = fletcher_4_superscalar4_valid,
232 .name = "superscalar4"