sys/dev/ena/ena_rss.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  *
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29  *
  30  */
  31
  32 #include <sys/cdefs.h>
  33 #include "opt_rss.h"
  34
  35 #include "ena_rss.h"
  36
  37 /*
  38  * This function should generate unique key for the whole driver.
  39  * If the key was already genereated in the previous call (for example
  40  * for another adapter), then it should be returned instead.
  41  */
  42 void
  43 ena_rss_key_fill(void *key, size_t size)
  44 {
  45         static bool key_generated;
  46         static uint8_t default_key[ENA_HASH_KEY_SIZE];
  47
  48         KASSERT(size <= ENA_HASH_KEY_SIZE,
  49             ("Requested more bytes than ENA RSS key can hold"));
  50
  51         if (!key_generated) {
  52                 arc4random_buf(default_key, ENA_HASH_KEY_SIZE);
  53                 key_generated = true;
  54         }
  55
  56         memcpy(key, default_key, size);
  57 }
  58
  59 /*
  60  * ENA HW expects the key to be in reverse-byte order.
  61  */
  62 static void
  63 ena_rss_reorder_hash_key(u8 *reordered_key, const u8 *key, size_t key_size)
  64 {
  65         int i;
  66
  67         key = key + key_size - 1;
  68
  69         for (i = 0; i < key_size; ++i)
  70                 *reordered_key++ = *key--;
  71 }
  72
  73 int
  74 ena_rss_set_hash(struct ena_com_dev *ena_dev, const u8 *key)
  75 {
  76         enum ena_admin_hash_functions ena_func = ENA_ADMIN_TOEPLITZ;
  77         u8 hw_key[ENA_HASH_KEY_SIZE];
  78
  79         ena_rss_reorder_hash_key(hw_key, key, ENA_HASH_KEY_SIZE);
  80
  81         return (ena_com_fill_hash_function(ena_dev, ena_func, hw_key,
  82             ENA_HASH_KEY_SIZE, 0x0));
  83 }
  84
  85 int
  86 ena_rss_get_hash_key(struct ena_com_dev *ena_dev, u8 *key)
  87 {
  88         u8 hw_key[ENA_HASH_KEY_SIZE];
  89         int rc;
  90
  91         rc = ena_com_get_hash_key(ena_dev, hw_key);
  92         if (rc != 0)
  93                 return rc;
  94
  95         ena_rss_reorder_hash_key(key, hw_key, ENA_HASH_KEY_SIZE);
  96
  97         return (0);
  98 }
  99
 100 static int
 101 ena_rss_init_default(struct ena_adapter *adapter)
 102 {
 103         struct ena_com_dev *ena_dev = adapter->ena_dev;
 104         device_t dev = adapter->pdev;
 105         int qid, rc, i;
 106
 107         rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
 108         if (unlikely(rc != 0)) {
 109                 ena_log(dev, ERR, "Cannot init indirect table\n");
 110                 return (rc);
 111         }
 112
 113         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
 114 #ifdef RSS
 115                 qid = rss_get_indirection_to_bucket(i) % adapter->num_io_queues;
 116 #else
 117                 qid = i % adapter->num_io_queues;
 118 #endif
 119                 rc = ena_com_indirect_table_fill_entry(ena_dev, i,
 120                     ENA_IO_RXQ_IDX(qid));
 121                 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
 122                         ena_log(dev, ERR, "Cannot fill indirect table\n");
 123                         goto err_rss_destroy;
 124                 }
 125         }
 126
 127
 128 #ifdef RSS
 129         uint8_t rss_algo = rss_gethashalgo();
 130         if (rss_algo == RSS_HASH_TOEPLITZ) {
 131                 uint8_t hash_key[RSS_KEYSIZE];
 132
 133                 rss_getkey(hash_key);
 134                 rc = ena_rss_set_hash(ena_dev, hash_key);
 135         } else
 136 #endif
 137                 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ,
 138                     NULL, ENA_HASH_KEY_SIZE, 0x0);
 139         if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
 140                 ena_log(dev, ERR, "Cannot fill hash function\n");
 141                 goto err_rss_destroy;
 142         }
 143
 144         rc = ena_com_set_default_hash_ctrl(ena_dev);
 145         if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
 146                 ena_log(dev, ERR, "Cannot fill hash control\n");
 147                 goto err_rss_destroy;
 148         }
 149
 150         rc = ena_rss_indir_init(adapter);
 151
 152         return (rc == EOPNOTSUPP ? 0 : rc);
 153
 154 err_rss_destroy:
 155         ena_com_rss_destroy(ena_dev);
 156         return (rc);
 157 }
 158
 159 /* Configure the Rx forwarding */
 160 int
 161 ena_rss_configure(struct ena_adapter *adapter)
 162 {
 163         struct ena_com_dev *ena_dev = adapter->ena_dev;
 164         int rc;
 165
 166         /* In case the RSS table was destroyed */
 167         if (!ena_dev->rss.tbl_log_size) {
 168                 rc = ena_rss_init_default(adapter);
 169                 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
 170                         ena_log(adapter->pdev, ERR,
 171                             "WARNING: RSS was not properly re-initialized,"
 172                             " it will affect bandwidth\n");
 173                         ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
 174                         return (rc);
 175                 }
 176         }
 177
 178         /* Set indirect table */
 179         rc = ena_com_indirect_table_set(ena_dev);
 180         if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
 181                 return (rc);
 182
 183         /* Configure hash function (if supported) */
 184         rc = ena_com_set_hash_function(ena_dev);
 185         if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
 186                 return (rc);
 187
 188         /* Configure hash inputs (if supported) */
 189         rc = ena_com_set_hash_ctrl(ena_dev);
 190         if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
 191                 return (rc);
 192
 193         return (0);
 194 }
 195
 196 static void
 197 ena_rss_init_default_deferred(void *arg)
 198 {
 199         struct ena_adapter *adapter;
 200         devclass_t dc;
 201         int max;
 202         int rc;
 203
 204         dc = devclass_find("ena");
 205         if (unlikely(dc == NULL)) {
 206                 ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__);
 207                 return;
 208         }
 209
 210         max = devclass_get_maxunit(dc);
 211         while (max-- >= 0) {
 212                 adapter = devclass_get_softc(dc, max);
 213                 if (adapter != NULL) {
 214                         rc = ena_rss_init_default(adapter);
 215                         ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
 216                         if (unlikely(rc != 0)) {
 217                                 ena_log(adapter->pdev, WARN,
 218                                     "WARNING: RSS was not properly initialized,"
 219                                     " it will affect bandwidth\n");
 220                                 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE,
 221                                     adapter);
 222                         }
 223                 }
 224         }
 225 }
 226 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND,
 227     ena_rss_init_default_deferred, NULL);
 228
 229 int
 230 ena_rss_indir_get(struct ena_adapter *adapter, uint32_t *table)
 231 {
 232         int rc, i;
 233
 234         rc = ena_com_indirect_table_get(adapter->ena_dev, table);
 235         if (rc != 0) {
 236                 if (rc == EOPNOTSUPP)
 237                         device_printf(adapter->pdev,
 238                             "Reading from indirection table not supported\n");
 239                 else
 240                         device_printf(adapter->pdev,
 241                             "Unable to get indirection table\n");
 242                 return (rc);
 243         }
 244
 245         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i)
 246                 table[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(table[i]);
 247
 248         return (0);
 249 }
 250
 251 int
 252 ena_rss_indir_set(struct ena_adapter *adapter, uint32_t *table)
 253 {
 254         int rc, i;
 255
 256         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) {
 257                 rc = ena_com_indirect_table_fill_entry(adapter->ena_dev, i,
 258                     ENA_IO_RXQ_IDX(table[i]));
 259                 if (rc != 0) {
 260                         device_printf(adapter->pdev,
 261                             "Cannot fill indirection table entry %d\n", i);
 262                         return (rc);
 263                 }
 264         }
 265
 266         rc = ena_com_indirect_table_set(adapter->ena_dev);
 267         if (rc == EOPNOTSUPP)
 268                 device_printf(adapter->pdev,
 269                     "Writing to indirection table not supported\n");
 270         else if (rc != 0)
 271                 device_printf(adapter->pdev, "Cannot set indirection table\n");
 272
 273         return (rc);
 274 }
 275
 276 int
 277 ena_rss_indir_init(struct ena_adapter *adapter)
 278 {
 279         struct ena_indir *indir = adapter->rss_indir;
 280         int rc;
 281
 282         if (indir == NULL) {
 283                 adapter->rss_indir = indir = malloc(sizeof(struct ena_indir),
 284                     M_DEVBUF, M_WAITOK | M_ZERO);
 285                 if (indir == NULL)
 286                         return (ENOMEM);
 287         }
 288
 289         rc = ena_rss_indir_get(adapter, indir->table);
 290         if (rc != 0) {
 291                 free(adapter->rss_indir, M_DEVBUF);
 292                 adapter->rss_indir = NULL;
 293
 294                 return (rc);
 295         }
 296
 297         ena_rss_copy_indir_buf(indir->sysctl_buf, indir->table);
 298
 299         return (0);
 300 }