sys/compat/linuxkpi/common/include/linux/net_dim.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
   3  *
   4  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
   5  * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  *
  35  * $FreeBSD$
  36  */
  37
  38 /* This file implements Dynamic Interrupt Moderation, DIM */
  39
  40 #ifndef NET_DIM_H
  41 #define NET_DIM_H
  42
  43 #include <asm/types.h>
  44
  45 #include <linux/workqueue.h>
  46 #include <linux/ktime.h>
  47
  48 struct net_dim_cq_moder {
  49         u16     usec;
  50         u16     pkts;
  51         u8      cq_period_mode;
  52 };
  53
  54 struct net_dim_sample {
  55         ktime_t time;
  56         u32     pkt_ctr;
  57         u32     byte_ctr;
  58         u16     event_ctr;
  59 };
  60
  61 struct net_dim_stats {
  62         int     ppms;                   /* packets per msec */
  63         int     bpms;                   /* bytes per msec */
  64         int     epms;                   /* events per msec */
  65 };
  66
  67 struct net_dim {                        /* Adaptive Moderation */
  68         u8      state;
  69         struct net_dim_stats prev_stats;
  70         struct net_dim_sample start_sample;
  71         struct work_struct work;
  72         u16     event_ctr;
  73         u8      profile_ix;
  74         u8      mode;
  75         u8      tune_state;
  76         u8      steps_right;
  77         u8      steps_left;
  78         u8      tired;
  79 };
  80
  81 enum {
  82         NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0,
  83         NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1,
  84         NET_DIM_CQ_PERIOD_NUM_MODES = 0x2,
  85         NET_DIM_CQ_PERIOD_MODE_DISABLED = 0xFF,
  86 };
  87
  88 /* Adaptive moderation logic */
  89 enum {
  90         NET_DIM_START_MEASURE,
  91         NET_DIM_MEASURE_IN_PROGRESS,
  92         NET_DIM_APPLY_NEW_PROFILE,
  93 };
  94
  95 enum {
  96         NET_DIM_PARKING_ON_TOP,
  97         NET_DIM_PARKING_TIRED,
  98         NET_DIM_GOING_RIGHT,
  99         NET_DIM_GOING_LEFT,
 100 };
 101
 102 enum {
 103         NET_DIM_STATS_WORSE,
 104         NET_DIM_STATS_SAME,
 105         NET_DIM_STATS_BETTER,
 106 };
 107
 108 enum {
 109         NET_DIM_STEPPED,
 110         NET_DIM_TOO_TIRED,
 111         NET_DIM_ON_EDGE,
 112 };
 113
 114 #define NET_DIM_PARAMS_NUM_PROFILES 5
 115 /* Adaptive moderation profiles */
 116 #define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
 117 #define NET_DIM_DEF_PROFILE_CQE 1
 118 #define NET_DIM_DEF_PROFILE_EQE 1
 119
 120 /* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */
 121 #define NET_DIM_EQE_PROFILES { \
 122         {1,   NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
 123         {8,   NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
 124         {64,  NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
 125         {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
 126         {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
 127 }
 128
 129 #define NET_DIM_CQE_PROFILES { \
 130         {2,  256},             \
 131         {8,  128},             \
 132         {16, 64},              \
 133         {32, 64},              \
 134         {64, 64}               \
 135 }
 136
 137 static const struct net_dim_cq_moder
 138         net_dim_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
 139         NET_DIM_EQE_PROFILES,
 140         NET_DIM_CQE_PROFILES,
 141 };
 142
 143 static inline struct net_dim_cq_moder
 144 net_dim_get_profile(u8 cq_period_mode,
 145     int ix)
 146 {
 147         struct net_dim_cq_moder cq_moder;
 148
 149         cq_moder = net_dim_profile[cq_period_mode][ix];
 150         cq_moder.cq_period_mode = cq_period_mode;
 151         return cq_moder;
 152 }
 153
 154 static inline struct net_dim_cq_moder
 155 net_dim_get_def_profile(u8 rx_cq_period_mode)
 156 {
 157         int default_profile_ix;
 158
 159         if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE)
 160                 default_profile_ix = NET_DIM_DEF_PROFILE_CQE;
 161         else    /* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */
 162                 default_profile_ix = NET_DIM_DEF_PROFILE_EQE;
 163
 164         return net_dim_get_profile(rx_cq_period_mode, default_profile_ix);
 165 }
 166
 167 static inline bool
 168 net_dim_on_top(struct net_dim *dim)
 169 {
 170         switch (dim->tune_state) {
 171         case NET_DIM_PARKING_ON_TOP:
 172         case NET_DIM_PARKING_TIRED:
 173                 return true;
 174         case NET_DIM_GOING_RIGHT:
 175                 return (dim->steps_left > 1) && (dim->steps_right == 1);
 176         default:        /* NET_DIM_GOING_LEFT */
 177                 return (dim->steps_right > 1) && (dim->steps_left == 1);
 178         }
 179 }
 180
 181 static inline void
 182 net_dim_turn(struct net_dim *dim)
 183 {
 184         switch (dim->tune_state) {
 185         case NET_DIM_PARKING_ON_TOP:
 186         case NET_DIM_PARKING_TIRED:
 187                 break;
 188         case NET_DIM_GOING_RIGHT:
 189                 dim->tune_state = NET_DIM_GOING_LEFT;
 190                 dim->steps_left = 0;
 191                 break;
 192         case NET_DIM_GOING_LEFT:
 193                 dim->tune_state = NET_DIM_GOING_RIGHT;
 194                 dim->steps_right = 0;
 195                 break;
 196         }
 197 }
 198
 199 static inline int
 200 net_dim_step(struct net_dim *dim)
 201 {
 202         if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
 203                 return NET_DIM_TOO_TIRED;
 204
 205         switch (dim->tune_state) {
 206         case NET_DIM_PARKING_ON_TOP:
 207         case NET_DIM_PARKING_TIRED:
 208                 break;
 209         case NET_DIM_GOING_RIGHT:
 210                 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
 211                         return NET_DIM_ON_EDGE;
 212                 dim->profile_ix++;
 213                 dim->steps_right++;
 214                 break;
 215         case NET_DIM_GOING_LEFT:
 216                 if (dim->profile_ix == 0)
 217                         return NET_DIM_ON_EDGE;
 218                 dim->profile_ix--;
 219                 dim->steps_left++;
 220                 break;
 221         }
 222
 223         dim->tired++;
 224         return NET_DIM_STEPPED;
 225 }
 226
 227 static inline void
 228 net_dim_park_on_top(struct net_dim *dim)
 229 {
 230         dim->steps_right = 0;
 231         dim->steps_left = 0;
 232         dim->tired = 0;
 233         dim->tune_state = NET_DIM_PARKING_ON_TOP;
 234 }
 235
 236 static inline void
 237 net_dim_park_tired(struct net_dim *dim)
 238 {
 239         dim->steps_right = 0;
 240         dim->steps_left = 0;
 241         dim->tune_state = NET_DIM_PARKING_TIRED;
 242 }
 243
 244 static inline void
 245 net_dim_exit_parking(struct net_dim *dim)
 246 {
 247         dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT :
 248         NET_DIM_GOING_RIGHT;
 249         net_dim_step(dim);
 250 }
 251
 252 #define IS_SIGNIFICANT_DIFF(val, ref) \
 253         (((100UL * abs((val) - (ref))) / (ref)) > 10)   /* more than 10%
 254                                                          * difference */
 255
 256 static inline int
 257 net_dim_stats_compare(struct net_dim_stats *curr,
 258     struct net_dim_stats *prev)
 259 {
 260         if (!prev->bpms)
 261                 return curr->bpms ? NET_DIM_STATS_BETTER :
 262                 NET_DIM_STATS_SAME;
 263
 264         if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
 265                 return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER :
 266                     NET_DIM_STATS_WORSE;
 267
 268         if (!prev->ppms)
 269                 return curr->ppms ? NET_DIM_STATS_BETTER :
 270                     NET_DIM_STATS_SAME;
 271
 272         if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
 273                 return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER :
 274                     NET_DIM_STATS_WORSE;
 275
 276         if (!prev->epms)
 277                 return NET_DIM_STATS_SAME;
 278
 279         if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
 280                 return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER :
 281                     NET_DIM_STATS_WORSE;
 282
 283         return NET_DIM_STATS_SAME;
 284 }
 285
 286 static inline bool
 287 net_dim_decision(struct net_dim_stats *curr_stats,
 288     struct net_dim *dim)
 289 {
 290         int prev_state = dim->tune_state;
 291         int prev_ix = dim->profile_ix;
 292         int stats_res;
 293         int step_res;
 294
 295         switch (dim->tune_state) {
 296         case NET_DIM_PARKING_ON_TOP:
 297                 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
 298                 if (stats_res != NET_DIM_STATS_SAME)
 299                         net_dim_exit_parking(dim);
 300                 break;
 301
 302         case NET_DIM_PARKING_TIRED:
 303                 dim->tired--;
 304                 if (!dim->tired)
 305                         net_dim_exit_parking(dim);
 306                 break;
 307
 308         case NET_DIM_GOING_RIGHT:
 309         case NET_DIM_GOING_LEFT:
 310                 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
 311                 if (stats_res != NET_DIM_STATS_BETTER)
 312                         net_dim_turn(dim);
 313
 314                 if (net_dim_on_top(dim)) {
 315                         net_dim_park_on_top(dim);
 316                         break;
 317                 }
 318                 step_res = net_dim_step(dim);
 319                 switch (step_res) {
 320                 case NET_DIM_ON_EDGE:
 321                         net_dim_park_on_top(dim);
 322                         break;
 323                 case NET_DIM_TOO_TIRED:
 324                         net_dim_park_tired(dim);
 325                         break;
 326                 }
 327
 328                 break;
 329         }
 330
 331         if ((prev_state != NET_DIM_PARKING_ON_TOP) ||
 332             (dim->tune_state != NET_DIM_PARKING_ON_TOP))
 333                 dim->prev_stats = *curr_stats;
 334
 335         return dim->profile_ix != prev_ix;
 336 }
 337
 338 static inline void
 339 net_dim_sample(u16 event_ctr,
 340     u64 packets,
 341     u64 bytes,
 342     struct net_dim_sample *s)
 343 {
 344         s->time = ktime_get();
 345         s->pkt_ctr = packets;
 346         s->byte_ctr = bytes;
 347         s->event_ctr = event_ctr;
 348 }
 349
 350 #define NET_DIM_NEVENTS 64
 351 #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1))
 352
 353 static inline void
 354 net_dim_calc_stats(struct net_dim_sample *start,
 355     struct net_dim_sample *end,
 356     struct net_dim_stats *curr_stats)
 357 {
 358         /* u32 holds up to 71 minutes, should be enough */
 359         u32 delta_us = ktime_us_delta(end->time, start->time);
 360         u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr);
 361         u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr,
 362             start->byte_ctr);
 363
 364         if (!delta_us)
 365                 return;
 366
 367         curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us);
 368         curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us);
 369         curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC,
 370             delta_us);
 371 }
 372
 373 static inline void
 374 net_dim(struct net_dim *dim,
 375     u64 packets, u64 bytes)
 376 {
 377         struct net_dim_stats curr_stats;
 378         struct net_dim_sample end_sample;
 379         u16 nevents;
 380
 381         dim->event_ctr++;
 382
 383         switch (dim->state) {
 384         case NET_DIM_MEASURE_IN_PROGRESS:
 385                 nevents = BIT_GAP(BITS_PER_TYPE(u16),
 386                     dim->event_ctr,
 387                     dim->start_sample.event_ctr);
 388                 if (nevents < NET_DIM_NEVENTS)
 389                         break;
 390                 net_dim_sample(dim->event_ctr, packets, bytes, &end_sample);
 391                 net_dim_calc_stats(&dim->start_sample, &end_sample,
 392                     &curr_stats);
 393                 if (net_dim_decision(&curr_stats, dim)) {
 394                         dim->state = NET_DIM_APPLY_NEW_PROFILE;
 395                         schedule_work(&dim->work);
 396                         break;
 397                 }
 398                 /* FALLTHROUGH */
 399         case NET_DIM_START_MEASURE:
 400                 net_dim_sample(dim->event_ctr, packets, bytes, &dim->start_sample);
 401                 dim->state = NET_DIM_MEASURE_IN_PROGRESS;
 402                 break;
 403         case NET_DIM_APPLY_NEW_PROFILE:
 404                 break;
 405         default:
 406                 break;
 407         }
 408 }
 409
 410 #endif                                  /* NET_DIM_H */