sys/netinet/cc/cc_cdg.c

   1 /*-
   2  * Copyright (c) 2009-2013
   3  *      Swinburne University of Technology, Melbourne, Australia
   4  * All rights reserved.
   5  *
   6  * This software was developed at the Centre for Advanced Internet
   7  * Architectures, Swinburne University of Technology, by David Hayes, made
   8  * possible in part by a gift from The Cisco University Research Program Fund,
   9  * a corporate advised fund of Silicon Valley Community Foundation. Development
  10  * and testing were further assisted by a grant from the FreeBSD Foundation.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in the
  19  *    documentation and/or other materials provided with the distribution.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * CAIA Delay-Gradient (CDG) congestion control algorithm
  36  *
  37  * An implemention of the delay-gradient congestion control algorithm proposed
  38  * in the following paper:
  39  *
  40  * D. A. Hayes and G. Armitage, "Revisiting TCP Congestion Control using Delay
  41  * Gradients", in IFIP Networking, Valencia, Spain, 9-13 May 2011.
  42  *
  43  * Developed as part of the NewTCP research project at Swinburne University of
  44  * Technology's Centre for Advanced Internet Architectures, Melbourne,
  45  * Australia. More details are available at:
  46  *   http://caia.swin.edu.au/urp/newtcp/
  47  */
  48
  49 #include <sys/cdefs.h>
  50 __FBSDID("$FreeBSD$");
  51
  52 #include <sys/param.h>
  53 #include <sys/hhook.h>
  54 #include <sys/kernel.h>
  55 #include <sys/khelp.h>
  56 #include <sys/limits.h>
  57 #include <sys/lock.h>
  58 #include <sys/malloc.h>
  59 #include <sys/module.h>
  60 #include <sys/queue.h>
  61 #include <sys/socket.h>
  62 #include <sys/socketvar.h>
  63 #include <sys/sysctl.h>
  64 #include <sys/systm.h>
  65
  66 #include <net/if.h>
  67 #include <net/vnet.h>
  68
  69 #include <netinet/cc.h>
  70 #include <netinet/tcp_seq.h>
  71 #include <netinet/tcp_timer.h>
  72 #include <netinet/tcp_var.h>
  73
  74 #include <netinet/cc/cc_module.h>
  75
  76 #include <netinet/khelp/h_ertt.h>
  77
  78 #include <vm/uma.h>
  79
  80 #define CDG_VERSION "0.1"
  81
  82 #define CAST_PTR_INT(X) (*((int*)(X)))
  83
  84 /* Private delay-gradient induced congestion control signal. */
  85 #define CC_CDG_DELAY 0x01000000
  86
  87 /* NewReno window deflation factor on loss (as a percentage). */
  88 #define RENO_BETA 50
  89
  90 /* Queue states. */
  91 #define CDG_Q_EMPTY     1
  92 #define CDG_Q_RISING    2
  93 #define CDG_Q_FALLING   3
  94 #define CDG_Q_FULL      4
  95 #define CDG_Q_UNKNOWN   9999
  96
  97 /* Number of bit shifts used in probexp lookup table. */
  98 #define EXP_PREC 15
  99
 100 /* Largest gradient represented in probexp lookup table. */
 101 #define MAXGRAD 5
 102
 103 /*
 104  * Delay Precision Enhance - number of bit shifts used for qtrend related
 105  * integer arithmetic precision.
 106  */
 107 #define D_P_E 7
 108
 109 struct qdiff_sample {
 110         long qdiff;
 111         STAILQ_ENTRY(qdiff_sample) qdiff_lnk;
 112 };
 113
 114 struct cdg {
 115         long max_qtrend;
 116         long min_qtrend;
 117         STAILQ_HEAD(minrtts_head, qdiff_sample) qdiffmin_q;
 118         STAILQ_HEAD(maxrtts_head, qdiff_sample) qdiffmax_q;
 119         long window_incr;
 120         /* rttcount for window increase when in congestion avoidance */
 121         long rtt_count;
 122         /* maximum measured rtt within an rtt period */
 123         int maxrtt_in_rtt;
 124         /* maximum measured rtt within prev rtt period */
 125         int maxrtt_in_prevrtt;
 126         /* minimum measured rtt within an rtt period */
 127         int minrtt_in_rtt;
 128         /* minimum measured rtt within prev rtt period */
 129         int minrtt_in_prevrtt;
 130         /* consecutive congestion episode counter */
 131         uint32_t consec_cong_cnt;
 132         /* when tracking a new reno type loss window */
 133         uint32_t shadow_w;
 134         /* maximum number of samples in the moving average queue */
 135         int sample_q_size;
 136         /* number of samples in the moving average queue */
 137         int num_samples;
 138         /* estimate of the queue state of the path */
 139         int queue_state;
 140 };
 141
 142 /*
 143  * Lookup table for:
 144  *   (1 - exp(-x)) << EXP_PREC, where x = [0,MAXGRAD] in 2^-7 increments
 145  *
 146  * Note: probexp[0] is set to 10 (not 0) as a safety for very low increase
 147  * gradients.
 148  */
 149 static const int probexp[641] = {
 150    10,255,508,759,1008,1255,1501,1744,1985,2225,2463,2698,2932,3165,3395,3624,
 151    3850,4075,4299,4520,4740,4958,5175,5389,5602,5814,6024,6232,6438,6643,6846,
 152    7048,7248,7447,7644,7839,8033,8226,8417,8606,8794,8981,9166,9350,9532,9713,
 153    9892,10070,10247,10422,10596,10769,10940,11110,11278,11445,11611,11776,11939,
 154    12101,12262,12422,12580,12737,12893,13048,13201,13354,13505,13655,13803,13951,
 155    14097,14243,14387,14530,14672,14813,14952,15091,15229,15365,15500,15635,15768,
 156    15900,16032,16162,16291,16419,16547,16673,16798,16922,17046,17168,17289,17410,
 157    17529,17648,17766,17882,17998,18113,18227,18340,18453,18564,18675,18784,18893,
 158    19001,19108,19215,19320,19425,19529,19632,19734,19835,19936,20036,20135,20233,
 159    20331,20427,20523,20619,20713,20807,20900,20993,21084,21175,21265,21355,21444,
 160    21532,21619,21706,21792,21878,21962,22046,22130,22213,22295,22376,22457,22537,
 161    22617,22696,22774,22852,22929,23006,23082,23157,23232,23306,23380,23453,23525,
 162    23597,23669,23739,23810,23879,23949,24017,24085,24153,24220,24286,24352,24418,
 163    24483,24547,24611,24675,24738,24800,24862,24924,24985,25045,25106,25165,25224,
 164    25283,25341,25399,25456,25513,25570,25626,25681,25737,25791,25846,25899,25953,
 165    26006,26059,26111,26163,26214,26265,26316,26366,26416,26465,26514,26563,26611,
 166    26659,26707,26754,26801,26847,26893,26939,26984,27029,27074,27118,27162,27206,
 167    27249,27292,27335,27377,27419,27460,27502,27543,27583,27624,27664,27703,27743,
 168    27782,27821,27859,27897,27935,27973,28010,28047,28084,28121,28157,28193,28228,
 169    28263,28299,28333,28368,28402,28436,28470,28503,28536,28569,28602,28634,28667,
 170    28699,28730,28762,28793,28824,28854,28885,28915,28945,28975,29004,29034,29063,
 171    29092,29120,29149,29177,29205,29232,29260,29287,29314,29341,29368,29394,29421,
 172    29447,29472,29498,29524,29549,29574,29599,29623,29648,29672,29696,29720,29744,
 173    29767,29791,29814,29837,29860,29882,29905,29927,29949,29971,29993,30014,30036,
 174    30057,30078,30099,30120,30141,30161,30181,30201,30221,30241,30261,30280,30300,
 175    30319,30338,30357,30376,30394,30413,30431,30449,30467,30485,30503,30521,30538,
 176    30555,30573,30590,30607,30624,30640,30657,30673,30690,30706,30722,30738,30753,
 177    30769,30785,30800,30815,30831,30846,30861,30876,30890,30905,30919,30934,30948,
 178    30962,30976,30990,31004,31018,31031,31045,31058,31072,31085,31098,31111,31124,
 179    31137,31149,31162,31174,31187,31199,31211,31223,31235,31247,31259,31271,31283,
 180    31294,31306,31317,31328,31339,31351,31362,31373,31383,31394,31405,31416,31426,
 181    31436,31447,31457,31467,31477,31487,31497,31507,31517,31527,31537,31546,31556,
 182    31565,31574,31584,31593,31602,31611,31620,31629,31638,31647,31655,31664,31673,
 183    31681,31690,31698,31706,31715,31723,31731,31739,31747,31755,31763,31771,31778,
 184    31786,31794,31801,31809,31816,31824,31831,31838,31846,31853,31860,31867,31874,
 185    31881,31888,31895,31902,31908,31915,31922,31928,31935,31941,31948,31954,31960,
 186    31967,31973,31979,31985,31991,31997,32003,32009,32015,32021,32027,32033,32038,
 187    32044,32050,32055,32061,32066,32072,32077,32083,32088,32093,32098,32104,32109,
 188    32114,32119,32124,32129,32134,32139,32144,32149,32154,32158,32163,32168,32173,
 189    32177,32182,32186,32191,32195,32200,32204,32209,32213,32217,32222,32226,32230,
 190    32234,32238,32242,32247,32251,32255,32259,32263,32267,32270,32274,32278,32282,
 191    32286,32290,32293,32297,32301,32304,32308,32311,32315,32318,32322,32325,32329,
 192    32332,32336,32339,32342,32346,32349,32352,32356,32359,32362,32365,32368,32371,
 193    32374,32377,32381,32384,32387,32389,32392,32395,32398,32401,32404,32407,32410,
 194    32412,32415,32418,32421,32423,32426,32429,32431,32434,32437,32439,32442,32444,
 195    32447,32449,32452,32454,32457,32459,32461,32464,32466,32469,32471,32473,32476,
 196    32478,32480,32482,32485,32487,32489,32491,32493,32495,32497,32500,32502,32504,
 197    32506,32508,32510,32512,32514,32516,32518,32520,32522,32524,32526,32527,32529,
 198    32531,32533,32535,32537,32538,32540,32542,32544,32545,32547};
 199
 200 static uma_zone_t qdiffsample_zone;
 201
 202 static MALLOC_DEFINE(M_CDG, "cdg data",
 203   "Per connection data required for the CDG congestion control algorithm");
 204
 205 static int ertt_id;
 206
 207 static VNET_DEFINE(uint32_t, cdg_alpha_inc);
 208 static VNET_DEFINE(uint32_t, cdg_beta_delay);
 209 static VNET_DEFINE(uint32_t, cdg_beta_loss);
 210 static VNET_DEFINE(uint32_t, cdg_smoothing_factor);
 211 static VNET_DEFINE(uint32_t, cdg_exp_backoff_scale);
 212 static VNET_DEFINE(uint32_t, cdg_consec_cong);
 213 static VNET_DEFINE(uint32_t, cdg_hold_backoff);
 214 #define V_cdg_alpha_inc         VNET(cdg_alpha_inc)
 215 #define V_cdg_beta_delay        VNET(cdg_beta_delay)
 216 #define V_cdg_beta_loss         VNET(cdg_beta_loss)
 217 #define V_cdg_smoothing_factor  VNET(cdg_smoothing_factor)
 218 #define V_cdg_exp_backoff_scale VNET(cdg_exp_backoff_scale)
 219 #define V_cdg_consec_cong       VNET(cdg_consec_cong)
 220 #define V_cdg_hold_backoff      VNET(cdg_hold_backoff)
 221
 222 /* Function prototypes. */
 223 static int cdg_mod_init(void);
 224 static int cdg_mod_destroy(void);
 225 static void cdg_conn_init(struct cc_var *ccv);
 226 static int cdg_cb_init(struct cc_var *ccv);
 227 static void cdg_cb_destroy(struct cc_var *ccv);
 228 static void cdg_cong_signal(struct cc_var *ccv, uint32_t signal_type);
 229 static void cdg_ack_received(struct cc_var *ccv, uint16_t ack_type);
 230
 231 struct cc_algo cdg_cc_algo = {
 232         .name = "cdg",
 233         .mod_init = cdg_mod_init,
 234         .ack_received = cdg_ack_received,
 235         .cb_destroy = cdg_cb_destroy,
 236         .cb_init = cdg_cb_init,
 237         .conn_init = cdg_conn_init,
 238         .cong_signal = cdg_cong_signal,
 239         .mod_destroy = cdg_mod_destroy
 240 };
 241
 242 /* Vnet created and being initialised. */
 243 static void
 244 cdg_init_vnet(const void *unused __unused)
 245 {
 246
 247         V_cdg_alpha_inc = 0;
 248         V_cdg_beta_delay = 70;
 249         V_cdg_beta_loss = 50;
 250         V_cdg_smoothing_factor = 8;
 251         V_cdg_exp_backoff_scale = 3;
 252         V_cdg_consec_cong = 5;
 253         V_cdg_hold_backoff = 5;
 254 }
 255
 256 static int
 257 cdg_mod_init(void)
 258 {
 259         VNET_ITERATOR_DECL(v);
 260
 261         ertt_id = khelp_get_id("ertt");
 262         if (ertt_id <= 0)
 263                 return (EINVAL);
 264
 265         qdiffsample_zone = uma_zcreate("cdg_qdiffsample",
 266             sizeof(struct qdiff_sample), NULL, NULL, NULL, NULL, 0, 0);
 267
 268         VNET_LIST_RLOCK();
 269         VNET_FOREACH(v) {
 270                 CURVNET_SET(v);
 271                 cdg_init_vnet(NULL);
 272                 CURVNET_RESTORE();
 273         }
 274         VNET_LIST_RUNLOCK();
 275
 276         cdg_cc_algo.post_recovery = newreno_cc_algo.post_recovery;
 277         cdg_cc_algo.after_idle = newreno_cc_algo.after_idle;
 278
 279         return (0);
 280 }
 281
 282 static int
 283 cdg_mod_destroy(void)
 284 {
 285
 286         uma_zdestroy(qdiffsample_zone);
 287         return (0);
 288 }
 289
 290 static int
 291 cdg_cb_init(struct cc_var *ccv)
 292 {
 293         struct cdg *cdg_data;
 294
 295         cdg_data = malloc(sizeof(struct cdg), M_CDG, M_NOWAIT);
 296         if (cdg_data == NULL)
 297                 return (ENOMEM);
 298
 299         cdg_data->shadow_w = 0;
 300         cdg_data->max_qtrend = 0;
 301         cdg_data->min_qtrend = 0;
 302         cdg_data->queue_state = CDG_Q_UNKNOWN;
 303         cdg_data->maxrtt_in_rtt = 0;
 304         cdg_data->maxrtt_in_prevrtt = 0;
 305         cdg_data->minrtt_in_rtt = INT_MAX;
 306         cdg_data->minrtt_in_prevrtt = 0;
 307         cdg_data->window_incr = 0;
 308         cdg_data->rtt_count = 0;
 309         cdg_data->consec_cong_cnt = 0;
 310         cdg_data->sample_q_size = V_cdg_smoothing_factor;
 311         cdg_data->num_samples = 0;
 312         STAILQ_INIT(&cdg_data->qdiffmin_q);
 313         STAILQ_INIT(&cdg_data->qdiffmax_q);
 314
 315         ccv->cc_data = cdg_data;
 316
 317         return (0);
 318 }
 319
 320 static void
 321 cdg_conn_init(struct cc_var *ccv)
 322 {
 323         struct cdg *cdg_data = ccv->cc_data;
 324
 325         /*
 326          * Initialise the shadow_cwnd in case we are competing with loss based
 327          * flows from the start
 328          */
 329         cdg_data->shadow_w = CCV(ccv, snd_cwnd);
 330 }
 331
 332 static void
 333 cdg_cb_destroy(struct cc_var *ccv)
 334 {
 335         struct cdg *cdg_data;
 336         struct qdiff_sample *qds, *qds_n;
 337
 338         cdg_data = ccv->cc_data;
 339
 340         qds = STAILQ_FIRST(&cdg_data->qdiffmin_q);
 341         while (qds != NULL) {
 342                 qds_n = STAILQ_NEXT(qds, qdiff_lnk);
 343                 uma_zfree(qdiffsample_zone,qds);
 344                 qds = qds_n;
 345         }
 346
 347         qds = STAILQ_FIRST(&cdg_data->qdiffmax_q);
 348         while (qds != NULL) {
 349                 qds_n = STAILQ_NEXT(qds, qdiff_lnk);
 350                 uma_zfree(qdiffsample_zone,qds);
 351                 qds = qds_n;
 352         }
 353
 354         free(ccv->cc_data, M_CDG);
 355 }
 356
 357 static int
 358 cdg_beta_handler(SYSCTL_HANDLER_ARGS)
 359 {
 360
 361         if (req->newptr != NULL &&
 362             (CAST_PTR_INT(req->newptr) == 0 || CAST_PTR_INT(req->newptr) > 100))
 363                 return (EINVAL);
 364
 365         return (sysctl_handle_int(oidp, arg1, arg2, req));
 366 }
 367
 368 static int
 369 cdg_exp_backoff_scale_handler(SYSCTL_HANDLER_ARGS)
 370 {
 371
 372         if (req->newptr != NULL && CAST_PTR_INT(req->newptr) < 1)
 373                 return (EINVAL);
 374
 375         return (sysctl_handle_int(oidp, arg1, arg2, req));
 376 }
 377
 378 static inline unsigned long
 379 cdg_window_decrease(struct cc_var *ccv, unsigned long owin, unsigned int beta)
 380 {
 381
 382         return ((ulmin(CCV(ccv, snd_wnd), owin) * beta) / 100);
 383 }
 384
 385 /*
 386  * Window increase function
 387  * This window increase function is independent of the initial window size
 388  * to ensure small window flows are not discriminated against (i.e. fairness).
 389  * It increases at 1pkt/rtt like Reno for alpha_inc rtts, and then 2pkts/rtt for
 390  * the next alpha_inc rtts, etc.
 391  */
 392 static void
 393 cdg_window_increase(struct cc_var *ccv, int new_measurement)
 394 {
 395         struct cdg *cdg_data;
 396         int incr, s_w_incr;
 397
 398         cdg_data = ccv->cc_data;
 399         incr = s_w_incr = 0;
 400
 401         if (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh)) {
 402                 /* Slow start. */
 403                 incr = CCV(ccv, t_maxseg);
 404                 s_w_incr = incr;
 405                 cdg_data->window_incr = cdg_data->rtt_count = 0;
 406         } else {
 407                 /* Congestion avoidance. */
 408                 if (new_measurement) {
 409                         s_w_incr = CCV(ccv, t_maxseg);
 410                         if (V_cdg_alpha_inc == 0) {
 411                                 incr = CCV(ccv, t_maxseg);
 412                         } else {
 413                                 if (++cdg_data->rtt_count >= V_cdg_alpha_inc) {
 414                                         cdg_data->window_incr++;
 415                                         cdg_data->rtt_count = 0;
 416                                 }
 417                                 incr = CCV(ccv, t_maxseg) *
 418                                     cdg_data->window_incr;
 419                         }
 420                 }
 421         }
 422
 423         if (cdg_data->shadow_w > 0)
 424                 cdg_data->shadow_w = ulmin(cdg_data->shadow_w + s_w_incr,
 425                     TCP_MAXWIN << CCV(ccv, snd_scale));
 426
 427         CCV(ccv, snd_cwnd) = ulmin(CCV(ccv, snd_cwnd) + incr,
 428             TCP_MAXWIN << CCV(ccv, snd_scale));
 429 }
 430
 431 static void
 432 cdg_cong_signal(struct cc_var *ccv, uint32_t signal_type)
 433 {
 434         struct cdg *cdg_data = ccv->cc_data;
 435
 436         switch(signal_type) {
 437         case CC_CDG_DELAY:
 438                 CCV(ccv, snd_ssthresh) = cdg_window_decrease(ccv,
 439                     CCV(ccv, snd_cwnd), V_cdg_beta_delay);
 440                 CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh);
 441                 CCV(ccv, snd_recover) = CCV(ccv, snd_max);
 442                 cdg_data->window_incr = cdg_data->rtt_count = 0;
 443                 ENTER_CONGRECOVERY(CCV(ccv, t_flags));
 444                 break;
 445         case CC_NDUPACK:
 446                 /*
 447                  * If already responding to congestion OR we have guessed no
 448                  * queue in the path is full.
 449                  */
 450                 if (IN_CONGRECOVERY(CCV(ccv, t_flags)) ||
 451                     cdg_data->queue_state < CDG_Q_FULL) {
 452                         CCV(ccv, snd_ssthresh) = CCV(ccv, snd_cwnd);
 453                         CCV(ccv, snd_recover) = CCV(ccv, snd_max);
 454                 } else {
 455                         /*
 456                          * Loss is likely to be congestion related. We have
 457                          * inferred a queue full state, so have shadow window
 458                          * react to loss as NewReno would.
 459                          */
 460                         if (cdg_data->shadow_w > 0)
 461                                 cdg_data->shadow_w = cdg_window_decrease(ccv,
 462                                     cdg_data->shadow_w, RENO_BETA);
 463
 464                         CCV(ccv, snd_ssthresh) = ulmax(cdg_data->shadow_w,
 465                             cdg_window_decrease(ccv, CCV(ccv, snd_cwnd),
 466                             V_cdg_beta_loss));
 467
 468                         cdg_data->window_incr = cdg_data->rtt_count = 0;
 469                 }
 470                 ENTER_RECOVERY(CCV(ccv, t_flags));
 471                 break;
 472         default:
 473                 newreno_cc_algo.cong_signal(ccv, signal_type);
 474                 break;
 475         }
 476 }
 477
 478 /*
 479  * Using a negative exponential probabilistic backoff so that sources with
 480  * varying RTTs which share the same link will, on average, have the same
 481  * probability of backoff over time.
 482  *
 483  * Prob_backoff = 1 - exp(-qtrend / V_cdg_exp_backoff_scale), where
 484  * V_cdg_exp_backoff_scale is the average qtrend for the exponential backoff.
 485  */
 486 static inline int
 487 prob_backoff(long qtrend)
 488 {
 489         int backoff, idx, p;
 490
 491         backoff = (qtrend > ((MAXGRAD * V_cdg_exp_backoff_scale) << D_P_E));
 492
 493         if (!backoff) {
 494                 if (V_cdg_exp_backoff_scale > 1)
 495                         idx = (qtrend + V_cdg_exp_backoff_scale / 2) /
 496                             V_cdg_exp_backoff_scale;
 497                 else
 498                         idx = qtrend;
 499
 500                 /* Backoff probability proportional to rate of queue growth. */
 501                 p = (INT_MAX / (1 << EXP_PREC)) * probexp[idx];
 502                 backoff = (random() < p);
 503         }
 504
 505         return (backoff);
 506 }
 507
 508 static inline void
 509 calc_moving_average(struct cdg *cdg_data, long qdiff_max, long qdiff_min)
 510 {
 511         struct qdiff_sample *qds;
 512
 513         ++cdg_data->num_samples;
 514         if (cdg_data->num_samples > cdg_data->sample_q_size) {
 515                 /* Minimum RTT. */
 516                 qds = STAILQ_FIRST(&cdg_data->qdiffmin_q);
 517                 cdg_data->min_qtrend =  cdg_data->min_qtrend +
 518                     (qdiff_min - qds->qdiff) / cdg_data->sample_q_size;
 519                 STAILQ_REMOVE_HEAD(&cdg_data->qdiffmin_q, qdiff_lnk);
 520                 qds->qdiff = qdiff_min;
 521                 STAILQ_INSERT_TAIL(&cdg_data->qdiffmin_q, qds, qdiff_lnk);
 522
 523                 /* Maximum RTT. */
 524                 qds = STAILQ_FIRST(&cdg_data->qdiffmax_q);
 525                 cdg_data->max_qtrend =  cdg_data->max_qtrend +
 526                     (qdiff_max - qds->qdiff) / cdg_data->sample_q_size;
 527                 STAILQ_REMOVE_HEAD(&cdg_data->qdiffmax_q, qdiff_lnk);
 528                 qds->qdiff = qdiff_max;
 529                 STAILQ_INSERT_TAIL(&cdg_data->qdiffmax_q, qds, qdiff_lnk);
 530                 --cdg_data->num_samples;
 531         } else {
 532                 qds = uma_zalloc(qdiffsample_zone, M_NOWAIT);
 533                 if (qds != NULL) {
 534                         cdg_data->min_qtrend = cdg_data->min_qtrend +
 535                             qdiff_min / cdg_data->sample_q_size;
 536                         qds->qdiff = qdiff_min;
 537                         STAILQ_INSERT_TAIL(&cdg_data->qdiffmin_q, qds,
 538                             qdiff_lnk);
 539                 }
 540
 541                 qds = uma_zalloc(qdiffsample_zone, M_NOWAIT);
 542                 if (qds) {
 543                         cdg_data->max_qtrend = cdg_data->max_qtrend +
 544                             qdiff_max / cdg_data->sample_q_size;
 545                         qds->qdiff = qdiff_max;
 546                         STAILQ_INSERT_TAIL(&cdg_data->qdiffmax_q, qds,
 547                             qdiff_lnk);
 548                 }
 549         }
 550 }
 551
 552 static void
 553 cdg_ack_received(struct cc_var *ccv, uint16_t ack_type)
 554 {
 555         struct cdg *cdg_data;
 556         struct ertt *e_t;
 557         long qdiff_max, qdiff_min;
 558         int congestion, new_measurement, slowstart;
 559
 560         cdg_data = ccv->cc_data;
 561         e_t = (struct ertt *)khelp_get_osd(CCV(ccv, osd), ertt_id);
 562         new_measurement = e_t->flags & ERTT_NEW_MEASUREMENT;
 563         congestion = 0;
 564         cdg_data->maxrtt_in_rtt = imax(e_t->rtt, cdg_data->maxrtt_in_rtt);
 565         cdg_data->minrtt_in_rtt = imin(e_t->rtt, cdg_data->minrtt_in_rtt);
 566
 567         if (new_measurement) {
 568                 slowstart = (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh));
 569                 /*
 570                  * Update smoothed gradient measurements. Since we are only
 571                  * using one measurement per RTT, use max or min rtt_in_rtt.
 572                  * This is also less noisy than a sample RTT measurement. Max
 573                  * RTT measurements can have trouble due to OS issues.
 574                  */
 575                 if (cdg_data->maxrtt_in_prevrtt) {
 576                         qdiff_max = ((long)(cdg_data->maxrtt_in_rtt -
 577                             cdg_data->maxrtt_in_prevrtt) << D_P_E );
 578                         qdiff_min = ((long)(cdg_data->minrtt_in_rtt -
 579                             cdg_data->minrtt_in_prevrtt) << D_P_E );
 580
 581                         calc_moving_average(cdg_data, qdiff_max, qdiff_min);
 582
 583                         /* Probabilistic backoff with respect to gradient. */
 584                         if (slowstart && qdiff_min > 0)
 585                                 congestion = prob_backoff(qdiff_min);
 586                         else if (cdg_data->min_qtrend > 0)
 587                                 congestion = prob_backoff(cdg_data->min_qtrend);
 588                         else if (slowstart && qdiff_max > 0)
 589                                 congestion = prob_backoff(qdiff_max);
 590                         else if (cdg_data->max_qtrend > 0)
 591                                 congestion = prob_backoff(cdg_data->max_qtrend);
 592
 593                         /* Update estimate of queue state. */
 594                         if (cdg_data->min_qtrend > 0 &&
 595                             cdg_data->max_qtrend <= 0) {
 596                                 cdg_data->queue_state = CDG_Q_FULL;
 597                         } else if (cdg_data->min_qtrend >= 0 &&
 598                             cdg_data->max_qtrend < 0) {
 599                                 cdg_data->queue_state = CDG_Q_EMPTY;
 600                                 cdg_data->shadow_w = 0;
 601                         } else if (cdg_data->min_qtrend > 0 &&
 602                             cdg_data->max_qtrend > 0) {
 603                                 cdg_data->queue_state = CDG_Q_RISING;
 604                         } else if (cdg_data->min_qtrend < 0 &&
 605                             cdg_data->max_qtrend < 0) {
 606                                 cdg_data->queue_state = CDG_Q_FALLING;
 607                         }
 608
 609                         if (cdg_data->min_qtrend < 0 ||
 610                             cdg_data->max_qtrend < 0)
 611                                 cdg_data->consec_cong_cnt = 0;
 612                 }
 613
 614                 cdg_data->minrtt_in_prevrtt = cdg_data->minrtt_in_rtt;
 615                 cdg_data->minrtt_in_rtt = INT_MAX;
 616                 cdg_data->maxrtt_in_prevrtt = cdg_data->maxrtt_in_rtt;
 617                 cdg_data->maxrtt_in_rtt = 0;
 618                 e_t->flags &= ~ERTT_NEW_MEASUREMENT;
 619         }
 620
 621         if (congestion) {
 622                 cdg_data->consec_cong_cnt++;
 623                 if (!IN_RECOVERY(CCV(ccv, t_flags))) {
 624                         if (cdg_data->consec_cong_cnt <= V_cdg_consec_cong)
 625                                 cdg_cong_signal(ccv, CC_CDG_DELAY);
 626                         else
 627                                 /*
 628                                  * We have been backing off but the queue is not
 629                                  * falling. Assume we are competing with
 630                                  * loss-based flows and don't back off for the
 631                                  * next V_cdg_hold_backoff RTT periods.
 632                                  */
 633                                 if (cdg_data->consec_cong_cnt >=
 634                                     V_cdg_consec_cong + V_cdg_hold_backoff)
 635                                         cdg_data->consec_cong_cnt = 0;
 636
 637                         /* Won't see effect until 2nd RTT. */
 638                         cdg_data->maxrtt_in_prevrtt = 0;
 639                         /*
 640                          * Resync shadow window in case we are competing with a
 641                          * loss based flow
 642                          */
 643                         cdg_data->shadow_w = ulmax(CCV(ccv, snd_cwnd),
 644                             cdg_data->shadow_w);
 645                 }
 646         } else if (ack_type == CC_ACK)
 647                 cdg_window_increase(ccv, new_measurement);
 648 }
 649
 650 /* When a vnet is created and being initialised, init the per-stack CDG vars. */
 651 VNET_SYSINIT(cdg_init_vnet, SI_SUB_PROTO_BEGIN, SI_ORDER_FIRST,
 652     cdg_init_vnet, NULL);
 653
 654 SYSCTL_DECL(_net_inet_tcp_cc_cdg);
 655 SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, cdg, CTLFLAG_RW, NULL,
 656     "CAIA delay-gradient congestion control related settings");
 657
 658 SYSCTL_STRING(_net_inet_tcp_cc_cdg, OID_AUTO, version,
 659     CTLFLAG_RD, CDG_VERSION, sizeof(CDG_VERSION) - 1,
 660     "Current algorithm/implementation version number");
 661
 662 SYSCTL_VNET_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, alpha_inc,
 663     CTLFLAG_RW, &VNET_NAME(cdg_alpha_inc), 0,
 664     "Increment the window increase factor alpha by 1 MSS segment every "
 665     "alpha_inc RTTs during congestion avoidance mode.");
 666
 667 SYSCTL_VNET_PROC(_net_inet_tcp_cc_cdg, OID_AUTO, beta_delay,
 668     CTLTYPE_UINT|CTLFLAG_RW, &VNET_NAME(cdg_beta_delay), 70,
 669     &cdg_beta_handler, "IU",
 670     "Delay-based window decrease factor as a percentage "
 671     "(on delay-based backoff, w = w * beta_delay / 100)");
 672
 673 SYSCTL_VNET_PROC(_net_inet_tcp_cc_cdg, OID_AUTO, beta_loss,
 674     CTLTYPE_UINT|CTLFLAG_RW, &VNET_NAME(cdg_beta_loss), 50,
 675     &cdg_beta_handler, "IU",
 676     "Loss-based window decrease factor as a percentage "
 677     "(on loss-based backoff, w = w * beta_loss / 100)");
 678
 679 SYSCTL_VNET_PROC(_net_inet_tcp_cc_cdg, OID_AUTO, exp_backoff_scale,
 680     CTLTYPE_UINT|CTLFLAG_RW, &VNET_NAME(cdg_exp_backoff_scale), 2,
 681     &cdg_exp_backoff_scale_handler, "IU",
 682     "Scaling parameter for the probabilistic exponential backoff");
 683
 684 SYSCTL_VNET_UINT(_net_inet_tcp_cc_cdg,  OID_AUTO, smoothing_factor,
 685     CTLFLAG_RW, &VNET_NAME(cdg_smoothing_factor), 8,
 686     "Number of samples used for moving average smoothing (0 = no smoothing)");
 687
 688 SYSCTL_VNET_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, loss_compete_consec_cong,
 689     CTLFLAG_RW, &VNET_NAME(cdg_consec_cong), 5,
 690     "Number of consecutive delay-gradient based congestion episodes which will "
 691     "trigger loss based CC compatibility");
 692
 693 SYSCTL_VNET_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, loss_compete_hold_backoff,
 694     CTLFLAG_RW, &VNET_NAME(cdg_hold_backoff), 5,
 695     "Number of consecutive delay-gradient based congestion episodes to hold "
 696     "the window backoff for loss based CC compatibility");
 697
 698 DECLARE_CC_MODULE(cdg, &cdg_cc_algo);
 699
 700 MODULE_DEPEND(cdg, ertt, 1, 1, 1);