drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c

   1 /*
   2  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include "en.h"
  34
  35 /* Adaptive moderation profiles */
  36 #define MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
  37 #define MLX5E_RX_AM_DEF_PROFILE_CQE 1
  38 #define MLX5E_RX_AM_DEF_PROFILE_EQE 1
  39 #define MLX5E_PARAMS_AM_NUM_PROFILES 5
  40
  41 /* All profiles sizes must be MLX5E_PARAMS_AM_NUM_PROFILES */
  42 #define MLX5_AM_EQE_PROFILES { \
  43         {1,   MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  44         {8,   MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  45         {64,  MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  46         {128, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  47         {256, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  48 }
  49
  50 #define MLX5_AM_CQE_PROFILES { \
  51         {2,  256},             \
  52         {8,  128},             \
  53         {16, 64},              \
  54         {32, 64},              \
  55         {64, 64}               \
  56 }
  57
  58 static const struct mlx5e_cq_moder
  59 profile[MLX5_CQ_PERIOD_NUM_MODES][MLX5E_PARAMS_AM_NUM_PROFILES] = {
  60         MLX5_AM_EQE_PROFILES,
  61         MLX5_AM_CQE_PROFILES,
  62 };
  63
  64 static inline struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix)
  65 {
  66         return profile[cq_period_mode][ix];
  67 }
  68
  69 struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode)
  70 {
  71         int default_profile_ix;
  72
  73         if (rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
  74                 default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_CQE;
  75         else /* MLX5_CQ_PERIOD_MODE_START_FROM_EQE */
  76                 default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_EQE;
  77
  78         return profile[rx_cq_period_mode][default_profile_ix];
  79 }
  80
  81 /* Adaptive moderation logic */
  82 enum {
  83         MLX5E_AM_START_MEASURE,
  84         MLX5E_AM_MEASURE_IN_PROGRESS,
  85         MLX5E_AM_APPLY_NEW_PROFILE,
  86 };
  87
  88 enum {
  89         MLX5E_AM_PARKING_ON_TOP,
  90         MLX5E_AM_PARKING_TIRED,
  91         MLX5E_AM_GOING_RIGHT,
  92         MLX5E_AM_GOING_LEFT,
  93 };
  94
  95 enum {
  96         MLX5E_AM_STATS_WORSE,
  97         MLX5E_AM_STATS_SAME,
  98         MLX5E_AM_STATS_BETTER,
  99 };
 100
 101 enum {
 102         MLX5E_AM_STEPPED,
 103         MLX5E_AM_TOO_TIRED,
 104         MLX5E_AM_ON_EDGE,
 105 };
 106
 107 static bool mlx5e_am_on_top(struct mlx5e_rx_am *am)
 108 {
 109         switch (am->tune_state) {
 110         case MLX5E_AM_PARKING_ON_TOP:
 111         case MLX5E_AM_PARKING_TIRED:
 112                 WARN_ONCE(true, "mlx5e_am_on_top: PARKING\n");
 113                 return true;
 114         case MLX5E_AM_GOING_RIGHT:
 115                 return (am->steps_left > 1) && (am->steps_right == 1);
 116         default: /* MLX5E_AM_GOING_LEFT */
 117                 return (am->steps_right > 1) && (am->steps_left == 1);
 118         }
 119 }
 120
 121 static void mlx5e_am_turn(struct mlx5e_rx_am *am)
 122 {
 123         switch (am->tune_state) {
 124         case MLX5E_AM_PARKING_ON_TOP:
 125         case MLX5E_AM_PARKING_TIRED:
 126                 WARN_ONCE(true, "mlx5e_am_turn: PARKING\n");
 127                 break;
 128         case MLX5E_AM_GOING_RIGHT:
 129                 am->tune_state = MLX5E_AM_GOING_LEFT;
 130                 am->steps_left = 0;
 131                 break;
 132         case MLX5E_AM_GOING_LEFT:
 133                 am->tune_state = MLX5E_AM_GOING_RIGHT;
 134                 am->steps_right = 0;
 135                 break;
 136         }
 137 }
 138
 139 static int mlx5e_am_step(struct mlx5e_rx_am *am)
 140 {
 141         if (am->tired == (MLX5E_PARAMS_AM_NUM_PROFILES * 2))
 142                 return MLX5E_AM_TOO_TIRED;
 143
 144         switch (am->tune_state) {
 145         case MLX5E_AM_PARKING_ON_TOP:
 146         case MLX5E_AM_PARKING_TIRED:
 147                 WARN_ONCE(true, "mlx5e_am_step: PARKING\n");
 148                 break;
 149         case MLX5E_AM_GOING_RIGHT:
 150                 if (am->profile_ix == (MLX5E_PARAMS_AM_NUM_PROFILES - 1))
 151                         return MLX5E_AM_ON_EDGE;
 152                 am->profile_ix++;
 153                 am->steps_right++;
 154                 break;
 155         case MLX5E_AM_GOING_LEFT:
 156                 if (am->profile_ix == 0)
 157                         return MLX5E_AM_ON_EDGE;
 158                 am->profile_ix--;
 159                 am->steps_left++;
 160                 break;
 161         }
 162
 163         am->tired++;
 164         return MLX5E_AM_STEPPED;
 165 }
 166
 167 static void mlx5e_am_park_on_top(struct mlx5e_rx_am *am)
 168 {
 169         am->steps_right  = 0;
 170         am->steps_left   = 0;
 171         am->tired        = 0;
 172         am->tune_state   = MLX5E_AM_PARKING_ON_TOP;
 173 }
 174
 175 static void mlx5e_am_park_tired(struct mlx5e_rx_am *am)
 176 {
 177         am->steps_right  = 0;
 178         am->steps_left   = 0;
 179         am->tune_state   = MLX5E_AM_PARKING_TIRED;
 180 }
 181
 182 static void mlx5e_am_exit_parking(struct mlx5e_rx_am *am)
 183 {
 184         am->tune_state = am->profile_ix ? MLX5E_AM_GOING_LEFT :
 185                                           MLX5E_AM_GOING_RIGHT;
 186         mlx5e_am_step(am);
 187 }
 188
 189 static int mlx5e_am_stats_compare(struct mlx5e_rx_am_stats *curr,
 190                                   struct mlx5e_rx_am_stats *prev)
 191 {
 192         int diff;
 193
 194         if (!prev->ppms)
 195                 return curr->ppms ? MLX5E_AM_STATS_BETTER :
 196                                     MLX5E_AM_STATS_SAME;
 197
 198         diff = curr->ppms - prev->ppms;
 199         if (((100 * abs(diff)) / prev->ppms) > 10) /* more than 10% diff */
 200                 return (diff > 0) ? MLX5E_AM_STATS_BETTER :
 201                                     MLX5E_AM_STATS_WORSE;
 202
 203         if (!prev->epms)
 204                 return curr->epms ? MLX5E_AM_STATS_WORSE :
 205                                     MLX5E_AM_STATS_SAME;
 206
 207         diff = curr->epms - prev->epms;
 208         if (((100 * abs(diff)) / prev->epms) > 10) /* more than 10% diff */
 209                 return (diff < 0) ? MLX5E_AM_STATS_BETTER :
 210                                     MLX5E_AM_STATS_WORSE;
 211
 212         return MLX5E_AM_STATS_SAME;
 213 }
 214
 215 static bool mlx5e_am_decision(struct mlx5e_rx_am_stats *curr_stats,
 216                               struct mlx5e_rx_am *am)
 217 {
 218         int prev_state = am->tune_state;
 219         int prev_ix = am->profile_ix;
 220         int stats_res;
 221         int step_res;
 222
 223         switch (am->tune_state) {
 224         case MLX5E_AM_PARKING_ON_TOP:
 225                 stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats);
 226                 if (stats_res != MLX5E_AM_STATS_SAME)
 227                         mlx5e_am_exit_parking(am);
 228                 break;
 229
 230         case MLX5E_AM_PARKING_TIRED:
 231                 am->tired--;
 232                 if (!am->tired)
 233                         mlx5e_am_exit_parking(am);
 234                 break;
 235
 236         case MLX5E_AM_GOING_RIGHT:
 237         case MLX5E_AM_GOING_LEFT:
 238                 stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats);
 239                 if (stats_res != MLX5E_AM_STATS_BETTER)
 240                         mlx5e_am_turn(am);
 241
 242                 if (mlx5e_am_on_top(am)) {
 243                         mlx5e_am_park_on_top(am);
 244                         break;
 245                 }
 246
 247                 step_res = mlx5e_am_step(am);
 248                 switch (step_res) {
 249                 case MLX5E_AM_ON_EDGE:
 250                         mlx5e_am_park_on_top(am);
 251                         break;
 252                 case MLX5E_AM_TOO_TIRED:
 253                         mlx5e_am_park_tired(am);
 254                         break;
 255                 }
 256
 257                 break;
 258         }
 259
 260         if ((prev_state     != MLX5E_AM_PARKING_ON_TOP) ||
 261             (am->tune_state != MLX5E_AM_PARKING_ON_TOP))
 262                 am->prev_stats = *curr_stats;
 263
 264         return am->profile_ix != prev_ix;
 265 }
 266
 267 static void mlx5e_am_sample(struct mlx5e_rq *rq,
 268                             struct mlx5e_rx_am_sample *s)
 269 {
 270         s->time      = ktime_get();
 271         s->pkt_ctr   = rq->stats.packets;
 272         s->event_ctr = rq->cq.event_ctr;
 273 }
 274
 275 #define MLX5E_AM_NEVENTS 64
 276
 277 static void mlx5e_am_calc_stats(struct mlx5e_rx_am_sample *start,
 278                                 struct mlx5e_rx_am_sample *end,
 279                                 struct mlx5e_rx_am_stats *curr_stats)
 280 {
 281         /* u32 holds up to 71 minutes, should be enough */
 282         u32 delta_us = ktime_us_delta(end->time, start->time);
 283         unsigned int npkts = end->pkt_ctr - start->pkt_ctr;
 284
 285         if (!delta_us) {
 286                 WARN_ONCE(true, "mlx5e_am_calc_stats: delta_us=0\n");
 287                 return;
 288         }
 289
 290         curr_stats->ppms =            (npkts * USEC_PER_MSEC) / delta_us;
 291         curr_stats->epms = (MLX5E_AM_NEVENTS * USEC_PER_MSEC) / delta_us;
 292 }
 293
 294 void mlx5e_rx_am_work(struct work_struct *work)
 295 {
 296         struct mlx5e_rx_am *am = container_of(work, struct mlx5e_rx_am,
 297                                               work);
 298         struct mlx5e_rq *rq = container_of(am, struct mlx5e_rq, am);
 299         struct mlx5e_cq_moder cur_profile = profile[am->mode][am->profile_ix];
 300
 301         mlx5_core_modify_cq_moderation(rq->priv->mdev, &rq->cq.mcq,
 302                                        cur_profile.usec, cur_profile.pkts);
 303
 304         am->state = MLX5E_AM_START_MEASURE;
 305 }
 306
 307 void mlx5e_rx_am(struct mlx5e_rq *rq)
 308 {
 309         struct mlx5e_rx_am *am = &rq->am;
 310         struct mlx5e_rx_am_sample end_sample;
 311         struct mlx5e_rx_am_stats curr_stats;
 312         u16 nevents;
 313
 314         switch (am->state) {
 315         case MLX5E_AM_MEASURE_IN_PROGRESS:
 316                 nevents = rq->cq.event_ctr - am->start_sample.event_ctr;
 317                 if (nevents < MLX5E_AM_NEVENTS)
 318                         break;
 319                 mlx5e_am_sample(rq, &end_sample);
 320                 mlx5e_am_calc_stats(&am->start_sample, &end_sample,
 321                                     &curr_stats);
 322                 if (mlx5e_am_decision(&curr_stats, am)) {
 323                         am->state = MLX5E_AM_APPLY_NEW_PROFILE;
 324                         schedule_work(&am->work);
 325                         break;
 326                 }
 327                 /* fall through */
 328         case MLX5E_AM_START_MEASURE:
 329                 mlx5e_am_sample(rq, &am->start_sample);
 330                 am->state = MLX5E_AM_MEASURE_IN_PROGRESS;
 331                 break;
 332         case MLX5E_AM_APPLY_NEW_PROFILE:
 333                 break;
 334         }
 335 }