drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <net/netevent.h>
  44 #include <net/neighbour.h>
  45 #include <net/arp.h>
  46 #include <net/ip_fib.h>
  47
  48 #include "spectrum.h"
  49 #include "core.h"
  50 #include "reg.h"
  51
  52 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
  53         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
  54
  55 static bool
  56 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
  57                              struct mlxsw_sp_prefix_usage *prefix_usage2)
  58 {
  59         unsigned char prefix;
  60
  61         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
  62                 if (!test_bit(prefix, prefix_usage2->b))
  63                         return false;
  64         }
  65         return true;
  66 }
  67
  68 static bool
  69 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
  70                          struct mlxsw_sp_prefix_usage *prefix_usage2)
  71 {
  72         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
  73 }
  74
  75 static bool
  76 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
  77 {
  78         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
  79
  80         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
  81 }
  82
  83 static void
  84 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
  85                           struct mlxsw_sp_prefix_usage *prefix_usage2)
  86 {
  87         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
  88 }
  89
  90 static void
  91 mlxsw_sp_prefix_usage_zero(struct mlxsw_sp_prefix_usage *prefix_usage)
  92 {
  93         memset(prefix_usage, 0, sizeof(*prefix_usage));
  94 }
  95
  96 static void
  97 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
  98                           unsigned char prefix_len)
  99 {
 100         set_bit(prefix_len, prefix_usage->b);
 101 }
 102
 103 static void
 104 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 105                             unsigned char prefix_len)
 106 {
 107         clear_bit(prefix_len, prefix_usage->b);
 108 }
 109
 110 struct mlxsw_sp_fib_key {
 111         struct net_device *dev;
 112         unsigned char addr[sizeof(struct in6_addr)];
 113         unsigned char prefix_len;
 114 };
 115
 116 enum mlxsw_sp_fib_entry_type {
 117         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 118         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 119         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 120 };
 121
 122 struct mlxsw_sp_nexthop_group;
 123
 124 struct mlxsw_sp_fib_entry {
 125         struct rhash_head ht_node;
 126         struct list_head list;
 127         struct mlxsw_sp_fib_key key;
 128         enum mlxsw_sp_fib_entry_type type;
 129         unsigned int ref_count;
 130         u16 rif; /* used for action local */
 131         struct mlxsw_sp_vr *vr;
 132         struct fib_info *fi;
 133         struct list_head nexthop_group_node;
 134         struct mlxsw_sp_nexthop_group *nh_group;
 135 };
 136
 137 struct mlxsw_sp_fib {
 138         struct rhashtable ht;
 139         struct list_head entry_list;
 140         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 141         struct mlxsw_sp_prefix_usage prefix_usage;
 142 };
 143
 144 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
 145         .key_offset = offsetof(struct mlxsw_sp_fib_entry, key),
 146         .head_offset = offsetof(struct mlxsw_sp_fib_entry, ht_node),
 147         .key_len = sizeof(struct mlxsw_sp_fib_key),
 148         .automatic_shrinking = true,
 149 };
 150
 151 static int mlxsw_sp_fib_entry_insert(struct mlxsw_sp_fib *fib,
 152                                      struct mlxsw_sp_fib_entry *fib_entry)
 153 {
 154         unsigned char prefix_len = fib_entry->key.prefix_len;
 155         int err;
 156
 157         err = rhashtable_insert_fast(&fib->ht, &fib_entry->ht_node,
 158                                      mlxsw_sp_fib_ht_params);
 159         if (err)
 160                 return err;
 161         list_add_tail(&fib_entry->list, &fib->entry_list);
 162         if (fib->prefix_ref_count[prefix_len]++ == 0)
 163                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
 164         return 0;
 165 }
 166
 167 static void mlxsw_sp_fib_entry_remove(struct mlxsw_sp_fib *fib,
 168                                       struct mlxsw_sp_fib_entry *fib_entry)
 169 {
 170         unsigned char prefix_len = fib_entry->key.prefix_len;
 171
 172         if (--fib->prefix_ref_count[prefix_len] == 0)
 173                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
 174         list_del(&fib_entry->list);
 175         rhashtable_remove_fast(&fib->ht, &fib_entry->ht_node,
 176                                mlxsw_sp_fib_ht_params);
 177 }
 178
 179 static struct mlxsw_sp_fib_entry *
 180 mlxsw_sp_fib_entry_create(struct mlxsw_sp_fib *fib, const void *addr,
 181                           size_t addr_len, unsigned char prefix_len,
 182                           struct net_device *dev)
 183 {
 184         struct mlxsw_sp_fib_entry *fib_entry;
 185
 186         fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
 187         if (!fib_entry)
 188                 return NULL;
 189         fib_entry->key.dev = dev;
 190         memcpy(fib_entry->key.addr, addr, addr_len);
 191         fib_entry->key.prefix_len = prefix_len;
 192         return fib_entry;
 193 }
 194
 195 static void mlxsw_sp_fib_entry_destroy(struct mlxsw_sp_fib_entry *fib_entry)
 196 {
 197         kfree(fib_entry);
 198 }
 199
 200 static struct mlxsw_sp_fib_entry *
 201 mlxsw_sp_fib_entry_lookup(struct mlxsw_sp_fib *fib, const void *addr,
 202                           size_t addr_len, unsigned char prefix_len,
 203                           struct net_device *dev)
 204 {
 205         struct mlxsw_sp_fib_key key;
 206
 207         memset(&key, 0, sizeof(key));
 208         key.dev = dev;
 209         memcpy(key.addr, addr, addr_len);
 210         key.prefix_len = prefix_len;
 211         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
 212 }
 213
 214 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(void)
 215 {
 216         struct mlxsw_sp_fib *fib;
 217         int err;
 218
 219         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 220         if (!fib)
 221                 return ERR_PTR(-ENOMEM);
 222         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 223         if (err)
 224                 goto err_rhashtable_init;
 225         INIT_LIST_HEAD(&fib->entry_list);
 226         return fib;
 227
 228 err_rhashtable_init:
 229         kfree(fib);
 230         return ERR_PTR(err);
 231 }
 232
 233 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 234 {
 235         rhashtable_destroy(&fib->ht);
 236         kfree(fib);
 237 }
 238
 239 static struct mlxsw_sp_lpm_tree *
 240 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp, bool one_reserved)
 241 {
 242         static struct mlxsw_sp_lpm_tree *lpm_tree;
 243         int i;
 244
 245         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 246                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 247                 if (lpm_tree->ref_count == 0) {
 248                         if (one_reserved)
 249                                 one_reserved = false;
 250                         else
 251                                 return lpm_tree;
 252                 }
 253         }
 254         return NULL;
 255 }
 256
 257 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 258                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 259 {
 260         char ralta_pl[MLXSW_REG_RALTA_LEN];
 261
 262         mlxsw_reg_ralta_pack(ralta_pl, true,
 263                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 264                              lpm_tree->id);
 265         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 266 }
 267
 268 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 269                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 270 {
 271         char ralta_pl[MLXSW_REG_RALTA_LEN];
 272
 273         mlxsw_reg_ralta_pack(ralta_pl, false,
 274                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 275                              lpm_tree->id);
 276         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 277 }
 278
 279 static int
 280 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 281                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 282                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 283 {
 284         char ralst_pl[MLXSW_REG_RALST_LEN];
 285         u8 root_bin = 0;
 286         u8 prefix;
 287         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 288
 289         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 290                 root_bin = prefix;
 291
 292         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 293         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 294                 if (prefix == 0)
 295                         continue;
 296                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 297                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 298                 last_prefix = prefix;
 299         }
 300         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 301 }
 302
 303 static struct mlxsw_sp_lpm_tree *
 304 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 305                          struct mlxsw_sp_prefix_usage *prefix_usage,
 306                          enum mlxsw_sp_l3proto proto, bool one_reserved)
 307 {
 308         struct mlxsw_sp_lpm_tree *lpm_tree;
 309         int err;
 310
 311         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp, one_reserved);
 312         if (!lpm_tree)
 313                 return ERR_PTR(-EBUSY);
 314         lpm_tree->proto = proto;
 315         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 316         if (err)
 317                 return ERR_PTR(err);
 318
 319         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 320                                                 lpm_tree);
 321         if (err)
 322                 goto err_left_struct_set;
 323         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 324                sizeof(lpm_tree->prefix_usage));
 325         return lpm_tree;
 326
 327 err_left_struct_set:
 328         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 329         return ERR_PTR(err);
 330 }
 331
 332 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 333                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 334 {
 335         return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 336 }
 337
 338 static struct mlxsw_sp_lpm_tree *
 339 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 340                       struct mlxsw_sp_prefix_usage *prefix_usage,
 341                       enum mlxsw_sp_l3proto proto, bool one_reserved)
 342 {
 343         struct mlxsw_sp_lpm_tree *lpm_tree;
 344         int i;
 345
 346         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 347                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 348                 if (lpm_tree->ref_count != 0 &&
 349                     lpm_tree->proto == proto &&
 350                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 351                                              prefix_usage))
 352                         goto inc_ref_count;
 353         }
 354         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 355                                             proto, one_reserved);
 356         if (IS_ERR(lpm_tree))
 357                 return lpm_tree;
 358
 359 inc_ref_count:
 360         lpm_tree->ref_count++;
 361         return lpm_tree;
 362 }
 363
 364 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 365                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 366 {
 367         if (--lpm_tree->ref_count == 0)
 368                 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 369         return 0;
 370 }
 371
 372 static void mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 373 {
 374         struct mlxsw_sp_lpm_tree *lpm_tree;
 375         int i;
 376
 377         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 378                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 379                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 380         }
 381 }
 382
 383 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 384 {
 385         struct mlxsw_resources *resources;
 386         struct mlxsw_sp_vr *vr;
 387         int i;
 388
 389         resources = mlxsw_core_resources_get(mlxsw_sp->core);
 390         for (i = 0; i < resources->max_virtual_routers; i++) {
 391                 vr = &mlxsw_sp->router.vrs[i];
 392                 if (!vr->used)
 393                         return vr;
 394         }
 395         return NULL;
 396 }
 397
 398 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 399                                      struct mlxsw_sp_vr *vr)
 400 {
 401         char raltb_pl[MLXSW_REG_RALTB_LEN];
 402
 403         mlxsw_reg_raltb_pack(raltb_pl, vr->id,
 404                              (enum mlxsw_reg_ralxx_protocol) vr->proto,
 405                              vr->lpm_tree->id);
 406         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 407 }
 408
 409 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 410                                        struct mlxsw_sp_vr *vr)
 411 {
 412         char raltb_pl[MLXSW_REG_RALTB_LEN];
 413
 414         /* Bind to tree 0 which is default */
 415         mlxsw_reg_raltb_pack(raltb_pl, vr->id,
 416                              (enum mlxsw_reg_ralxx_protocol) vr->proto, 0);
 417         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 418 }
 419
 420 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 421 {
 422         /* For our purpose, squash main and local table into one */
 423         if (tb_id == RT_TABLE_LOCAL)
 424                 tb_id = RT_TABLE_MAIN;
 425         return tb_id;
 426 }
 427
 428 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 429                                             u32 tb_id,
 430                                             enum mlxsw_sp_l3proto proto)
 431 {
 432         struct mlxsw_resources *resources;
 433         struct mlxsw_sp_vr *vr;
 434         int i;
 435
 436         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 437
 438         resources = mlxsw_core_resources_get(mlxsw_sp->core);
 439         for (i = 0; i < resources->max_virtual_routers; i++) {
 440                 vr = &mlxsw_sp->router.vrs[i];
 441                 if (vr->used && vr->proto == proto && vr->tb_id == tb_id)
 442                         return vr;
 443         }
 444         return NULL;
 445 }
 446
 447 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 448                                               unsigned char prefix_len,
 449                                               u32 tb_id,
 450                                               enum mlxsw_sp_l3proto proto)
 451 {
 452         struct mlxsw_sp_prefix_usage req_prefix_usage;
 453         struct mlxsw_sp_lpm_tree *lpm_tree;
 454         struct mlxsw_sp_vr *vr;
 455         int err;
 456
 457         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 458         if (!vr)
 459                 return ERR_PTR(-EBUSY);
 460         vr->fib = mlxsw_sp_fib_create();
 461         if (IS_ERR(vr->fib))
 462                 return ERR_CAST(vr->fib);
 463
 464         vr->proto = proto;
 465         vr->tb_id = tb_id;
 466         mlxsw_sp_prefix_usage_zero(&req_prefix_usage);
 467         mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
 468         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 469                                          proto, true);
 470         if (IS_ERR(lpm_tree)) {
 471                 err = PTR_ERR(lpm_tree);
 472                 goto err_tree_get;
 473         }
 474         vr->lpm_tree = lpm_tree;
 475         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
 476         if (err)
 477                 goto err_tree_bind;
 478
 479         vr->used = true;
 480         return vr;
 481
 482 err_tree_bind:
 483         mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
 484 err_tree_get:
 485         mlxsw_sp_fib_destroy(vr->fib);
 486
 487         return ERR_PTR(err);
 488 }
 489
 490 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
 491                                 struct mlxsw_sp_vr *vr)
 492 {
 493         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
 494         mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
 495         mlxsw_sp_fib_destroy(vr->fib);
 496         vr->used = false;
 497 }
 498
 499 static int
 500 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
 501                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 502 {
 503         struct mlxsw_sp_lpm_tree *lpm_tree;
 504
 505         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage,
 506                                      &vr->lpm_tree->prefix_usage))
 507                 return 0;
 508
 509         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 510                                          vr->proto, false);
 511         if (IS_ERR(lpm_tree)) {
 512                 /* We failed to get a tree according to the required
 513                  * prefix usage. However, the current tree might be still good
 514                  * for us if our requirement is subset of the prefixes used
 515                  * in the tree.
 516                  */
 517                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 518                                                  &vr->lpm_tree->prefix_usage))
 519                         return 0;
 520                 return PTR_ERR(lpm_tree);
 521         }
 522
 523         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
 524         mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
 525         vr->lpm_tree = lpm_tree;
 526         return mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
 527 }
 528
 529 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp,
 530                                            unsigned char prefix_len,
 531                                            u32 tb_id,
 532                                            enum mlxsw_sp_l3proto proto)
 533 {
 534         struct mlxsw_sp_vr *vr;
 535         int err;
 536
 537         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 538         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id, proto);
 539         if (!vr) {
 540                 vr = mlxsw_sp_vr_create(mlxsw_sp, prefix_len, tb_id, proto);
 541                 if (IS_ERR(vr))
 542                         return vr;
 543         } else {
 544                 struct mlxsw_sp_prefix_usage req_prefix_usage;
 545
 546                 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
 547                                           &vr->fib->prefix_usage);
 548                 mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
 549                 /* Need to replace LPM tree in case new prefix is required. */
 550                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
 551                                                  &req_prefix_usage);
 552                 if (err)
 553                         return ERR_PTR(err);
 554         }
 555         return vr;
 556 }
 557
 558 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 559 {
 560         /* Destroy virtual router entity in case the associated FIB is empty
 561          * and allow it to be used for other tables in future. Otherwise,
 562          * check if some prefix usage did not disappear and change tree if
 563          * that is the case. Note that in case new, smaller tree cannot be
 564          * allocated, the original one will be kept being used.
 565          */
 566         if (mlxsw_sp_prefix_usage_none(&vr->fib->prefix_usage))
 567                 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 568         else
 569                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
 570                                            &vr->fib->prefix_usage);
 571 }
 572
 573 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 574 {
 575         struct mlxsw_resources *resources;
 576         struct mlxsw_sp_vr *vr;
 577         int i;
 578
 579         resources = mlxsw_core_resources_get(mlxsw_sp->core);
 580         if (!resources->max_virtual_routers_valid)
 581                 return -EIO;
 582
 583         mlxsw_sp->router.vrs = kcalloc(resources->max_virtual_routers,
 584                                        sizeof(struct mlxsw_sp_vr),
 585                                        GFP_KERNEL);
 586         if (!mlxsw_sp->router.vrs)
 587                 return -ENOMEM;
 588
 589         for (i = 0; i < resources->max_virtual_routers; i++) {
 590                 vr = &mlxsw_sp->router.vrs[i];
 591                 vr->id = i;
 592         }
 593
 594         return 0;
 595 }
 596
 597 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 598
 599 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 600 {
 601         mlxsw_sp_router_fib_flush(mlxsw_sp);
 602         kfree(mlxsw_sp->router.vrs);
 603 }
 604
 605 struct mlxsw_sp_neigh_key {
 606         struct neighbour *n;
 607 };
 608
 609 struct mlxsw_sp_neigh_entry {
 610         struct rhash_head ht_node;
 611         struct mlxsw_sp_neigh_key key;
 612         u16 rif;
 613         bool offloaded;
 614         struct delayed_work dw;
 615         struct mlxsw_sp_port *mlxsw_sp_port;
 616         unsigned char ha[ETH_ALEN];
 617         struct list_head nexthop_list; /* list of nexthops using
 618                                         * this neigh entry
 619                                         */
 620         struct list_head nexthop_neighs_list_node;
 621 };
 622
 623 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 624         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 625         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 626         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 627 };
 628
 629 static int
 630 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 631                             struct mlxsw_sp_neigh_entry *neigh_entry)
 632 {
 633         return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
 634                                       &neigh_entry->ht_node,
 635                                       mlxsw_sp_neigh_ht_params);
 636 }
 637
 638 static void
 639 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 640                             struct mlxsw_sp_neigh_entry *neigh_entry)
 641 {
 642         rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
 643                                &neigh_entry->ht_node,
 644                                mlxsw_sp_neigh_ht_params);
 645 }
 646
 647 static void mlxsw_sp_router_neigh_update_hw(struct work_struct *work);
 648
 649 static struct mlxsw_sp_neigh_entry *
 650 mlxsw_sp_neigh_entry_create(struct neighbour *n, u16 rif)
 651 {
 652         struct mlxsw_sp_neigh_entry *neigh_entry;
 653
 654         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_ATOMIC);
 655         if (!neigh_entry)
 656                 return NULL;
 657         neigh_entry->key.n = n;
 658         neigh_entry->rif = rif;
 659         INIT_DELAYED_WORK(&neigh_entry->dw, mlxsw_sp_router_neigh_update_hw);
 660         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 661         return neigh_entry;
 662 }
 663
 664 static void
 665 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp_neigh_entry *neigh_entry)
 666 {
 667         kfree(neigh_entry);
 668 }
 669
 670 static struct mlxsw_sp_neigh_entry *
 671 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 672 {
 673         struct mlxsw_sp_neigh_key key;
 674
 675         key.n = n;
 676         return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
 677                                       &key, mlxsw_sp_neigh_ht_params);
 678 }
 679
 680 int mlxsw_sp_router_neigh_construct(struct net_device *dev,
 681                                     struct neighbour *n)
 682 {
 683         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
 684         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 685         struct mlxsw_sp_neigh_entry *neigh_entry;
 686         struct mlxsw_sp_rif *r;
 687         int err;
 688
 689         if (n->tbl != &arp_tbl)
 690                 return 0;
 691
 692         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
 693         if (neigh_entry)
 694                 return 0;
 695
 696         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 697         if (WARN_ON(!r))
 698                 return -EINVAL;
 699
 700         neigh_entry = mlxsw_sp_neigh_entry_create(n, r->rif);
 701         if (!neigh_entry)
 702                 return -ENOMEM;
 703         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 704         if (err)
 705                 goto err_neigh_entry_insert;
 706         return 0;
 707
 708 err_neigh_entry_insert:
 709         mlxsw_sp_neigh_entry_destroy(neigh_entry);
 710         return err;
 711 }
 712
 713 void mlxsw_sp_router_neigh_destroy(struct net_device *dev,
 714                                    struct neighbour *n)
 715 {
 716         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
 717         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 718         struct mlxsw_sp_neigh_entry *neigh_entry;
 719
 720         if (n->tbl != &arp_tbl)
 721                 return;
 722
 723         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
 724         if (!neigh_entry)
 725                 return;
 726         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 727         mlxsw_sp_neigh_entry_destroy(neigh_entry);
 728 }
 729
 730 static void
 731 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 732 {
 733         unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 734
 735         mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
 736 }
 737
 738 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 739                                                    char *rauhtd_pl,
 740                                                    int ent_index)
 741 {
 742         struct net_device *dev;
 743         struct neighbour *n;
 744         __be32 dipn;
 745         u32 dip;
 746         u16 rif;
 747
 748         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 749
 750         if (!mlxsw_sp->rifs[rif]) {
 751                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 752                 return;
 753         }
 754
 755         dipn = htonl(dip);
 756         dev = mlxsw_sp->rifs[rif]->dev;
 757         n = neigh_lookup(&arp_tbl, &dipn, dev);
 758         if (!n) {
 759                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
 760                            &dip);
 761                 return;
 762         }
 763
 764         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
 765         neigh_event_send(n, NULL);
 766         neigh_release(n);
 767 }
 768
 769 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 770                                                    char *rauhtd_pl,
 771                                                    int rec_index)
 772 {
 773         u8 num_entries;
 774         int i;
 775
 776         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 777                                                                 rec_index);
 778         /* Hardware starts counting at 0, so add 1. */
 779         num_entries++;
 780
 781         /* Each record consists of several neighbour entries. */
 782         for (i = 0; i < num_entries; i++) {
 783                 int ent_index;
 784
 785                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
 786                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
 787                                                        ent_index);
 788         }
 789
 790 }
 791
 792 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 793                                               char *rauhtd_pl, int rec_index)
 794 {
 795         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
 796         case MLXSW_REG_RAUHTD_TYPE_IPV4:
 797                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
 798                                                        rec_index);
 799                 break;
 800         case MLXSW_REG_RAUHTD_TYPE_IPV6:
 801                 WARN_ON_ONCE(1);
 802                 break;
 803         }
 804 }
 805
 806 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
 807 {
 808         u8 num_rec, last_rec_index, num_entries;
 809
 810         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 811         last_rec_index = num_rec - 1;
 812
 813         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
 814                 return false;
 815         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
 816             MLXSW_REG_RAUHTD_TYPE_IPV6)
 817                 return true;
 818
 819         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 820                                                                 last_rec_index);
 821         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
 822                 return true;
 823         return false;
 824 }
 825
 826 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
 827 {
 828         char *rauhtd_pl;
 829         u8 num_rec;
 830         int i, err;
 831
 832         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
 833         if (!rauhtd_pl)
 834                 return -ENOMEM;
 835
 836         /* Make sure the neighbour's netdev isn't removed in the
 837          * process.
 838          */
 839         rtnl_lock();
 840         do {
 841                 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
 842                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
 843                                       rauhtd_pl);
 844                 if (err) {
 845                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
 846                         break;
 847                 }
 848                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 849                 for (i = 0; i < num_rec; i++)
 850                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
 851                                                           i);
 852         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
 853         rtnl_unlock();
 854
 855         kfree(rauhtd_pl);
 856         return err;
 857 }
 858
 859 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
 860 {
 861         struct mlxsw_sp_neigh_entry *neigh_entry;
 862
 863         /* Take RTNL mutex here to prevent lists from changes */
 864         rtnl_lock();
 865         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 866                             nexthop_neighs_list_node) {
 867                 /* If this neigh have nexthops, make the kernel think this neigh
 868                  * is active regardless of the traffic.
 869                  */
 870                 if (!list_empty(&neigh_entry->nexthop_list))
 871                         neigh_event_send(neigh_entry->key.n, NULL);
 872         }
 873         rtnl_unlock();
 874 }
 875
 876 static void
 877 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
 878 {
 879         unsigned long interval = mlxsw_sp->router.neighs_update.interval;
 880
 881         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
 882                                msecs_to_jiffies(interval));
 883 }
 884
 885 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
 886 {
 887         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 888                                                  router.neighs_update.dw.work);
 889         int err;
 890
 891         err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
 892         if (err)
 893                 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
 894
 895         mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
 896
 897         mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
 898 }
 899
 900 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
 901 {
 902         struct mlxsw_sp_neigh_entry *neigh_entry;
 903         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 904                                                  router.nexthop_probe_dw.work);
 905
 906         /* Iterate over nexthop neighbours, find those who are unresolved and
 907          * send arp on them. This solves the chicken-egg problem when
 908          * the nexthop wouldn't get offloaded until the neighbor is resolved
 909          * but it wouldn't get resolved ever in case traffic is flowing in HW
 910          * using different nexthop.
 911          *
 912          * Take RTNL mutex here to prevent lists from changes.
 913          */
 914         rtnl_lock();
 915         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 916                             nexthop_neighs_list_node) {
 917                 if (!(neigh_entry->key.n->nud_state & NUD_VALID) &&
 918                     !list_empty(&neigh_entry->nexthop_list))
 919                         neigh_event_send(neigh_entry->key.n, NULL);
 920         }
 921         rtnl_unlock();
 922
 923         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
 924                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
 925 }
 926
 927 static void
 928 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
 929                               struct mlxsw_sp_neigh_entry *neigh_entry,
 930                               bool removing);
 931
 932 static void mlxsw_sp_router_neigh_update_hw(struct work_struct *work)
 933 {
 934         struct mlxsw_sp_neigh_entry *neigh_entry =
 935                 container_of(work, struct mlxsw_sp_neigh_entry, dw.work);
 936         struct neighbour *n = neigh_entry->key.n;
 937         struct mlxsw_sp_port *mlxsw_sp_port = neigh_entry->mlxsw_sp_port;
 938         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 939         char rauht_pl[MLXSW_REG_RAUHT_LEN];
 940         struct net_device *dev;
 941         bool entry_connected;
 942         u8 nud_state;
 943         bool updating;
 944         bool removing;
 945         bool adding;
 946         u32 dip;
 947         int err;
 948
 949         read_lock_bh(&n->lock);
 950         dip = ntohl(*((__be32 *) n->primary_key));
 951         memcpy(neigh_entry->ha, n->ha, sizeof(neigh_entry->ha));
 952         nud_state = n->nud_state;
 953         dev = n->dev;
 954         read_unlock_bh(&n->lock);
 955
 956         entry_connected = nud_state & NUD_VALID;
 957         adding = (!neigh_entry->offloaded) && entry_connected;
 958         updating = neigh_entry->offloaded && entry_connected;
 959         removing = neigh_entry->offloaded && !entry_connected;
 960
 961         if (adding || updating) {
 962                 mlxsw_reg_rauht_pack4(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_ADD,
 963                                       neigh_entry->rif,
 964                                       neigh_entry->ha, dip);
 965                 err = mlxsw_reg_write(mlxsw_sp->core,
 966                                       MLXSW_REG(rauht), rauht_pl);
 967                 if (err) {
 968                         netdev_err(dev, "Could not add neigh %pI4h\n", &dip);
 969                         neigh_entry->offloaded = false;
 970                 } else {
 971                         neigh_entry->offloaded = true;
 972                 }
 973                 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, false);
 974         } else if (removing) {
 975                 mlxsw_reg_rauht_pack4(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE,
 976                                       neigh_entry->rif,
 977                                       neigh_entry->ha, dip);
 978                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht),
 979                                       rauht_pl);
 980                 if (err) {
 981                         netdev_err(dev, "Could not delete neigh %pI4h\n", &dip);
 982                         neigh_entry->offloaded = true;
 983                 } else {
 984                         neigh_entry->offloaded = false;
 985                 }
 986                 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, true);
 987         }
 988
 989         neigh_release(n);
 990         mlxsw_sp_port_dev_put(mlxsw_sp_port);
 991 }
 992
 993 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
 994                                    unsigned long event, void *ptr)
 995 {
 996         struct mlxsw_sp_neigh_entry *neigh_entry;
 997         struct mlxsw_sp_port *mlxsw_sp_port;
 998         struct mlxsw_sp *mlxsw_sp;
 999         unsigned long interval;
1000         struct net_device *dev;
1001         struct neigh_parms *p;
1002         struct neighbour *n;
1003         u32 dip;
1004
1005         switch (event) {
1006         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1007                 p = ptr;
1008
1009                 /* We don't care about changes in the default table. */
1010                 if (!p->dev || p->tbl != &arp_tbl)
1011                         return NOTIFY_DONE;
1012
1013                 /* We are in atomic context and can't take RTNL mutex,
1014                  * so use RCU variant to walk the device chain.
1015                  */
1016                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1017                 if (!mlxsw_sp_port)
1018                         return NOTIFY_DONE;
1019
1020                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1021                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1022                 mlxsw_sp->router.neighs_update.interval = interval;
1023
1024                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1025                 break;
1026         case NETEVENT_NEIGH_UPDATE:
1027                 n = ptr;
1028                 dev = n->dev;
1029
1030                 if (n->tbl != &arp_tbl)
1031                         return NOTIFY_DONE;
1032
1033                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(dev);
1034                 if (!mlxsw_sp_port)
1035                         return NOTIFY_DONE;
1036
1037                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1038                 dip = ntohl(*((__be32 *) n->primary_key));
1039                 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1040                 if (WARN_ON(!neigh_entry)) {
1041                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1042                         return NOTIFY_DONE;
1043                 }
1044                 neigh_entry->mlxsw_sp_port = mlxsw_sp_port;
1045
1046                 /* Take a reference to ensure the neighbour won't be
1047                  * destructed until we drop the reference in delayed
1048                  * work.
1049                  */
1050                 neigh_clone(n);
1051                 if (!mlxsw_core_schedule_dw(&neigh_entry->dw, 0)) {
1052                         neigh_release(n);
1053                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1054                 }
1055                 break;
1056         }
1057
1058         return NOTIFY_DONE;
1059 }
1060
1061 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1062 {
1063         int err;
1064
1065         err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
1066                               &mlxsw_sp_neigh_ht_params);
1067         if (err)
1068                 return err;
1069
1070         /* Initialize the polling interval according to the default
1071          * table.
1072          */
1073         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1074
1075         /* Create the delayed works for the activity_update */
1076         INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1077                           mlxsw_sp_router_neighs_update_work);
1078         INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1079                           mlxsw_sp_router_probe_unresolved_nexthops);
1080         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1081         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1082         return 0;
1083 }
1084
1085 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1086 {
1087         cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1088         cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1089         rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1090 }
1091
1092 struct mlxsw_sp_nexthop {
1093         struct list_head neigh_list_node; /* member of neigh entry list */
1094         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1095                                                 * this belongs to
1096                                                 */
1097         u8 should_offload:1, /* set indicates this neigh is connected and
1098                               * should be put to KVD linear area of this group.
1099                               */
1100            offloaded:1, /* set in case the neigh is actually put into
1101                          * KVD linear area of this group.
1102                          */
1103            update:1; /* set indicates that MAC of this neigh should be
1104                       * updated in HW
1105                       */
1106         struct mlxsw_sp_neigh_entry *neigh_entry;
1107 };
1108
1109 struct mlxsw_sp_nexthop_group {
1110         struct list_head list; /* node in mlxsw->router.nexthop_group_list */
1111         struct list_head fib_list; /* list of fib entries that use this group */
1112         u8 adj_index_valid:1;
1113         u32 adj_index;
1114         u16 ecmp_size;
1115         u16 count;
1116         struct mlxsw_sp_nexthop nexthops[0];
1117 };
1118
1119 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1120                                              struct mlxsw_sp_vr *vr,
1121                                              u32 adj_index, u16 ecmp_size,
1122                                              u32 new_adj_index,
1123                                              u16 new_ecmp_size)
1124 {
1125         char raleu_pl[MLXSW_REG_RALEU_LEN];
1126
1127         mlxsw_reg_raleu_pack(raleu_pl,
1128                              (enum mlxsw_reg_ralxx_protocol) vr->proto, vr->id,
1129                              adj_index, ecmp_size, new_adj_index,
1130                              new_ecmp_size);
1131         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1132 }
1133
1134 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1135                                           struct mlxsw_sp_nexthop_group *nh_grp,
1136                                           u32 old_adj_index, u16 old_ecmp_size)
1137 {
1138         struct mlxsw_sp_fib_entry *fib_entry;
1139         struct mlxsw_sp_vr *vr = NULL;
1140         int err;
1141
1142         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1143                 if (vr == fib_entry->vr)
1144                         continue;
1145                 vr = fib_entry->vr;
1146                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr,
1147                                                         old_adj_index,
1148                                                         old_ecmp_size,
1149                                                         nh_grp->adj_index,
1150                                                         nh_grp->ecmp_size);
1151                 if (err)
1152                         return err;
1153         }
1154         return 0;
1155 }
1156
1157 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1158                                        struct mlxsw_sp_nexthop *nh)
1159 {
1160         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1161         char ratr_pl[MLXSW_REG_RATR_LEN];
1162
1163         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1164                             true, adj_index, neigh_entry->rif);
1165         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1166         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1167 }
1168
1169 static int
1170 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1171                                   struct mlxsw_sp_nexthop_group *nh_grp)
1172 {
1173         u32 adj_index = nh_grp->adj_index; /* base */
1174         struct mlxsw_sp_nexthop *nh;
1175         int i;
1176         int err;
1177
1178         for (i = 0; i < nh_grp->count; i++) {
1179                 nh = &nh_grp->nexthops[i];
1180
1181                 if (!nh->should_offload) {
1182                         nh->offloaded = 0;
1183                         continue;
1184                 }
1185
1186                 if (nh->update) {
1187                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1188                                                           adj_index, nh);
1189                         if (err)
1190                                 return err;
1191                         nh->update = 0;
1192                         nh->offloaded = 1;
1193                 }
1194                 adj_index++;
1195         }
1196         return 0;
1197 }
1198
1199 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1200                                      struct mlxsw_sp_fib_entry *fib_entry);
1201
1202 static int
1203 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1204                                     struct mlxsw_sp_nexthop_group *nh_grp)
1205 {
1206         struct mlxsw_sp_fib_entry *fib_entry;
1207         int err;
1208
1209         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1210                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1211                 if (err)
1212                         return err;
1213         }
1214         return 0;
1215 }
1216
1217 static void
1218 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1219                                struct mlxsw_sp_nexthop_group *nh_grp)
1220 {
1221         struct mlxsw_sp_nexthop *nh;
1222         bool offload_change = false;
1223         u32 adj_index;
1224         u16 ecmp_size = 0;
1225         bool old_adj_index_valid;
1226         u32 old_adj_index;
1227         u16 old_ecmp_size;
1228         int ret;
1229         int i;
1230         int err;
1231
1232         for (i = 0; i < nh_grp->count; i++) {
1233                 nh = &nh_grp->nexthops[i];
1234
1235                 if (nh->should_offload ^ nh->offloaded) {
1236                         offload_change = true;
1237                         if (nh->should_offload)
1238                                 nh->update = 1;
1239                 }
1240                 if (nh->should_offload)
1241                         ecmp_size++;
1242         }
1243         if (!offload_change) {
1244                 /* Nothing was added or removed, so no need to reallocate. Just
1245                  * update MAC on existing adjacency indexes.
1246                  */
1247                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp);
1248                 if (err) {
1249                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1250                         goto set_trap;
1251                 }
1252                 return;
1253         }
1254         if (!ecmp_size)
1255                 /* No neigh of this group is connected so we just set
1256                  * the trap and let everthing flow through kernel.
1257                  */
1258                 goto set_trap;
1259
1260         ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1261         if (ret < 0) {
1262                 /* We ran out of KVD linear space, just set the
1263                  * trap and let everything flow through kernel.
1264                  */
1265                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1266                 goto set_trap;
1267         }
1268         adj_index = ret;
1269         old_adj_index_valid = nh_grp->adj_index_valid;
1270         old_adj_index = nh_grp->adj_index;
1271         old_ecmp_size = nh_grp->ecmp_size;
1272         nh_grp->adj_index_valid = 1;
1273         nh_grp->adj_index = adj_index;
1274         nh_grp->ecmp_size = ecmp_size;
1275         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp);
1276         if (err) {
1277                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1278                 goto set_trap;
1279         }
1280
1281         if (!old_adj_index_valid) {
1282                 /* The trap was set for fib entries, so we have to call
1283                  * fib entry update to unset it and use adjacency index.
1284                  */
1285                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1286                 if (err) {
1287                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1288                         goto set_trap;
1289                 }
1290                 return;
1291         }
1292
1293         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1294                                              old_adj_index, old_ecmp_size);
1295         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1296         if (err) {
1297                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1298                 goto set_trap;
1299         }
1300         return;
1301
1302 set_trap:
1303         old_adj_index_valid = nh_grp->adj_index_valid;
1304         nh_grp->adj_index_valid = 0;
1305         for (i = 0; i < nh_grp->count; i++) {
1306                 nh = &nh_grp->nexthops[i];
1307                 nh->offloaded = 0;
1308         }
1309         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1310         if (err)
1311                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1312         if (old_adj_index_valid)
1313                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1314 }
1315
1316 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1317                                             bool removing)
1318 {
1319         if (!removing && !nh->should_offload)
1320                 nh->should_offload = 1;
1321         else if (removing && nh->offloaded)
1322                 nh->should_offload = 0;
1323         nh->update = 1;
1324 }
1325
1326 static void
1327 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1328                               struct mlxsw_sp_neigh_entry *neigh_entry,
1329                               bool removing)
1330 {
1331         struct mlxsw_sp_nexthop *nh;
1332
1333         /* Take RTNL mutex here to prevent lists from changes */
1334         rtnl_lock();
1335         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1336                             neigh_list_node) {
1337                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1338                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1339         }
1340         rtnl_unlock();
1341 }
1342
1343 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1344                                  struct mlxsw_sp_nexthop_group *nh_grp,
1345                                  struct mlxsw_sp_nexthop *nh,
1346                                  struct fib_nh *fib_nh)
1347 {
1348         struct mlxsw_sp_neigh_entry *neigh_entry;
1349         struct net_device *dev = fib_nh->nh_dev;
1350         struct neighbour *n;
1351         u8 nud_state;
1352
1353         /* Take a reference of neigh here ensuring that neigh would
1354          * not be detructed before the nexthop entry is finished.
1355          * The reference is taken either in neigh_lookup() or
1356          * in neith_create() in case n is not found.
1357          */
1358         n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, dev);
1359         if (!n) {
1360                 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, dev);
1361                 if (IS_ERR(n))
1362                         return PTR_ERR(n);
1363                 neigh_event_send(n, NULL);
1364         }
1365         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1366         if (!neigh_entry) {
1367                 neigh_release(n);
1368                 return -EINVAL;
1369         }
1370
1371         /* If that is the first nexthop connected to that neigh, add to
1372          * nexthop_neighs_list
1373          */
1374         if (list_empty(&neigh_entry->nexthop_list))
1375                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1376                               &mlxsw_sp->router.nexthop_neighs_list);
1377
1378         nh->nh_grp = nh_grp;
1379         nh->neigh_entry = neigh_entry;
1380         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1381         read_lock_bh(&n->lock);
1382         nud_state = n->nud_state;
1383         read_unlock_bh(&n->lock);
1384         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID));
1385
1386         return 0;
1387 }
1388
1389 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1390                                   struct mlxsw_sp_nexthop *nh)
1391 {
1392         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1393
1394         list_del(&nh->neigh_list_node);
1395
1396         /* If that is the last nexthop connected to that neigh, remove from
1397          * nexthop_neighs_list
1398          */
1399         if (list_empty(&nh->neigh_entry->nexthop_list))
1400                 list_del(&nh->neigh_entry->nexthop_neighs_list_node);
1401
1402         neigh_release(neigh_entry->key.n);
1403 }
1404
1405 static struct mlxsw_sp_nexthop_group *
1406 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1407 {
1408         struct mlxsw_sp_nexthop_group *nh_grp;
1409         struct mlxsw_sp_nexthop *nh;
1410         struct fib_nh *fib_nh;
1411         size_t alloc_size;
1412         int i;
1413         int err;
1414
1415         alloc_size = sizeof(*nh_grp) +
1416                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1417         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1418         if (!nh_grp)
1419                 return ERR_PTR(-ENOMEM);
1420         INIT_LIST_HEAD(&nh_grp->fib_list);
1421         nh_grp->count = fi->fib_nhs;
1422         for (i = 0; i < nh_grp->count; i++) {
1423                 nh = &nh_grp->nexthops[i];
1424                 fib_nh = &fi->fib_nh[i];
1425                 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1426                 if (err)
1427                         goto err_nexthop_init;
1428         }
1429         list_add_tail(&nh_grp->list, &mlxsw_sp->router.nexthop_group_list);
1430         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1431         return nh_grp;
1432
1433 err_nexthop_init:
1434         for (i--; i >= 0; i--)
1435                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1436         kfree(nh_grp);
1437         return ERR_PTR(err);
1438 }
1439
1440 static void
1441 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1442                                struct mlxsw_sp_nexthop_group *nh_grp)
1443 {
1444         struct mlxsw_sp_nexthop *nh;
1445         int i;
1446
1447         list_del(&nh_grp->list);
1448         for (i = 0; i < nh_grp->count; i++) {
1449                 nh = &nh_grp->nexthops[i];
1450                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1451         }
1452         kfree(nh_grp);
1453 }
1454
1455 static bool mlxsw_sp_nexthop_match(struct mlxsw_sp_nexthop *nh,
1456                                    struct fib_info *fi)
1457 {
1458         int i;
1459
1460         for (i = 0; i < fi->fib_nhs; i++) {
1461                 struct fib_nh *fib_nh = &fi->fib_nh[i];
1462                 struct neighbour *n = nh->neigh_entry->key.n;
1463
1464                 if (memcmp(n->primary_key, &fib_nh->nh_gw,
1465                            sizeof(fib_nh->nh_gw)) == 0 &&
1466                     n->dev == fib_nh->nh_dev)
1467                         return true;
1468         }
1469         return false;
1470 }
1471
1472 static bool mlxsw_sp_nexthop_group_match(struct mlxsw_sp_nexthop_group *nh_grp,
1473                                          struct fib_info *fi)
1474 {
1475         int i;
1476
1477         if (nh_grp->count != fi->fib_nhs)
1478                 return false;
1479         for (i = 0; i < nh_grp->count; i++) {
1480                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
1481
1482                 if (!mlxsw_sp_nexthop_match(nh, fi))
1483                         return false;
1484         }
1485         return true;
1486 }
1487
1488 static struct mlxsw_sp_nexthop_group *
1489 mlxsw_sp_nexthop_group_find(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1490 {
1491         struct mlxsw_sp_nexthop_group *nh_grp;
1492
1493         list_for_each_entry(nh_grp, &mlxsw_sp->router.nexthop_group_list,
1494                             list) {
1495                 if (mlxsw_sp_nexthop_group_match(nh_grp, fi))
1496                         return nh_grp;
1497         }
1498         return NULL;
1499 }
1500
1501 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1502                                       struct mlxsw_sp_fib_entry *fib_entry,
1503                                       struct fib_info *fi)
1504 {
1505         struct mlxsw_sp_nexthop_group *nh_grp;
1506
1507         nh_grp = mlxsw_sp_nexthop_group_find(mlxsw_sp, fi);
1508         if (!nh_grp) {
1509                 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1510                 if (IS_ERR(nh_grp))
1511                         return PTR_ERR(nh_grp);
1512         }
1513         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1514         fib_entry->nh_group = nh_grp;
1515         return 0;
1516 }
1517
1518 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1519                                        struct mlxsw_sp_fib_entry *fib_entry)
1520 {
1521         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1522
1523         list_del(&fib_entry->nexthop_group_node);
1524         if (!list_empty(&nh_grp->fib_list))
1525                 return;
1526         mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1527 }
1528
1529 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1530                                          struct mlxsw_sp_fib_entry *fib_entry,
1531                                          enum mlxsw_reg_ralue_op op)
1532 {
1533         char ralue_pl[MLXSW_REG_RALUE_LEN];
1534         u32 *p_dip = (u32 *) fib_entry->key.addr;
1535         struct mlxsw_sp_vr *vr = fib_entry->vr;
1536         enum mlxsw_reg_ralue_trap_action trap_action;
1537         u16 trap_id = 0;
1538         u32 adjacency_index = 0;
1539         u16 ecmp_size = 0;
1540
1541         /* In case the nexthop group adjacency index is valid, use it
1542          * with provided ECMP size. Otherwise, setup trap and pass
1543          * traffic to kernel.
1544          */
1545         if (fib_entry->nh_group->adj_index_valid) {
1546                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1547                 adjacency_index = fib_entry->nh_group->adj_index;
1548                 ecmp_size = fib_entry->nh_group->ecmp_size;
1549         } else {
1550                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1551                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1552         }
1553
1554         mlxsw_reg_ralue_pack4(ralue_pl,
1555                               (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1556                               vr->id, fib_entry->key.prefix_len, *p_dip);
1557         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1558                                         adjacency_index, ecmp_size);
1559         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1560 }
1561
1562 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1563                                         struct mlxsw_sp_fib_entry *fib_entry,
1564                                         enum mlxsw_reg_ralue_op op)
1565 {
1566         char ralue_pl[MLXSW_REG_RALUE_LEN];
1567         u32 *p_dip = (u32 *) fib_entry->key.addr;
1568         struct mlxsw_sp_vr *vr = fib_entry->vr;
1569
1570         mlxsw_reg_ralue_pack4(ralue_pl,
1571                               (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1572                               vr->id, fib_entry->key.prefix_len, *p_dip);
1573         mlxsw_reg_ralue_act_local_pack(ralue_pl,
1574                                        MLXSW_REG_RALUE_TRAP_ACTION_NOP, 0,
1575                                        fib_entry->rif);
1576         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1577 }
1578
1579 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1580                                        struct mlxsw_sp_fib_entry *fib_entry,
1581                                        enum mlxsw_reg_ralue_op op)
1582 {
1583         char ralue_pl[MLXSW_REG_RALUE_LEN];
1584         u32 *p_dip = (u32 *) fib_entry->key.addr;
1585         struct mlxsw_sp_vr *vr = fib_entry->vr;
1586
1587         mlxsw_reg_ralue_pack4(ralue_pl,
1588                               (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1589                               vr->id, fib_entry->key.prefix_len, *p_dip);
1590         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1591         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1592 }
1593
1594 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1595                                   struct mlxsw_sp_fib_entry *fib_entry,
1596                                   enum mlxsw_reg_ralue_op op)
1597 {
1598         switch (fib_entry->type) {
1599         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1600                 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1601         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1602                 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1603         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1604                 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1605         }
1606         return -EINVAL;
1607 }
1608
1609 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1610                                  struct mlxsw_sp_fib_entry *fib_entry,
1611                                  enum mlxsw_reg_ralue_op op)
1612 {
1613         switch (fib_entry->vr->proto) {
1614         case MLXSW_SP_L3_PROTO_IPV4:
1615                 return mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1616         case MLXSW_SP_L3_PROTO_IPV6:
1617                 return -EINVAL;
1618         }
1619         return -EINVAL;
1620 }
1621
1622 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1623                                      struct mlxsw_sp_fib_entry *fib_entry)
1624 {
1625         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1626                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
1627 }
1628
1629 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1630                                   struct mlxsw_sp_fib_entry *fib_entry)
1631 {
1632         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1633                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
1634 }
1635
1636 static int
1637 mlxsw_sp_router_fib4_entry_init(struct mlxsw_sp *mlxsw_sp,
1638                                 const struct fib_entry_notifier_info *fen_info,
1639                                 struct mlxsw_sp_fib_entry *fib_entry)
1640 {
1641         struct fib_info *fi = fen_info->fi;
1642         struct mlxsw_sp_rif *r = NULL;
1643         int nhsel;
1644         int err;
1645
1646         if (fen_info->type == RTN_LOCAL || fen_info->type == RTN_BROADCAST) {
1647                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1648                 return 0;
1649         }
1650         if (fen_info->type != RTN_UNICAST)
1651                 return -EINVAL;
1652
1653         for (nhsel = 0; nhsel < fi->fib_nhs; nhsel++) {
1654                 const struct fib_nh *nh = &fi->fib_nh[nhsel];
1655
1656                 if (!nh->nh_dev)
1657                         continue;
1658                 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, nh->nh_dev);
1659                 if (!r) {
1660                         /* In case router interface is not found for
1661                          * at least one of the nexthops, that means
1662                          * the nexthop points to some device unrelated
1663                          * to us. Set trap and pass the packets for
1664                          * this prefix to kernel.
1665                          */
1666                         break;
1667                 }
1668         }
1669
1670         if (!r) {
1671                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1672                 return 0;
1673         }
1674
1675         if (fi->fib_scope != RT_SCOPE_UNIVERSE) {
1676                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1677                 fib_entry->rif = r->rif;
1678         } else {
1679                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1680                 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fi);
1681                 if (err)
1682                         return err;
1683         }
1684         fib_info_offload_inc(fen_info->fi);
1685         return 0;
1686 }
1687
1688 static void
1689 mlxsw_sp_router_fib4_entry_fini(struct mlxsw_sp *mlxsw_sp,
1690                                 struct mlxsw_sp_fib_entry *fib_entry)
1691 {
1692         if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1693                 fib_info_offload_dec(fib_entry->fi);
1694         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_REMOTE)
1695                 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1696 }
1697
1698 static struct mlxsw_sp_fib_entry *
1699 mlxsw_sp_fib_entry_get(struct mlxsw_sp *mlxsw_sp,
1700                        const struct fib_entry_notifier_info *fen_info)
1701 {
1702         struct mlxsw_sp_fib_entry *fib_entry;
1703         struct fib_info *fi = fen_info->fi;
1704         struct mlxsw_sp_vr *vr;
1705         int err;
1706
1707         vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->dst_len, fen_info->tb_id,
1708                              MLXSW_SP_L3_PROTO_IPV4);
1709         if (IS_ERR(vr))
1710                 return ERR_CAST(vr);
1711
1712         fib_entry = mlxsw_sp_fib_entry_lookup(vr->fib, &fen_info->dst,
1713                                               sizeof(fen_info->dst),
1714                                               fen_info->dst_len, fi->fib_dev);
1715         if (fib_entry) {
1716                 /* Already exists, just take a reference */
1717                 fib_entry->ref_count++;
1718                 return fib_entry;
1719         }
1720         fib_entry = mlxsw_sp_fib_entry_create(vr->fib, &fen_info->dst,
1721                                               sizeof(fen_info->dst),
1722                                               fen_info->dst_len, fi->fib_dev);
1723         if (!fib_entry) {
1724                 err = -ENOMEM;
1725                 goto err_fib_entry_create;
1726         }
1727         fib_entry->vr = vr;
1728         fib_entry->fi = fi;
1729         fib_entry->ref_count = 1;
1730
1731         err = mlxsw_sp_router_fib4_entry_init(mlxsw_sp, fen_info, fib_entry);
1732         if (err)
1733                 goto err_fib4_entry_init;
1734
1735         return fib_entry;
1736
1737 err_fib4_entry_init:
1738         mlxsw_sp_fib_entry_destroy(fib_entry);
1739 err_fib_entry_create:
1740         mlxsw_sp_vr_put(mlxsw_sp, vr);
1741
1742         return ERR_PTR(err);
1743 }
1744
1745 static struct mlxsw_sp_fib_entry *
1746 mlxsw_sp_fib_entry_find(struct mlxsw_sp *mlxsw_sp,
1747                         const struct fib_entry_notifier_info *fen_info)
1748 {
1749         struct mlxsw_sp_vr *vr;
1750
1751         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id,
1752                               MLXSW_SP_L3_PROTO_IPV4);
1753         if (!vr)
1754                 return NULL;
1755
1756         return mlxsw_sp_fib_entry_lookup(vr->fib, &fen_info->dst,
1757                                          sizeof(fen_info->dst),
1758                                          fen_info->dst_len,
1759                                          fen_info->fi->fib_dev);
1760 }
1761
1762 static void mlxsw_sp_fib_entry_put(struct mlxsw_sp *mlxsw_sp,
1763                                    struct mlxsw_sp_fib_entry *fib_entry)
1764 {
1765         struct mlxsw_sp_vr *vr = fib_entry->vr;
1766
1767         if (--fib_entry->ref_count == 0) {
1768                 mlxsw_sp_router_fib4_entry_fini(mlxsw_sp, fib_entry);
1769                 mlxsw_sp_fib_entry_destroy(fib_entry);
1770         }
1771         mlxsw_sp_vr_put(mlxsw_sp, vr);
1772 }
1773
1774 static void mlxsw_sp_fib_entry_put_all(struct mlxsw_sp *mlxsw_sp,
1775                                        struct mlxsw_sp_fib_entry *fib_entry)
1776 {
1777         unsigned int last_ref_count;
1778
1779         do {
1780                 last_ref_count = fib_entry->ref_count;
1781                 mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1782         } while (last_ref_count != 1);
1783 }
1784
1785 static int mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
1786                                     struct fib_entry_notifier_info *fen_info)
1787 {
1788         struct mlxsw_sp_fib_entry *fib_entry;
1789         struct mlxsw_sp_vr *vr;
1790         int err;
1791
1792         if (mlxsw_sp->router.aborted)
1793                 return 0;
1794
1795         fib_entry = mlxsw_sp_fib_entry_get(mlxsw_sp, fen_info);
1796         if (IS_ERR(fib_entry)) {
1797                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB4 entry being added.\n");
1798                 return PTR_ERR(fib_entry);
1799         }
1800
1801         if (fib_entry->ref_count != 1)
1802                 return 0;
1803
1804         vr = fib_entry->vr;
1805         err = mlxsw_sp_fib_entry_insert(vr->fib, fib_entry);
1806         if (err) {
1807                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to insert FIB4 entry being added.\n");
1808                 goto err_fib_entry_insert;
1809         }
1810         err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1811         if (err)
1812                 goto err_fib_entry_add;
1813         return 0;
1814
1815 err_fib_entry_add:
1816         mlxsw_sp_fib_entry_remove(vr->fib, fib_entry);
1817 err_fib_entry_insert:
1818         mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1819         return err;
1820 }
1821
1822 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
1823                                      struct fib_entry_notifier_info *fen_info)
1824 {
1825         struct mlxsw_sp_fib_entry *fib_entry;
1826
1827         if (mlxsw_sp->router.aborted)
1828                 return;
1829
1830         fib_entry = mlxsw_sp_fib_entry_find(mlxsw_sp, fen_info);
1831         if (!fib_entry)
1832                 return;
1833
1834         if (fib_entry->ref_count == 1) {
1835                 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
1836                 mlxsw_sp_fib_entry_remove(fib_entry->vr->fib, fib_entry);
1837         }
1838
1839         mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1840 }
1841
1842 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
1843 {
1844         char ralta_pl[MLXSW_REG_RALTA_LEN];
1845         char ralst_pl[MLXSW_REG_RALST_LEN];
1846         char raltb_pl[MLXSW_REG_RALTB_LEN];
1847         char ralue_pl[MLXSW_REG_RALUE_LEN];
1848         int err;
1849
1850         mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
1851                              MLXSW_SP_LPM_TREE_MIN);
1852         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
1853         if (err)
1854                 return err;
1855
1856         mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
1857         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
1858         if (err)
1859                 return err;
1860
1861         mlxsw_reg_raltb_pack(raltb_pl, 0, MLXSW_REG_RALXX_PROTOCOL_IPV4,
1862                              MLXSW_SP_LPM_TREE_MIN);
1863         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
1864         if (err)
1865                 return err;
1866
1867         mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
1868                               MLXSW_REG_RALUE_OP_WRITE_WRITE, 0, 0, 0);
1869         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1870         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1871 }
1872
1873 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
1874 {
1875         struct mlxsw_resources *resources;
1876         struct mlxsw_sp_fib_entry *fib_entry;
1877         struct mlxsw_sp_fib_entry *tmp;
1878         struct mlxsw_sp_vr *vr;
1879         int i;
1880
1881         resources = mlxsw_core_resources_get(mlxsw_sp->core);
1882         for (i = 0; i < resources->max_virtual_routers; i++) {
1883                 vr = &mlxsw_sp->router.vrs[i];
1884
1885                 if (!vr->used)
1886                         continue;
1887
1888                 list_for_each_entry_safe(fib_entry, tmp,
1889                                          &vr->fib->entry_list, list) {
1890                         bool do_break = &tmp->list == &vr->fib->entry_list;
1891
1892                         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
1893                         mlxsw_sp_fib_entry_remove(fib_entry->vr->fib,
1894                                                   fib_entry);
1895                         mlxsw_sp_fib_entry_put_all(mlxsw_sp, fib_entry);
1896                         if (do_break)
1897                                 break;
1898                 }
1899         }
1900 }
1901
1902 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
1903 {
1904         int err;
1905
1906         mlxsw_sp_router_fib_flush(mlxsw_sp);
1907         mlxsw_sp->router.aborted = true;
1908         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
1909         if (err)
1910                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
1911 }
1912
1913 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
1914 {
1915         struct mlxsw_resources *resources;
1916         char rgcr_pl[MLXSW_REG_RGCR_LEN];
1917         int err;
1918
1919         resources = mlxsw_core_resources_get(mlxsw_sp->core);
1920         if (!resources->max_rif_valid)
1921                 return -EIO;
1922
1923         mlxsw_sp->rifs = kcalloc(resources->max_rif,
1924                                  sizeof(struct mlxsw_sp_rif *), GFP_KERNEL);
1925         if (!mlxsw_sp->rifs)
1926                 return -ENOMEM;
1927
1928         mlxsw_reg_rgcr_pack(rgcr_pl, true);
1929         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, resources->max_rif);
1930         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
1931         if (err)
1932                 goto err_rgcr_fail;
1933
1934         return 0;
1935
1936 err_rgcr_fail:
1937         kfree(mlxsw_sp->rifs);
1938         return err;
1939 }
1940
1941 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
1942 {
1943         struct mlxsw_resources *resources;
1944         char rgcr_pl[MLXSW_REG_RGCR_LEN];
1945         int i;
1946
1947         mlxsw_reg_rgcr_pack(rgcr_pl, false);
1948         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
1949
1950         resources = mlxsw_core_resources_get(mlxsw_sp->core);
1951         for (i = 0; i < resources->max_rif; i++)
1952                 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
1953
1954         kfree(mlxsw_sp->rifs);
1955 }
1956
1957 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
1958                                      unsigned long event, void *ptr)
1959 {
1960         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
1961         struct fib_entry_notifier_info *fen_info = ptr;
1962         int err;
1963
1964         if (!net_eq(fen_info->info.net, &init_net))
1965                 return NOTIFY_DONE;
1966
1967         switch (event) {
1968         case FIB_EVENT_ENTRY_ADD:
1969                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, fen_info);
1970                 if (err)
1971                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
1972                 break;
1973         case FIB_EVENT_ENTRY_DEL:
1974                 mlxsw_sp_router_fib4_del(mlxsw_sp, fen_info);
1975                 break;
1976         case FIB_EVENT_RULE_ADD: /* fall through */
1977         case FIB_EVENT_RULE_DEL:
1978                 mlxsw_sp_router_fib4_abort(mlxsw_sp);
1979                 break;
1980         }
1981         return NOTIFY_DONE;
1982 }
1983
1984 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
1985 {
1986         int err;
1987
1988         INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
1989         INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_group_list);
1990         err = __mlxsw_sp_router_init(mlxsw_sp);
1991         if (err)
1992                 return err;
1993
1994         mlxsw_sp_lpm_init(mlxsw_sp);
1995         err = mlxsw_sp_vrs_init(mlxsw_sp);
1996         if (err)
1997                 goto err_vrs_init;
1998
1999         err =  mlxsw_sp_neigh_init(mlxsw_sp);
2000         if (err)
2001                 goto err_neigh_init;
2002
2003         mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
2004         register_fib_notifier(&mlxsw_sp->fib_nb);
2005         return 0;
2006
2007 err_neigh_init:
2008         mlxsw_sp_vrs_fini(mlxsw_sp);
2009 err_vrs_init:
2010         __mlxsw_sp_router_fini(mlxsw_sp);
2011         return err;
2012 }
2013
2014 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
2015 {
2016         unregister_fib_notifier(&mlxsw_sp->fib_nb);
2017         mlxsw_sp_neigh_fini(mlxsw_sp);
2018         mlxsw_sp_vrs_fini(mlxsw_sp);
2019         __mlxsw_sp_router_fini(mlxsw_sp);
2020 }