drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  * 3. Neither the names of the copyright holders nor the names of its
  18  *    contributors may be used to endorse or promote products derived from
  19  *    this software without specific prior written permission.
  20  *
  21  * Alternatively, this software may be distributed under the terms of the
  22  * GNU General Public License ("GPL") version 2 as published by the Free
  23  * Software Foundation.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  26  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  29  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  35  * POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 #include <linux/kernel.h>
  39 #include <linux/types.h>
  40 #include <linux/rhashtable.h>
  41 #include <linux/bitops.h>
  42 #include <linux/in6.h>
  43 #include <linux/notifier.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/netdevice.h>
  46 #include <linux/if_bridge.h>
  47 #include <linux/socket.h>
  48 #include <linux/route.h>
  49 #include <net/netevent.h>
  50 #include <net/neighbour.h>
  51 #include <net/arp.h>
  52 #include <net/ip_fib.h>
  53 #include <net/ip6_fib.h>
  54 #include <net/fib_rules.h>
  55 #include <net/ip_tunnels.h>
  56 #include <net/l3mdev.h>
  57 #include <net/addrconf.h>
  58 #include <net/ndisc.h>
  59 #include <net/ipv6.h>
  60 #include <net/fib_notifier.h>
  61
  62 #include "spectrum.h"
  63 #include "core.h"
  64 #include "reg.h"
  65 #include "spectrum_cnt.h"
  66 #include "spectrum_dpipe.h"
  67 #include "spectrum_ipip.h"
  68 #include "spectrum_router.h"
  69
  70 struct mlxsw_sp_vr;
  71 struct mlxsw_sp_lpm_tree;
  72 struct mlxsw_sp_rif_ops;
  73
  74 struct mlxsw_sp_router {
  75         struct mlxsw_sp *mlxsw_sp;
  76         struct mlxsw_sp_rif **rifs;
  77         struct mlxsw_sp_vr *vrs;
  78         struct rhashtable neigh_ht;
  79         struct rhashtable nexthop_group_ht;
  80         struct rhashtable nexthop_ht;
  81         struct {
  82                 struct mlxsw_sp_lpm_tree *trees;
  83                 unsigned int tree_count;
  84         } lpm;
  85         struct {
  86                 struct delayed_work dw;
  87                 unsigned long interval; /* ms */
  88         } neighs_update;
  89         struct delayed_work nexthop_probe_dw;
  90 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  91         struct list_head nexthop_neighs_list;
  92         struct list_head ipip_list;
  93         bool aborted;
  94         struct notifier_block fib_nb;
  95         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  96         const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
  97 };
  98
  99 struct mlxsw_sp_rif {
 100         struct list_head nexthop_list;
 101         struct list_head neigh_list;
 102         struct net_device *dev;
 103         struct mlxsw_sp_fid *fid;
 104         unsigned char addr[ETH_ALEN];
 105         int mtu;
 106         u16 rif_index;
 107         u16 vr_id;
 108         const struct mlxsw_sp_rif_ops *ops;
 109         struct mlxsw_sp *mlxsw_sp;
 110
 111         unsigned int counter_ingress;
 112         bool counter_ingress_valid;
 113         unsigned int counter_egress;
 114         bool counter_egress_valid;
 115 };
 116
 117 struct mlxsw_sp_rif_params {
 118         struct net_device *dev;
 119         union {
 120                 u16 system_port;
 121                 u16 lag_id;
 122         };
 123         u16 vid;
 124         bool lag;
 125 };
 126
 127 struct mlxsw_sp_rif_subport {
 128         struct mlxsw_sp_rif common;
 129         union {
 130                 u16 system_port;
 131                 u16 lag_id;
 132         };
 133         u16 vid;
 134         bool lag;
 135 };
 136
 137 struct mlxsw_sp_rif_ipip_lb {
 138         struct mlxsw_sp_rif common;
 139         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 140         u16 ul_vr_id; /* Reserved for Spectrum-2. */
 141 };
 142
 143 struct mlxsw_sp_rif_params_ipip_lb {
 144         struct mlxsw_sp_rif_params common;
 145         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 146 };
 147
 148 struct mlxsw_sp_rif_ops {
 149         enum mlxsw_sp_rif_type type;
 150         size_t rif_size;
 151
 152         void (*setup)(struct mlxsw_sp_rif *rif,
 153                       const struct mlxsw_sp_rif_params *params);
 154         int (*configure)(struct mlxsw_sp_rif *rif);
 155         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 156         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 157 };
 158
 159 static unsigned int *
 160 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 161                            enum mlxsw_sp_rif_counter_dir dir)
 162 {
 163         switch (dir) {
 164         case MLXSW_SP_RIF_COUNTER_EGRESS:
 165                 return &rif->counter_egress;
 166         case MLXSW_SP_RIF_COUNTER_INGRESS:
 167                 return &rif->counter_ingress;
 168         }
 169         return NULL;
 170 }
 171
 172 static bool
 173 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 174                                enum mlxsw_sp_rif_counter_dir dir)
 175 {
 176         switch (dir) {
 177         case MLXSW_SP_RIF_COUNTER_EGRESS:
 178                 return rif->counter_egress_valid;
 179         case MLXSW_SP_RIF_COUNTER_INGRESS:
 180                 return rif->counter_ingress_valid;
 181         }
 182         return false;
 183 }
 184
 185 static void
 186 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 187                                enum mlxsw_sp_rif_counter_dir dir,
 188                                bool valid)
 189 {
 190         switch (dir) {
 191         case MLXSW_SP_RIF_COUNTER_EGRESS:
 192                 rif->counter_egress_valid = valid;
 193                 break;
 194         case MLXSW_SP_RIF_COUNTER_INGRESS:
 195                 rif->counter_ingress_valid = valid;
 196                 break;
 197         }
 198 }
 199
 200 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 201                                      unsigned int counter_index, bool enable,
 202                                      enum mlxsw_sp_rif_counter_dir dir)
 203 {
 204         char ritr_pl[MLXSW_REG_RITR_LEN];
 205         bool is_egress = false;
 206         int err;
 207
 208         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 209                 is_egress = true;
 210         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 211         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 212         if (err)
 213                 return err;
 214
 215         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 216                                     is_egress);
 217         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 218 }
 219
 220 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 221                                    struct mlxsw_sp_rif *rif,
 222                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 223 {
 224         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 225         unsigned int *p_counter_index;
 226         bool valid;
 227         int err;
 228
 229         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 230         if (!valid)
 231                 return -EINVAL;
 232
 233         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 234         if (!p_counter_index)
 235                 return -EINVAL;
 236         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 237                              MLXSW_REG_RICNT_OPCODE_NOP);
 238         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 239         if (err)
 240                 return err;
 241         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 242         return 0;
 243 }
 244
 245 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 246                                       unsigned int counter_index)
 247 {
 248         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 249
 250         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 251                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 252         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 253 }
 254
 255 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 256                                struct mlxsw_sp_rif *rif,
 257                                enum mlxsw_sp_rif_counter_dir dir)
 258 {
 259         unsigned int *p_counter_index;
 260         int err;
 261
 262         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 263         if (!p_counter_index)
 264                 return -EINVAL;
 265         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 266                                      p_counter_index);
 267         if (err)
 268                 return err;
 269
 270         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 271         if (err)
 272                 goto err_counter_clear;
 273
 274         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 275                                         *p_counter_index, true, dir);
 276         if (err)
 277                 goto err_counter_edit;
 278         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 279         return 0;
 280
 281 err_counter_edit:
 282 err_counter_clear:
 283         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 284                               *p_counter_index);
 285         return err;
 286 }
 287
 288 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 289                                struct mlxsw_sp_rif *rif,
 290                                enum mlxsw_sp_rif_counter_dir dir)
 291 {
 292         unsigned int *p_counter_index;
 293
 294         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 295                 return;
 296
 297         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 298         if (WARN_ON(!p_counter_index))
 299                 return;
 300         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 301                                   *p_counter_index, false, dir);
 302         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 303                               *p_counter_index);
 304         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 305 }
 306
 307 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 308 {
 309         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 310         struct devlink *devlink;
 311
 312         devlink = priv_to_devlink(mlxsw_sp->core);
 313         if (!devlink_dpipe_table_counter_enabled(devlink,
 314                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 315                 return;
 316         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 317 }
 318
 319 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 320 {
 321         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 322
 323         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 324 }
 325
 326 static struct mlxsw_sp_rif *
 327 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 328                          const struct net_device *dev);
 329
 330 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 331
 332 struct mlxsw_sp_prefix_usage {
 333         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 334 };
 335
 336 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 337         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 338
 339 static bool
 340 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 341                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 342 {
 343         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 344 }
 345
 346 static bool
 347 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
 348 {
 349         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
 350
 351         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 352 }
 353
 354 static void
 355 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 356                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 357 {
 358         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 359 }
 360
 361 static void
 362 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 363                           unsigned char prefix_len)
 364 {
 365         set_bit(prefix_len, prefix_usage->b);
 366 }
 367
 368 static void
 369 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 370                             unsigned char prefix_len)
 371 {
 372         clear_bit(prefix_len, prefix_usage->b);
 373 }
 374
 375 struct mlxsw_sp_fib_key {
 376         unsigned char addr[sizeof(struct in6_addr)];
 377         unsigned char prefix_len;
 378 };
 379
 380 enum mlxsw_sp_fib_entry_type {
 381         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 382         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 383         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 384
 385         /* This is a special case of local delivery, where a packet should be
 386          * decapsulated on reception. Note that there is no corresponding ENCAP,
 387          * because that's a type of next hop, not of FIB entry. (There can be
 388          * several next hops in a REMOTE entry, and some of them may be
 389          * encapsulating entries.)
 390          */
 391         MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 392 };
 393
 394 struct mlxsw_sp_nexthop_group;
 395 struct mlxsw_sp_fib;
 396
 397 struct mlxsw_sp_fib_node {
 398         struct list_head entry_list;
 399         struct list_head list;
 400         struct rhash_head ht_node;
 401         struct mlxsw_sp_fib *fib;
 402         struct mlxsw_sp_fib_key key;
 403 };
 404
 405 struct mlxsw_sp_fib_entry_decap {
 406         struct mlxsw_sp_ipip_entry *ipip_entry;
 407         u32 tunnel_index;
 408 };
 409
 410 struct mlxsw_sp_fib_entry {
 411         struct list_head list;
 412         struct mlxsw_sp_fib_node *fib_node;
 413         enum mlxsw_sp_fib_entry_type type;
 414         struct list_head nexthop_group_node;
 415         struct mlxsw_sp_nexthop_group *nh_group;
 416         struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 417 };
 418
 419 struct mlxsw_sp_fib4_entry {
 420         struct mlxsw_sp_fib_entry common;
 421         u32 tb_id;
 422         u32 prio;
 423         u8 tos;
 424         u8 type;
 425 };
 426
 427 struct mlxsw_sp_fib6_entry {
 428         struct mlxsw_sp_fib_entry common;
 429         struct list_head rt6_list;
 430         unsigned int nrt6;
 431 };
 432
 433 struct mlxsw_sp_rt6 {
 434         struct list_head list;
 435         struct rt6_info *rt;
 436 };
 437
 438 struct mlxsw_sp_lpm_tree {
 439         u8 id; /* tree ID */
 440         unsigned int ref_count;
 441         enum mlxsw_sp_l3proto proto;
 442         struct mlxsw_sp_prefix_usage prefix_usage;
 443 };
 444
 445 struct mlxsw_sp_fib {
 446         struct rhashtable ht;
 447         struct list_head node_list;
 448         struct mlxsw_sp_vr *vr;
 449         struct mlxsw_sp_lpm_tree *lpm_tree;
 450         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 451         struct mlxsw_sp_prefix_usage prefix_usage;
 452         enum mlxsw_sp_l3proto proto;
 453 };
 454
 455 struct mlxsw_sp_vr {
 456         u16 id; /* virtual router ID */
 457         u32 tb_id; /* kernel fib table id */
 458         unsigned int rif_count;
 459         struct mlxsw_sp_fib *fib4;
 460         struct mlxsw_sp_fib *fib6;
 461 };
 462
 463 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 464
 465 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 466                                                 enum mlxsw_sp_l3proto proto)
 467 {
 468         struct mlxsw_sp_fib *fib;
 469         int err;
 470
 471         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 472         if (!fib)
 473                 return ERR_PTR(-ENOMEM);
 474         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 475         if (err)
 476                 goto err_rhashtable_init;
 477         INIT_LIST_HEAD(&fib->node_list);
 478         fib->proto = proto;
 479         fib->vr = vr;
 480         return fib;
 481
 482 err_rhashtable_init:
 483         kfree(fib);
 484         return ERR_PTR(err);
 485 }
 486
 487 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 488 {
 489         WARN_ON(!list_empty(&fib->node_list));
 490         WARN_ON(fib->lpm_tree);
 491         rhashtable_destroy(&fib->ht);
 492         kfree(fib);
 493 }
 494
 495 static struct mlxsw_sp_lpm_tree *
 496 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 497 {
 498         static struct mlxsw_sp_lpm_tree *lpm_tree;
 499         int i;
 500
 501         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 502                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 503                 if (lpm_tree->ref_count == 0)
 504                         return lpm_tree;
 505         }
 506         return NULL;
 507 }
 508
 509 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 510                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 511 {
 512         char ralta_pl[MLXSW_REG_RALTA_LEN];
 513
 514         mlxsw_reg_ralta_pack(ralta_pl, true,
 515                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 516                              lpm_tree->id);
 517         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 518 }
 519
 520 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 521                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 522 {
 523         char ralta_pl[MLXSW_REG_RALTA_LEN];
 524
 525         mlxsw_reg_ralta_pack(ralta_pl, false,
 526                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 527                              lpm_tree->id);
 528         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 529 }
 530
 531 static int
 532 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 533                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 534                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 535 {
 536         char ralst_pl[MLXSW_REG_RALST_LEN];
 537         u8 root_bin = 0;
 538         u8 prefix;
 539         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 540
 541         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 542                 root_bin = prefix;
 543
 544         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 545         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 546                 if (prefix == 0)
 547                         continue;
 548                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 549                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 550                 last_prefix = prefix;
 551         }
 552         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 553 }
 554
 555 static struct mlxsw_sp_lpm_tree *
 556 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 557                          struct mlxsw_sp_prefix_usage *prefix_usage,
 558                          enum mlxsw_sp_l3proto proto)
 559 {
 560         struct mlxsw_sp_lpm_tree *lpm_tree;
 561         int err;
 562
 563         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 564         if (!lpm_tree)
 565                 return ERR_PTR(-EBUSY);
 566         lpm_tree->proto = proto;
 567         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 568         if (err)
 569                 return ERR_PTR(err);
 570
 571         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 572                                                 lpm_tree);
 573         if (err)
 574                 goto err_left_struct_set;
 575         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 576                sizeof(lpm_tree->prefix_usage));
 577         return lpm_tree;
 578
 579 err_left_struct_set:
 580         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 581         return ERR_PTR(err);
 582 }
 583
 584 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 585                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 586 {
 587         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 588 }
 589
 590 static struct mlxsw_sp_lpm_tree *
 591 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 592                       struct mlxsw_sp_prefix_usage *prefix_usage,
 593                       enum mlxsw_sp_l3proto proto)
 594 {
 595         struct mlxsw_sp_lpm_tree *lpm_tree;
 596         int i;
 597
 598         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 599                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 600                 if (lpm_tree->ref_count != 0 &&
 601                     lpm_tree->proto == proto &&
 602                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 603                                              prefix_usage))
 604                         return lpm_tree;
 605         }
 606         return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
 607 }
 608
 609 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 610 {
 611         lpm_tree->ref_count++;
 612 }
 613
 614 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 615                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 616 {
 617         if (--lpm_tree->ref_count == 0)
 618                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 619 }
 620
 621 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 622
 623 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 624 {
 625         struct mlxsw_sp_lpm_tree *lpm_tree;
 626         u64 max_trees;
 627         int i;
 628
 629         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 630                 return -EIO;
 631
 632         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 633         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 634         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 635                                              sizeof(struct mlxsw_sp_lpm_tree),
 636                                              GFP_KERNEL);
 637         if (!mlxsw_sp->router->lpm.trees)
 638                 return -ENOMEM;
 639
 640         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 641                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 642                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 643         }
 644
 645         return 0;
 646 }
 647
 648 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 649 {
 650         kfree(mlxsw_sp->router->lpm.trees);
 651 }
 652
 653 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 654 {
 655         return !!vr->fib4 || !!vr->fib6;
 656 }
 657
 658 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 659 {
 660         struct mlxsw_sp_vr *vr;
 661         int i;
 662
 663         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 664                 vr = &mlxsw_sp->router->vrs[i];
 665                 if (!mlxsw_sp_vr_is_used(vr))
 666                         return vr;
 667         }
 668         return NULL;
 669 }
 670
 671 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 672                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 673 {
 674         char raltb_pl[MLXSW_REG_RALTB_LEN];
 675
 676         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 677                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 678                              tree_id);
 679         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 680 }
 681
 682 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 683                                        const struct mlxsw_sp_fib *fib)
 684 {
 685         char raltb_pl[MLXSW_REG_RALTB_LEN];
 686
 687         /* Bind to tree 0 which is default */
 688         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 689                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 690         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 691 }
 692
 693 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 694 {
 695         /* For our purpose, squash main and local table into one */
 696         if (tb_id == RT_TABLE_LOCAL)
 697                 tb_id = RT_TABLE_MAIN;
 698         return tb_id;
 699 }
 700
 701 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 702                                             u32 tb_id)
 703 {
 704         struct mlxsw_sp_vr *vr;
 705         int i;
 706
 707         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 708
 709         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 710                 vr = &mlxsw_sp->router->vrs[i];
 711                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 712                         return vr;
 713         }
 714         return NULL;
 715 }
 716
 717 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 718                                             enum mlxsw_sp_l3proto proto)
 719 {
 720         switch (proto) {
 721         case MLXSW_SP_L3_PROTO_IPV4:
 722                 return vr->fib4;
 723         case MLXSW_SP_L3_PROTO_IPV6:
 724                 return vr->fib6;
 725         }
 726         return NULL;
 727 }
 728
 729 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 730                                               u32 tb_id)
 731 {
 732         struct mlxsw_sp_fib *fib4;
 733         struct mlxsw_sp_fib *fib6;
 734         struct mlxsw_sp_vr *vr;
 735         int err;
 736
 737         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 738         if (!vr)
 739                 return ERR_PTR(-EBUSY);
 740         fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 741         if (IS_ERR(fib4))
 742                 return ERR_CAST(fib4);
 743         fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
 744         if (IS_ERR(fib6)) {
 745                 err = PTR_ERR(fib6);
 746                 goto err_fib6_create;
 747         }
 748         vr->fib4 = fib4;
 749         vr->fib6 = fib6;
 750         vr->tb_id = tb_id;
 751         return vr;
 752
 753 err_fib6_create:
 754         mlxsw_sp_fib_destroy(fib4);
 755         return ERR_PTR(err);
 756 }
 757
 758 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 759 {
 760         mlxsw_sp_fib_destroy(vr->fib6);
 761         vr->fib6 = NULL;
 762         mlxsw_sp_fib_destroy(vr->fib4);
 763         vr->fib4 = NULL;
 764 }
 765
 766 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 767 {
 768         struct mlxsw_sp_vr *vr;
 769
 770         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 771         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 772         if (!vr)
 773                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 774         return vr;
 775 }
 776
 777 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 778 {
 779         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 780             list_empty(&vr->fib6->node_list))
 781                 mlxsw_sp_vr_destroy(vr);
 782 }
 783
 784 static bool
 785 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 786                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 787 {
 788         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 789
 790         if (!mlxsw_sp_vr_is_used(vr))
 791                 return false;
 792         if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
 793                 return true;
 794         return false;
 795 }
 796
 797 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 798                                         struct mlxsw_sp_fib *fib,
 799                                         struct mlxsw_sp_lpm_tree *new_tree)
 800 {
 801         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 802         int err;
 803
 804         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 805         if (err)
 806                 return err;
 807         fib->lpm_tree = new_tree;
 808         mlxsw_sp_lpm_tree_hold(new_tree);
 809         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 810         return 0;
 811 }
 812
 813 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 814                                          struct mlxsw_sp_fib *fib,
 815                                          struct mlxsw_sp_lpm_tree *new_tree)
 816 {
 817         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 818         enum mlxsw_sp_l3proto proto = fib->proto;
 819         u8 old_id, new_id = new_tree->id;
 820         struct mlxsw_sp_vr *vr;
 821         int i, err;
 822
 823         if (!old_tree)
 824                 goto no_replace;
 825         old_id = old_tree->id;
 826
 827         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 828                 vr = &mlxsw_sp->router->vrs[i];
 829                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 830                         continue;
 831                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 832                                                    mlxsw_sp_vr_fib(vr, proto),
 833                                                    new_tree);
 834                 if (err)
 835                         goto err_tree_replace;
 836         }
 837
 838         return 0;
 839
 840 err_tree_replace:
 841         for (i--; i >= 0; i--) {
 842                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 843                         continue;
 844                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 845                                              mlxsw_sp_vr_fib(vr, proto),
 846                                              old_tree);
 847         }
 848         return err;
 849
 850 no_replace:
 851         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 852         if (err)
 853                 return err;
 854         fib->lpm_tree = new_tree;
 855         mlxsw_sp_lpm_tree_hold(new_tree);
 856         return 0;
 857 }
 858
 859 static void
 860 mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
 861                       enum mlxsw_sp_l3proto proto,
 862                       struct mlxsw_sp_prefix_usage *req_prefix_usage)
 863 {
 864         int i;
 865
 866         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 867                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
 868                 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 869                 unsigned char prefix;
 870
 871                 if (!mlxsw_sp_vr_is_used(vr))
 872                         continue;
 873                 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
 874                         mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
 875         }
 876 }
 877
 878 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 879 {
 880         struct mlxsw_sp_vr *vr;
 881         u64 max_vrs;
 882         int i;
 883
 884         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 885                 return -EIO;
 886
 887         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 888         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 889                                         GFP_KERNEL);
 890         if (!mlxsw_sp->router->vrs)
 891                 return -ENOMEM;
 892
 893         for (i = 0; i < max_vrs; i++) {
 894                 vr = &mlxsw_sp->router->vrs[i];
 895                 vr->id = i;
 896         }
 897
 898         return 0;
 899 }
 900
 901 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 902
 903 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 904 {
 905         /* At this stage we're guaranteed not to have new incoming
 906          * FIB notifications and the work queue is free from FIBs
 907          * sitting on top of mlxsw netdevs. However, we can still
 908          * have other FIBs queued. Flush the queue before flushing
 909          * the device's tables. No need for locks, as we're the only
 910          * writer.
 911          */
 912         mlxsw_core_flush_owq();
 913         mlxsw_sp_router_fib_flush(mlxsw_sp);
 914         kfree(mlxsw_sp->router->vrs);
 915 }
 916
 917 static struct net_device *
 918 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
 919 {
 920         struct ip_tunnel *tun = netdev_priv(ol_dev);
 921         struct net *net = dev_net(ol_dev);
 922
 923         return __dev_get_by_index(net, tun->parms.link);
 924 }
 925
 926 static u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
 927 {
 928         struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
 929
 930         if (d)
 931                 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
 932         else
 933                 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
 934 }
 935
 936 static struct mlxsw_sp_rif *
 937 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 938                     const struct mlxsw_sp_rif_params *params);
 939
 940 static struct mlxsw_sp_rif_ipip_lb *
 941 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
 942                                 enum mlxsw_sp_ipip_type ipipt,
 943                                 struct net_device *ol_dev)
 944 {
 945         struct mlxsw_sp_rif_params_ipip_lb lb_params;
 946         const struct mlxsw_sp_ipip_ops *ipip_ops;
 947         struct mlxsw_sp_rif *rif;
 948
 949         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
 950         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
 951                 .common.dev = ol_dev,
 952                 .common.lag = false,
 953                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
 954         };
 955
 956         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common);
 957         if (IS_ERR(rif))
 958                 return ERR_CAST(rif);
 959         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
 960 }
 961
 962 static struct mlxsw_sp_ipip_entry *
 963 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
 964                           enum mlxsw_sp_ipip_type ipipt,
 965                           struct net_device *ol_dev)
 966 {
 967         struct mlxsw_sp_ipip_entry *ipip_entry;
 968         struct mlxsw_sp_ipip_entry *ret = NULL;
 969
 970         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
 971         if (!ipip_entry)
 972                 return ERR_PTR(-ENOMEM);
 973
 974         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
 975                                                             ol_dev);
 976         if (IS_ERR(ipip_entry->ol_lb)) {
 977                 ret = ERR_CAST(ipip_entry->ol_lb);
 978                 goto err_ol_ipip_lb_create;
 979         }
 980
 981         ipip_entry->ipipt = ipipt;
 982         ipip_entry->ol_dev = ol_dev;
 983
 984         return ipip_entry;
 985
 986 err_ol_ipip_lb_create:
 987         kfree(ipip_entry);
 988         return ret;
 989 }
 990
 991 static void
 992 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp_ipip_entry *ipip_entry)
 993 {
 994         WARN_ON(ipip_entry->ref_count > 0);
 995         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
 996         kfree(ipip_entry);
 997 }
 998
 999 static __be32
1000 mlxsw_sp_ipip_netdev_saddr4(const struct net_device *ol_dev)
1001 {
1002         struct ip_tunnel *tun = netdev_priv(ol_dev);
1003
1004         return tun->parms.iph.saddr;
1005 }
1006
1007 union mlxsw_sp_l3addr
1008 mlxsw_sp_ipip_netdev_saddr(enum mlxsw_sp_l3proto proto,
1009                            const struct net_device *ol_dev)
1010 {
1011         switch (proto) {
1012         case MLXSW_SP_L3_PROTO_IPV4:
1013                 return (union mlxsw_sp_l3addr) {
1014                         .addr4 = mlxsw_sp_ipip_netdev_saddr4(ol_dev),
1015                 };
1016         case MLXSW_SP_L3_PROTO_IPV6:
1017                 break;
1018         };
1019
1020         WARN_ON(1);
1021         return (union mlxsw_sp_l3addr) {
1022                 .addr4 = 0,
1023         };
1024 }
1025
1026 __be32 mlxsw_sp_ipip_netdev_daddr4(const struct net_device *ol_dev)
1027 {
1028         struct ip_tunnel *tun = netdev_priv(ol_dev);
1029
1030         return tun->parms.iph.daddr;
1031 }
1032
1033 union mlxsw_sp_l3addr
1034 mlxsw_sp_ipip_netdev_daddr(enum mlxsw_sp_l3proto proto,
1035                            const struct net_device *ol_dev)
1036 {
1037         switch (proto) {
1038         case MLXSW_SP_L3_PROTO_IPV4:
1039                 return (union mlxsw_sp_l3addr) {
1040                         .addr4 = mlxsw_sp_ipip_netdev_daddr4(ol_dev),
1041                 };
1042         case MLXSW_SP_L3_PROTO_IPV6:
1043                 break;
1044         };
1045
1046         WARN_ON(1);
1047         return (union mlxsw_sp_l3addr) {
1048                 .addr4 = 0,
1049         };
1050 }
1051
1052 static bool mlxsw_sp_l3addr_eq(const union mlxsw_sp_l3addr *addr1,
1053                                const union mlxsw_sp_l3addr *addr2)
1054 {
1055         return !memcmp(addr1, addr2, sizeof(*addr1));
1056 }
1057
1058 static bool
1059 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1060                                   const enum mlxsw_sp_l3proto ul_proto,
1061                                   union mlxsw_sp_l3addr saddr,
1062                                   u32 ul_tb_id,
1063                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1064 {
1065         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1066         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1067         union mlxsw_sp_l3addr tun_saddr;
1068
1069         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1070                 return false;
1071
1072         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1073         return tun_ul_tb_id == ul_tb_id &&
1074                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1075 }
1076
1077 static int
1078 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1079                               struct mlxsw_sp_fib_entry *fib_entry,
1080                               struct mlxsw_sp_ipip_entry *ipip_entry)
1081 {
1082         u32 tunnel_index;
1083         int err;
1084
1085         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, 1, &tunnel_index);
1086         if (err)
1087                 return err;
1088
1089         ipip_entry->decap_fib_entry = fib_entry;
1090         fib_entry->decap.ipip_entry = ipip_entry;
1091         fib_entry->decap.tunnel_index = tunnel_index;
1092         return 0;
1093 }
1094
1095 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1096                                           struct mlxsw_sp_fib_entry *fib_entry)
1097 {
1098         /* Unlink this node from the IPIP entry that it's the decap entry of. */
1099         fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1100         fib_entry->decap.ipip_entry = NULL;
1101         mlxsw_sp_kvdl_free(mlxsw_sp, fib_entry->decap.tunnel_index);
1102 }
1103
1104 static struct mlxsw_sp_fib_node *
1105 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1106                          size_t addr_len, unsigned char prefix_len);
1107 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1108                                      struct mlxsw_sp_fib_entry *fib_entry);
1109
1110 static void
1111 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1112                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1113 {
1114         struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1115
1116         mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1117         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1118
1119         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1120 }
1121
1122 static void
1123 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1124                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1125                                   struct mlxsw_sp_fib_entry *decap_fib_entry)
1126 {
1127         if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1128                                           ipip_entry))
1129                 return;
1130         decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1131
1132         if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1133                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1134 }
1135
1136 /* Given an IPIP entry, find the corresponding decap route. */
1137 static struct mlxsw_sp_fib_entry *
1138 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1139                                struct mlxsw_sp_ipip_entry *ipip_entry)
1140 {
1141         static struct mlxsw_sp_fib_node *fib_node;
1142         const struct mlxsw_sp_ipip_ops *ipip_ops;
1143         struct mlxsw_sp_fib_entry *fib_entry;
1144         unsigned char saddr_prefix_len;
1145         union mlxsw_sp_l3addr saddr;
1146         struct mlxsw_sp_fib *ul_fib;
1147         struct mlxsw_sp_vr *ul_vr;
1148         const void *saddrp;
1149         size_t saddr_len;
1150         u32 ul_tb_id;
1151         u32 saddr4;
1152
1153         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1154
1155         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1156         ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1157         if (!ul_vr)
1158                 return NULL;
1159
1160         ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1161         saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1162                                            ipip_entry->ol_dev);
1163
1164         switch (ipip_ops->ul_proto) {
1165         case MLXSW_SP_L3_PROTO_IPV4:
1166                 saddr4 = be32_to_cpu(saddr.addr4);
1167                 saddrp = &saddr4;
1168                 saddr_len = 4;
1169                 saddr_prefix_len = 32;
1170                 break;
1171         case MLXSW_SP_L3_PROTO_IPV6:
1172                 WARN_ON(1);
1173                 return NULL;
1174         }
1175
1176         fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1177                                             saddr_prefix_len);
1178         if (!fib_node || list_empty(&fib_node->entry_list))
1179                 return NULL;
1180
1181         fib_entry = list_first_entry(&fib_node->entry_list,
1182                                      struct mlxsw_sp_fib_entry, list);
1183         if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1184                 return NULL;
1185
1186         return fib_entry;
1187 }
1188
1189 static struct mlxsw_sp_ipip_entry *
1190 mlxsw_sp_ipip_entry_get(struct mlxsw_sp *mlxsw_sp,
1191                         enum mlxsw_sp_ipip_type ipipt,
1192                         struct net_device *ol_dev)
1193 {
1194         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1195         struct mlxsw_sp_router *router = mlxsw_sp->router;
1196         struct mlxsw_sp_fib_entry *decap_fib_entry;
1197         struct mlxsw_sp_ipip_entry *ipip_entry;
1198         enum mlxsw_sp_l3proto ul_proto;
1199         union mlxsw_sp_l3addr saddr;
1200
1201         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1202                             ipip_list_node) {
1203                 if (ipip_entry->ol_dev == ol_dev)
1204                         goto inc_ref_count;
1205
1206                 /* The configuration where several tunnels have the same local
1207                  * address in the same underlay table needs special treatment in
1208                  * the HW. That is currently not implemented in the driver.
1209                  */
1210                 ul_proto = router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1211                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1212                 if (mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1213                                                       ul_tb_id, ipip_entry))
1214                         return ERR_PTR(-EEXIST);
1215         }
1216
1217         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1218         if (IS_ERR(ipip_entry))
1219                 return ipip_entry;
1220
1221         decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1222         if (decap_fib_entry)
1223                 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1224                                                   decap_fib_entry);
1225
1226         list_add_tail(&ipip_entry->ipip_list_node,
1227                       &mlxsw_sp->router->ipip_list);
1228
1229 inc_ref_count:
1230         ++ipip_entry->ref_count;
1231         return ipip_entry;
1232 }
1233
1234 static void
1235 mlxsw_sp_ipip_entry_put(struct mlxsw_sp *mlxsw_sp,
1236                         struct mlxsw_sp_ipip_entry *ipip_entry)
1237 {
1238         if (--ipip_entry->ref_count == 0) {
1239                 list_del(&ipip_entry->ipip_list_node);
1240                 if (ipip_entry->decap_fib_entry)
1241                         mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1242                 mlxsw_sp_ipip_entry_destroy(ipip_entry);
1243         }
1244 }
1245
1246 static bool
1247 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1248                                   const struct net_device *ul_dev,
1249                                   enum mlxsw_sp_l3proto ul_proto,
1250                                   union mlxsw_sp_l3addr ul_dip,
1251                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1252 {
1253         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1254         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1255
1256         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1257                 return false;
1258
1259         return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1260                                                  ul_tb_id, ipip_entry);
1261 }
1262
1263 /* Given decap parameters, find the corresponding IPIP entry. */
1264 static struct mlxsw_sp_ipip_entry *
1265 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp,
1266                                   const struct net_device *ul_dev,
1267                                   enum mlxsw_sp_l3proto ul_proto,
1268                                   union mlxsw_sp_l3addr ul_dip)
1269 {
1270         struct mlxsw_sp_ipip_entry *ipip_entry;
1271
1272         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1273                             ipip_list_node)
1274                 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1275                                                       ul_proto, ul_dip,
1276                                                       ipip_entry))
1277                         return ipip_entry;
1278
1279         return NULL;
1280 }
1281
1282 struct mlxsw_sp_neigh_key {
1283         struct neighbour *n;
1284 };
1285
1286 struct mlxsw_sp_neigh_entry {
1287         struct list_head rif_list_node;
1288         struct rhash_head ht_node;
1289         struct mlxsw_sp_neigh_key key;
1290         u16 rif;
1291         bool connected;
1292         unsigned char ha[ETH_ALEN];
1293         struct list_head nexthop_list; /* list of nexthops using
1294                                         * this neigh entry
1295                                         */
1296         struct list_head nexthop_neighs_list_node;
1297         unsigned int counter_index;
1298         bool counter_valid;
1299 };
1300
1301 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
1302         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
1303         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
1304         .key_len = sizeof(struct mlxsw_sp_neigh_key),
1305 };
1306
1307 struct mlxsw_sp_neigh_entry *
1308 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
1309                         struct mlxsw_sp_neigh_entry *neigh_entry)
1310 {
1311         if (!neigh_entry) {
1312                 if (list_empty(&rif->neigh_list))
1313                         return NULL;
1314                 else
1315                         return list_first_entry(&rif->neigh_list,
1316                                                 typeof(*neigh_entry),
1317                                                 rif_list_node);
1318         }
1319         if (neigh_entry->rif_list_node.next == &rif->neigh_list)
1320                 return NULL;
1321         return list_next_entry(neigh_entry, rif_list_node);
1322 }
1323
1324 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
1325 {
1326         return neigh_entry->key.n->tbl->family;
1327 }
1328
1329 unsigned char *
1330 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
1331 {
1332         return neigh_entry->ha;
1333 }
1334
1335 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1336 {
1337         struct neighbour *n;
1338
1339         n = neigh_entry->key.n;
1340         return ntohl(*((__be32 *) n->primary_key));
1341 }
1342
1343 struct in6_addr *
1344 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1345 {
1346         struct neighbour *n;
1347
1348         n = neigh_entry->key.n;
1349         return (struct in6_addr *) &n->primary_key;
1350 }
1351
1352 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
1353                                struct mlxsw_sp_neigh_entry *neigh_entry,
1354                                u64 *p_counter)
1355 {
1356         if (!neigh_entry->counter_valid)
1357                 return -EINVAL;
1358
1359         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
1360                                          p_counter, NULL);
1361 }
1362
1363 static struct mlxsw_sp_neigh_entry *
1364 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
1365                            u16 rif)
1366 {
1367         struct mlxsw_sp_neigh_entry *neigh_entry;
1368
1369         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
1370         if (!neigh_entry)
1371                 return NULL;
1372
1373         neigh_entry->key.n = n;
1374         neigh_entry->rif = rif;
1375         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
1376
1377         return neigh_entry;
1378 }
1379
1380 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
1381 {
1382         kfree(neigh_entry);
1383 }
1384
1385 static int
1386 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
1387                             struct mlxsw_sp_neigh_entry *neigh_entry)
1388 {
1389         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
1390                                       &neigh_entry->ht_node,
1391                                       mlxsw_sp_neigh_ht_params);
1392 }
1393
1394 static void
1395 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
1396                             struct mlxsw_sp_neigh_entry *neigh_entry)
1397 {
1398         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
1399                                &neigh_entry->ht_node,
1400                                mlxsw_sp_neigh_ht_params);
1401 }
1402
1403 static bool
1404 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
1405                                     struct mlxsw_sp_neigh_entry *neigh_entry)
1406 {
1407         struct devlink *devlink;
1408         const char *table_name;
1409
1410         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
1411         case AF_INET:
1412                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
1413                 break;
1414         case AF_INET6:
1415                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
1416                 break;
1417         default:
1418                 WARN_ON(1);
1419                 return false;
1420         }
1421
1422         devlink = priv_to_devlink(mlxsw_sp->core);
1423         return devlink_dpipe_table_counter_enabled(devlink, table_name);
1424 }
1425
1426 static void
1427 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
1428                              struct mlxsw_sp_neigh_entry *neigh_entry)
1429 {
1430         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
1431                 return;
1432
1433         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
1434                 return;
1435
1436         neigh_entry->counter_valid = true;
1437 }
1438
1439 static void
1440 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
1441                             struct mlxsw_sp_neigh_entry *neigh_entry)
1442 {
1443         if (!neigh_entry->counter_valid)
1444                 return;
1445         mlxsw_sp_flow_counter_free(mlxsw_sp,
1446                                    neigh_entry->counter_index);
1447         neigh_entry->counter_valid = false;
1448 }
1449
1450 static struct mlxsw_sp_neigh_entry *
1451 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1452 {
1453         struct mlxsw_sp_neigh_entry *neigh_entry;
1454         struct mlxsw_sp_rif *rif;
1455         int err;
1456
1457         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
1458         if (!rif)
1459                 return ERR_PTR(-EINVAL);
1460
1461         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
1462         if (!neigh_entry)
1463                 return ERR_PTR(-ENOMEM);
1464
1465         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
1466         if (err)
1467                 goto err_neigh_entry_insert;
1468
1469         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1470         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
1471
1472         return neigh_entry;
1473
1474 err_neigh_entry_insert:
1475         mlxsw_sp_neigh_entry_free(neigh_entry);
1476         return ERR_PTR(err);
1477 }
1478
1479 static void
1480 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1481                              struct mlxsw_sp_neigh_entry *neigh_entry)
1482 {
1483         list_del(&neigh_entry->rif_list_node);
1484         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1485         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
1486         mlxsw_sp_neigh_entry_free(neigh_entry);
1487 }
1488
1489 static struct mlxsw_sp_neigh_entry *
1490 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1491 {
1492         struct mlxsw_sp_neigh_key key;
1493
1494         key.n = n;
1495         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
1496                                       &key, mlxsw_sp_neigh_ht_params);
1497 }
1498
1499 static void
1500 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
1501 {
1502         unsigned long interval;
1503
1504 #if IS_ENABLED(CONFIG_IPV6)
1505         interval = min_t(unsigned long,
1506                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
1507                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
1508 #else
1509         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
1510 #endif
1511         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
1512 }
1513
1514 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1515                                                    char *rauhtd_pl,
1516                                                    int ent_index)
1517 {
1518         struct net_device *dev;
1519         struct neighbour *n;
1520         __be32 dipn;
1521         u32 dip;
1522         u16 rif;
1523
1524         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
1525
1526         if (!mlxsw_sp->router->rifs[rif]) {
1527                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1528                 return;
1529         }
1530
1531         dipn = htonl(dip);
1532         dev = mlxsw_sp->router->rifs[rif]->dev;
1533         n = neigh_lookup(&arp_tbl, &dipn, dev);
1534         if (!n)
1535                 return;
1536
1537         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
1538         neigh_event_send(n, NULL);
1539         neigh_release(n);
1540 }
1541
1542 #if IS_ENABLED(CONFIG_IPV6)
1543 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1544                                                    char *rauhtd_pl,
1545                                                    int rec_index)
1546 {
1547         struct net_device *dev;
1548         struct neighbour *n;
1549         struct in6_addr dip;
1550         u16 rif;
1551
1552         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
1553                                          (char *) &dip);
1554
1555         if (!mlxsw_sp->router->rifs[rif]) {
1556                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1557                 return;
1558         }
1559
1560         dev = mlxsw_sp->router->rifs[rif]->dev;
1561         n = neigh_lookup(&nd_tbl, &dip, dev);
1562         if (!n)
1563                 return;
1564
1565         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1566         neigh_event_send(n, NULL);
1567         neigh_release(n);
1568 }
1569 #else
1570 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1571                                                    char *rauhtd_pl,
1572                                                    int rec_index)
1573 {
1574 }
1575 #endif
1576
1577 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1578                                                    char *rauhtd_pl,
1579                                                    int rec_index)
1580 {
1581         u8 num_entries;
1582         int i;
1583
1584         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1585                                                                 rec_index);
1586         /* Hardware starts counting at 0, so add 1. */
1587         num_entries++;
1588
1589         /* Each record consists of several neighbour entries. */
1590         for (i = 0; i < num_entries; i++) {
1591                 int ent_index;
1592
1593                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1594                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1595                                                        ent_index);
1596         }
1597
1598 }
1599
1600 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1601                                                    char *rauhtd_pl,
1602                                                    int rec_index)
1603 {
1604         /* One record contains one entry. */
1605         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1606                                                rec_index);
1607 }
1608
1609 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1610                                               char *rauhtd_pl, int rec_index)
1611 {
1612         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1613         case MLXSW_REG_RAUHTD_TYPE_IPV4:
1614                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1615                                                        rec_index);
1616                 break;
1617         case MLXSW_REG_RAUHTD_TYPE_IPV6:
1618                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1619                                                        rec_index);
1620                 break;
1621         }
1622 }
1623
1624 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1625 {
1626         u8 num_rec, last_rec_index, num_entries;
1627
1628         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1629         last_rec_index = num_rec - 1;
1630
1631         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1632                 return false;
1633         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1634             MLXSW_REG_RAUHTD_TYPE_IPV6)
1635                 return true;
1636
1637         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1638                                                                 last_rec_index);
1639         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1640                 return true;
1641         return false;
1642 }
1643
1644 static int
1645 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1646                                        char *rauhtd_pl,
1647                                        enum mlxsw_reg_rauhtd_type type)
1648 {
1649         int i, num_rec;
1650         int err;
1651
1652         /* Make sure the neighbour's netdev isn't removed in the
1653          * process.
1654          */
1655         rtnl_lock();
1656         do {
1657                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
1658                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1659                                       rauhtd_pl);
1660                 if (err) {
1661                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1662                         break;
1663                 }
1664                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1665                 for (i = 0; i < num_rec; i++)
1666                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1667                                                           i);
1668         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1669         rtnl_unlock();
1670
1671         return err;
1672 }
1673
1674 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1675 {
1676         enum mlxsw_reg_rauhtd_type type;
1677         char *rauhtd_pl;
1678         int err;
1679
1680         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1681         if (!rauhtd_pl)
1682                 return -ENOMEM;
1683
1684         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1685         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1686         if (err)
1687                 goto out;
1688
1689         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1690         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1691 out:
1692         kfree(rauhtd_pl);
1693         return err;
1694 }
1695
1696 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1697 {
1698         struct mlxsw_sp_neigh_entry *neigh_entry;
1699
1700         /* Take RTNL mutex here to prevent lists from changes */
1701         rtnl_lock();
1702         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1703                             nexthop_neighs_list_node)
1704                 /* If this neigh have nexthops, make the kernel think this neigh
1705                  * is active regardless of the traffic.
1706                  */
1707                 neigh_event_send(neigh_entry->key.n, NULL);
1708         rtnl_unlock();
1709 }
1710
1711 static void
1712 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1713 {
1714         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1715
1716         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1717                                msecs_to_jiffies(interval));
1718 }
1719
1720 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1721 {
1722         struct mlxsw_sp_router *router;
1723         int err;
1724
1725         router = container_of(work, struct mlxsw_sp_router,
1726                               neighs_update.dw.work);
1727         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1728         if (err)
1729                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1730
1731         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1732
1733         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1734 }
1735
1736 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1737 {
1738         struct mlxsw_sp_neigh_entry *neigh_entry;
1739         struct mlxsw_sp_router *router;
1740
1741         router = container_of(work, struct mlxsw_sp_router,
1742                               nexthop_probe_dw.work);
1743         /* Iterate over nexthop neighbours, find those who are unresolved and
1744          * send arp on them. This solves the chicken-egg problem when
1745          * the nexthop wouldn't get offloaded until the neighbor is resolved
1746          * but it wouldn't get resolved ever in case traffic is flowing in HW
1747          * using different nexthop.
1748          *
1749          * Take RTNL mutex here to prevent lists from changes.
1750          */
1751         rtnl_lock();
1752         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1753                             nexthop_neighs_list_node)
1754                 if (!neigh_entry->connected)
1755                         neigh_event_send(neigh_entry->key.n, NULL);
1756         rtnl_unlock();
1757
1758         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1759                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1760 }
1761
1762 static void
1763 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1764                               struct mlxsw_sp_neigh_entry *neigh_entry,
1765                               bool removing, bool dead);
1766
1767 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1768 {
1769         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1770                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1771 }
1772
1773 static void
1774 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1775                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1776                                 enum mlxsw_reg_rauht_op op)
1777 {
1778         struct neighbour *n = neigh_entry->key.n;
1779         u32 dip = ntohl(*((__be32 *) n->primary_key));
1780         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1781
1782         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1783                               dip);
1784         if (neigh_entry->counter_valid)
1785                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1786                                              neigh_entry->counter_index);
1787         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1788 }
1789
1790 static void
1791 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1792                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1793                                 enum mlxsw_reg_rauht_op op)
1794 {
1795         struct neighbour *n = neigh_entry->key.n;
1796         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1797         const char *dip = n->primary_key;
1798
1799         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1800                               dip);
1801         if (neigh_entry->counter_valid)
1802                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1803                                              neigh_entry->counter_index);
1804         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1805 }
1806
1807 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
1808 {
1809         struct neighbour *n = neigh_entry->key.n;
1810
1811         /* Packets with a link-local destination address are trapped
1812          * after LPM lookup and never reach the neighbour table, so
1813          * there is no need to program such neighbours to the device.
1814          */
1815         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1816             IPV6_ADDR_LINKLOCAL)
1817                 return true;
1818         return false;
1819 }
1820
1821 static void
1822 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1823                             struct mlxsw_sp_neigh_entry *neigh_entry,
1824                             bool adding)
1825 {
1826         if (!adding && !neigh_entry->connected)
1827                 return;
1828         neigh_entry->connected = adding;
1829         if (neigh_entry->key.n->tbl->family == AF_INET) {
1830                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1831                                                 mlxsw_sp_rauht_op(adding));
1832         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
1833                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
1834                         return;
1835                 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1836                                                 mlxsw_sp_rauht_op(adding));
1837         } else {
1838                 WARN_ON_ONCE(1);
1839         }
1840 }
1841
1842 void
1843 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
1844                                     struct mlxsw_sp_neigh_entry *neigh_entry,
1845                                     bool adding)
1846 {
1847         if (adding)
1848                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1849         else
1850                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1851         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
1852 }
1853
1854 struct mlxsw_sp_neigh_event_work {
1855         struct work_struct work;
1856         struct mlxsw_sp *mlxsw_sp;
1857         struct neighbour *n;
1858 };
1859
1860 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1861 {
1862         struct mlxsw_sp_neigh_event_work *neigh_work =
1863                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1864         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1865         struct mlxsw_sp_neigh_entry *neigh_entry;
1866         struct neighbour *n = neigh_work->n;
1867         unsigned char ha[ETH_ALEN];
1868         bool entry_connected;
1869         u8 nud_state, dead;
1870
1871         /* If these parameters are changed after we release the lock,
1872          * then we are guaranteed to receive another event letting us
1873          * know about it.
1874          */
1875         read_lock_bh(&n->lock);
1876         memcpy(ha, n->ha, ETH_ALEN);
1877         nud_state = n->nud_state;
1878         dead = n->dead;
1879         read_unlock_bh(&n->lock);
1880
1881         rtnl_lock();
1882         entry_connected = nud_state & NUD_VALID && !dead;
1883         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1884         if (!entry_connected && !neigh_entry)
1885                 goto out;
1886         if (!neigh_entry) {
1887                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1888                 if (IS_ERR(neigh_entry))
1889                         goto out;
1890         }
1891
1892         memcpy(neigh_entry->ha, ha, ETH_ALEN);
1893         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1894         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
1895                                       dead);
1896
1897         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1898                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1899
1900 out:
1901         rtnl_unlock();
1902         neigh_release(n);
1903         kfree(neigh_work);
1904 }
1905
1906 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1907                                    unsigned long event, void *ptr)
1908 {
1909         struct mlxsw_sp_neigh_event_work *neigh_work;
1910         struct mlxsw_sp_port *mlxsw_sp_port;
1911         struct mlxsw_sp *mlxsw_sp;
1912         unsigned long interval;
1913         struct neigh_parms *p;
1914         struct neighbour *n;
1915
1916         switch (event) {
1917         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1918                 p = ptr;
1919
1920                 /* We don't care about changes in the default table. */
1921                 if (!p->dev || (p->tbl->family != AF_INET &&
1922                                 p->tbl->family != AF_INET6))
1923                         return NOTIFY_DONE;
1924
1925                 /* We are in atomic context and can't take RTNL mutex,
1926                  * so use RCU variant to walk the device chain.
1927                  */
1928                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1929                 if (!mlxsw_sp_port)
1930                         return NOTIFY_DONE;
1931
1932                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1933                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1934                 mlxsw_sp->router->neighs_update.interval = interval;
1935
1936                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1937                 break;
1938         case NETEVENT_NEIGH_UPDATE:
1939                 n = ptr;
1940
1941                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
1942                         return NOTIFY_DONE;
1943
1944                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1945                 if (!mlxsw_sp_port)
1946                         return NOTIFY_DONE;
1947
1948                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1949                 if (!neigh_work) {
1950                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1951                         return NOTIFY_BAD;
1952                 }
1953
1954                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1955                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1956                 neigh_work->n = n;
1957
1958                 /* Take a reference to ensure the neighbour won't be
1959                  * destructed until we drop the reference in delayed
1960                  * work.
1961                  */
1962                 neigh_clone(n);
1963                 mlxsw_core_schedule_work(&neigh_work->work);
1964                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1965                 break;
1966         }
1967
1968         return NOTIFY_DONE;
1969 }
1970
1971 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1972 {
1973         int err;
1974
1975         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1976                               &mlxsw_sp_neigh_ht_params);
1977         if (err)
1978                 return err;
1979
1980         /* Initialize the polling interval according to the default
1981          * table.
1982          */
1983         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1984
1985         /* Create the delayed works for the activity_update */
1986         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1987                           mlxsw_sp_router_neighs_update_work);
1988         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1989                           mlxsw_sp_router_probe_unresolved_nexthops);
1990         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1991         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1992         return 0;
1993 }
1994
1995 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1996 {
1997         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1998         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1999         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2000 }
2001
2002 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2003                                          struct mlxsw_sp_rif *rif)
2004 {
2005         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2006
2007         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2008                                  rif_list_node) {
2009                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2010                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2011         }
2012 }
2013
2014 enum mlxsw_sp_nexthop_type {
2015         MLXSW_SP_NEXTHOP_TYPE_ETH,
2016         MLXSW_SP_NEXTHOP_TYPE_IPIP,
2017 };
2018
2019 struct mlxsw_sp_nexthop_key {
2020         struct fib_nh *fib_nh;
2021 };
2022
2023 struct mlxsw_sp_nexthop {
2024         struct list_head neigh_list_node; /* member of neigh entry list */
2025         struct list_head rif_list_node;
2026         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
2027                                                 * this belongs to
2028                                                 */
2029         struct rhash_head ht_node;
2030         struct mlxsw_sp_nexthop_key key;
2031         unsigned char gw_addr[sizeof(struct in6_addr)];
2032         int ifindex;
2033         struct mlxsw_sp_rif *rif;
2034         u8 should_offload:1, /* set indicates this neigh is connected and
2035                               * should be put to KVD linear area of this group.
2036                               */
2037            offloaded:1, /* set in case the neigh is actually put into
2038                          * KVD linear area of this group.
2039                          */
2040            update:1; /* set indicates that MAC of this neigh should be
2041                       * updated in HW
2042                       */
2043         enum mlxsw_sp_nexthop_type type;
2044         union {
2045                 struct mlxsw_sp_neigh_entry *neigh_entry;
2046                 struct mlxsw_sp_ipip_entry *ipip_entry;
2047         };
2048 };
2049
2050 struct mlxsw_sp_nexthop_group {
2051         void *priv;
2052         struct rhash_head ht_node;
2053         struct list_head fib_list; /* list of fib entries that use this group */
2054         struct neigh_table *neigh_tbl;
2055         u8 adj_index_valid:1,
2056            gateway:1; /* routes using the group use a gateway */
2057         u32 adj_index;
2058         u16 ecmp_size;
2059         u16 count;
2060         struct mlxsw_sp_nexthop nexthops[0];
2061 #define nh_rif  nexthops[0].rif
2062 };
2063
2064 static struct fib_info *
2065 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
2066 {
2067         return nh_grp->priv;
2068 }
2069
2070 struct mlxsw_sp_nexthop_group_cmp_arg {
2071         enum mlxsw_sp_l3proto proto;
2072         union {
2073                 struct fib_info *fi;
2074                 struct mlxsw_sp_fib6_entry *fib6_entry;
2075         };
2076 };
2077
2078 static bool
2079 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
2080                                     const struct in6_addr *gw, int ifindex)
2081 {
2082         int i;
2083
2084         for (i = 0; i < nh_grp->count; i++) {
2085                 const struct mlxsw_sp_nexthop *nh;
2086
2087                 nh = &nh_grp->nexthops[i];
2088                 if (nh->ifindex == ifindex &&
2089                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
2090                         return true;
2091         }
2092
2093         return false;
2094 }
2095
2096 static bool
2097 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
2098                             const struct mlxsw_sp_fib6_entry *fib6_entry)
2099 {
2100         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2101
2102         if (nh_grp->count != fib6_entry->nrt6)
2103                 return false;
2104
2105         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2106                 struct in6_addr *gw;
2107                 int ifindex;
2108
2109                 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
2110                 gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
2111                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
2112                         return false;
2113         }
2114
2115         return true;
2116 }
2117
2118 static int
2119 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
2120 {
2121         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
2122         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
2123
2124         switch (cmp_arg->proto) {
2125         case MLXSW_SP_L3_PROTO_IPV4:
2126                 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
2127         case MLXSW_SP_L3_PROTO_IPV6:
2128                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
2129                                                     cmp_arg->fib6_entry);
2130         default:
2131                 WARN_ON(1);
2132                 return 1;
2133         }
2134 }
2135
2136 static int
2137 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
2138 {
2139         return nh_grp->neigh_tbl->family;
2140 }
2141
2142 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
2143 {
2144         const struct mlxsw_sp_nexthop_group *nh_grp = data;
2145         const struct mlxsw_sp_nexthop *nh;
2146         struct fib_info *fi;
2147         unsigned int val;
2148         int i;
2149
2150         switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
2151         case AF_INET:
2152                 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
2153                 return jhash(&fi, sizeof(fi), seed);
2154         case AF_INET6:
2155                 val = nh_grp->count;
2156                 for (i = 0; i < nh_grp->count; i++) {
2157                         nh = &nh_grp->nexthops[i];
2158                         val ^= nh->ifindex;
2159                 }
2160                 return jhash(&val, sizeof(val), seed);
2161         default:
2162                 WARN_ON(1);
2163                 return 0;
2164         }
2165 }
2166
2167 static u32
2168 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
2169 {
2170         unsigned int val = fib6_entry->nrt6;
2171         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2172         struct net_device *dev;
2173
2174         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2175                 dev = mlxsw_sp_rt6->rt->dst.dev;
2176                 val ^= dev->ifindex;
2177         }
2178
2179         return jhash(&val, sizeof(val), seed);
2180 }
2181
2182 static u32
2183 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
2184 {
2185         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
2186
2187         switch (cmp_arg->proto) {
2188         case MLXSW_SP_L3_PROTO_IPV4:
2189                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
2190         case MLXSW_SP_L3_PROTO_IPV6:
2191                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
2192         default:
2193                 WARN_ON(1);
2194                 return 0;
2195         }
2196 }
2197
2198 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
2199         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
2200         .hashfn      = mlxsw_sp_nexthop_group_hash,
2201         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
2202         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
2203 };
2204
2205 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
2206                                          struct mlxsw_sp_nexthop_group *nh_grp)
2207 {
2208         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2209             !nh_grp->gateway)
2210                 return 0;
2211
2212         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
2213                                       &nh_grp->ht_node,
2214                                       mlxsw_sp_nexthop_group_ht_params);
2215 }
2216
2217 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
2218                                           struct mlxsw_sp_nexthop_group *nh_grp)
2219 {
2220         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2221             !nh_grp->gateway)
2222                 return;
2223
2224         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
2225                                &nh_grp->ht_node,
2226                                mlxsw_sp_nexthop_group_ht_params);
2227 }
2228
2229 static struct mlxsw_sp_nexthop_group *
2230 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
2231                                struct fib_info *fi)
2232 {
2233         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2234
2235         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
2236         cmp_arg.fi = fi;
2237         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2238                                       &cmp_arg,
2239                                       mlxsw_sp_nexthop_group_ht_params);
2240 }
2241
2242 static struct mlxsw_sp_nexthop_group *
2243 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
2244                                struct mlxsw_sp_fib6_entry *fib6_entry)
2245 {
2246         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2247
2248         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
2249         cmp_arg.fib6_entry = fib6_entry;
2250         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2251                                       &cmp_arg,
2252                                       mlxsw_sp_nexthop_group_ht_params);
2253 }
2254
2255 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
2256         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
2257         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
2258         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
2259 };
2260
2261 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
2262                                    struct mlxsw_sp_nexthop *nh)
2263 {
2264         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
2265                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
2266 }
2267
2268 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
2269                                     struct mlxsw_sp_nexthop *nh)
2270 {
2271         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
2272                                mlxsw_sp_nexthop_ht_params);
2273 }
2274
2275 static struct mlxsw_sp_nexthop *
2276 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
2277                         struct mlxsw_sp_nexthop_key key)
2278 {
2279         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
2280                                       mlxsw_sp_nexthop_ht_params);
2281 }
2282
2283 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
2284                                              const struct mlxsw_sp_fib *fib,
2285                                              u32 adj_index, u16 ecmp_size,
2286                                              u32 new_adj_index,
2287                                              u16 new_ecmp_size)
2288 {
2289         char raleu_pl[MLXSW_REG_RALEU_LEN];
2290
2291         mlxsw_reg_raleu_pack(raleu_pl,
2292                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
2293                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
2294                              new_ecmp_size);
2295         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
2296 }
2297
2298 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
2299                                           struct mlxsw_sp_nexthop_group *nh_grp,
2300                                           u32 old_adj_index, u16 old_ecmp_size)
2301 {
2302         struct mlxsw_sp_fib_entry *fib_entry;
2303         struct mlxsw_sp_fib *fib = NULL;
2304         int err;
2305
2306         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2307                 if (fib == fib_entry->fib_node->fib)
2308                         continue;
2309                 fib = fib_entry->fib_node->fib;
2310                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
2311                                                         old_adj_index,
2312                                                         old_ecmp_size,
2313                                                         nh_grp->adj_index,
2314                                                         nh_grp->ecmp_size);
2315                 if (err)
2316                         return err;
2317         }
2318         return 0;
2319 }
2320
2321 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
2322                                        struct mlxsw_sp_nexthop *nh)
2323 {
2324         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2325         char ratr_pl[MLXSW_REG_RATR_LEN];
2326
2327         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
2328                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
2329                             adj_index, neigh_entry->rif);
2330         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
2331         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
2332 }
2333
2334 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
2335                                         u32 adj_index,
2336                                         struct mlxsw_sp_nexthop *nh)
2337 {
2338         const struct mlxsw_sp_ipip_ops *ipip_ops;
2339
2340         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
2341         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
2342 }
2343
2344 static int
2345 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
2346                               struct mlxsw_sp_nexthop_group *nh_grp,
2347                               bool reallocate)
2348 {
2349         u32 adj_index = nh_grp->adj_index; /* base */
2350         struct mlxsw_sp_nexthop *nh;
2351         int i;
2352         int err;
2353
2354         for (i = 0; i < nh_grp->count; i++) {
2355                 nh = &nh_grp->nexthops[i];
2356
2357                 if (!nh->should_offload) {
2358                         nh->offloaded = 0;
2359                         continue;
2360                 }
2361
2362                 if (nh->update || reallocate) {
2363                         switch (nh->type) {
2364                         case MLXSW_SP_NEXTHOP_TYPE_ETH:
2365                                 err = mlxsw_sp_nexthop_mac_update
2366                                             (mlxsw_sp, adj_index, nh);
2367                                 break;
2368                         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2369                                 err = mlxsw_sp_nexthop_ipip_update
2370                                             (mlxsw_sp, adj_index, nh);
2371                                 break;
2372                         }
2373                         if (err)
2374                                 return err;
2375                         nh->update = 0;
2376                         nh->offloaded = 1;
2377                 }
2378                 adj_index++;
2379         }
2380         return 0;
2381 }
2382
2383 static bool
2384 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2385                                  const struct mlxsw_sp_fib_entry *fib_entry);
2386
2387 static int
2388 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
2389                                     struct mlxsw_sp_nexthop_group *nh_grp)
2390 {
2391         struct mlxsw_sp_fib_entry *fib_entry;
2392         int err;
2393
2394         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2395                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2396                                                       fib_entry))
2397                         continue;
2398                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2399                 if (err)
2400                         return err;
2401         }
2402         return 0;
2403 }
2404
2405 static void
2406 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2407                                    enum mlxsw_reg_ralue_op op, int err);
2408
2409 static void
2410 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
2411 {
2412         enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
2413         struct mlxsw_sp_fib_entry *fib_entry;
2414
2415         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2416                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2417                                                       fib_entry))
2418                         continue;
2419                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2420         }
2421 }
2422
2423 static void
2424 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
2425                                struct mlxsw_sp_nexthop_group *nh_grp)
2426 {
2427         struct mlxsw_sp_nexthop *nh;
2428         bool offload_change = false;
2429         u32 adj_index;
2430         u16 ecmp_size = 0;
2431         bool old_adj_index_valid;
2432         u32 old_adj_index;
2433         u16 old_ecmp_size;
2434         int i;
2435         int err;
2436
2437         if (!nh_grp->gateway) {
2438                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2439                 return;
2440         }
2441
2442         for (i = 0; i < nh_grp->count; i++) {
2443                 nh = &nh_grp->nexthops[i];
2444
2445                 if (nh->should_offload != nh->offloaded) {
2446                         offload_change = true;
2447                         if (nh->should_offload)
2448                                 nh->update = 1;
2449                 }
2450                 if (nh->should_offload)
2451                         ecmp_size++;
2452         }
2453         if (!offload_change) {
2454                 /* Nothing was added or removed, so no need to reallocate. Just
2455                  * update MAC on existing adjacency indexes.
2456                  */
2457                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false);
2458                 if (err) {
2459                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2460                         goto set_trap;
2461                 }
2462                 return;
2463         }
2464         if (!ecmp_size)
2465                 /* No neigh of this group is connected so we just set
2466                  * the trap and let everthing flow through kernel.
2467                  */
2468                 goto set_trap;
2469
2470         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
2471         if (err) {
2472                 /* We ran out of KVD linear space, just set the
2473                  * trap and let everything flow through kernel.
2474                  */
2475                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
2476                 goto set_trap;
2477         }
2478         old_adj_index_valid = nh_grp->adj_index_valid;
2479         old_adj_index = nh_grp->adj_index;
2480         old_ecmp_size = nh_grp->ecmp_size;
2481         nh_grp->adj_index_valid = 1;
2482         nh_grp->adj_index = adj_index;
2483         nh_grp->ecmp_size = ecmp_size;
2484         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true);
2485         if (err) {
2486                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2487                 goto set_trap;
2488         }
2489
2490         if (!old_adj_index_valid) {
2491                 /* The trap was set for fib entries, so we have to call
2492                  * fib entry update to unset it and use adjacency index.
2493                  */
2494                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2495                 if (err) {
2496                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
2497                         goto set_trap;
2498                 }
2499                 return;
2500         }
2501
2502         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
2503                                              old_adj_index, old_ecmp_size);
2504         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
2505         if (err) {
2506                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
2507                 goto set_trap;
2508         }
2509
2510         /* Offload state within the group changed, so update the flags. */
2511         mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
2512
2513         return;
2514
2515 set_trap:
2516         old_adj_index_valid = nh_grp->adj_index_valid;
2517         nh_grp->adj_index_valid = 0;
2518         for (i = 0; i < nh_grp->count; i++) {
2519                 nh = &nh_grp->nexthops[i];
2520                 nh->offloaded = 0;
2521         }
2522         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2523         if (err)
2524                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
2525         if (old_adj_index_valid)
2526                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
2527 }
2528
2529 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
2530                                             bool removing)
2531 {
2532         if (!removing)
2533                 nh->should_offload = 1;
2534         else
2535                 nh->should_offload = 0;
2536         nh->update = 1;
2537 }
2538
2539 static int
2540 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
2541                                     struct mlxsw_sp_neigh_entry *neigh_entry)
2542 {
2543         struct neighbour *n, *old_n = neigh_entry->key.n;
2544         struct mlxsw_sp_nexthop *nh;
2545         bool entry_connected;
2546         u8 nud_state, dead;
2547         int err;
2548
2549         nh = list_first_entry(&neigh_entry->nexthop_list,
2550                               struct mlxsw_sp_nexthop, neigh_list_node);
2551
2552         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
2553         if (!n) {
2554                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
2555                                  nh->rif->dev);
2556                 if (IS_ERR(n))
2557                         return PTR_ERR(n);
2558                 neigh_event_send(n, NULL);
2559         }
2560
2561         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2562         neigh_entry->key.n = n;
2563         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2564         if (err)
2565                 goto err_neigh_entry_insert;
2566
2567         read_lock_bh(&n->lock);
2568         nud_state = n->nud_state;
2569         dead = n->dead;
2570         read_unlock_bh(&n->lock);
2571         entry_connected = nud_state & NUD_VALID && !dead;
2572
2573         list_for_each_entry(nh, &neigh_entry->nexthop_list,
2574                             neigh_list_node) {
2575                 neigh_release(old_n);
2576                 neigh_clone(n);
2577                 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
2578                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2579         }
2580
2581         neigh_release(n);
2582
2583         return 0;
2584
2585 err_neigh_entry_insert:
2586         neigh_entry->key.n = old_n;
2587         mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2588         neigh_release(n);
2589         return err;
2590 }
2591
2592 static void
2593 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2594                               struct mlxsw_sp_neigh_entry *neigh_entry,
2595                               bool removing, bool dead)
2596 {
2597         struct mlxsw_sp_nexthop *nh;
2598
2599         if (list_empty(&neigh_entry->nexthop_list))
2600                 return;
2601
2602         if (dead) {
2603                 int err;
2604
2605                 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
2606                                                           neigh_entry);
2607                 if (err)
2608                         dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
2609                 return;
2610         }
2611
2612         list_for_each_entry(nh, &neigh_entry->nexthop_list,
2613                             neigh_list_node) {
2614                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
2615                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2616         }
2617 }
2618
2619 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
2620                                       struct mlxsw_sp_rif *rif)
2621 {
2622         if (nh->rif)
2623                 return;
2624
2625         nh->rif = rif;
2626         list_add(&nh->rif_list_node, &rif->nexthop_list);
2627 }
2628
2629 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
2630 {
2631         if (!nh->rif)
2632                 return;
2633
2634         list_del(&nh->rif_list_node);
2635         nh->rif = NULL;
2636 }
2637
2638 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
2639                                        struct mlxsw_sp_nexthop *nh)
2640 {
2641         struct mlxsw_sp_neigh_entry *neigh_entry;
2642         struct neighbour *n;
2643         u8 nud_state, dead;
2644         int err;
2645
2646         if (!nh->nh_grp->gateway || nh->neigh_entry)
2647                 return 0;
2648
2649         /* Take a reference of neigh here ensuring that neigh would
2650          * not be destructed before the nexthop entry is finished.
2651          * The reference is taken either in neigh_lookup() or
2652          * in neigh_create() in case n is not found.
2653          */
2654         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
2655         if (!n) {
2656                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
2657                                  nh->rif->dev);
2658                 if (IS_ERR(n))
2659                         return PTR_ERR(n);
2660                 neigh_event_send(n, NULL);
2661         }
2662         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2663         if (!neigh_entry) {
2664                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2665                 if (IS_ERR(neigh_entry)) {
2666                         err = -EINVAL;
2667                         goto err_neigh_entry_create;
2668                 }
2669         }
2670
2671         /* If that is the first nexthop connected to that neigh, add to
2672          * nexthop_neighs_list
2673          */
2674         if (list_empty(&neigh_entry->nexthop_list))
2675                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
2676                               &mlxsw_sp->router->nexthop_neighs_list);
2677
2678         nh->neigh_entry = neigh_entry;
2679         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
2680         read_lock_bh(&n->lock);
2681         nud_state = n->nud_state;
2682         dead = n->dead;
2683         read_unlock_bh(&n->lock);
2684         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
2685
2686         return 0;
2687
2688 err_neigh_entry_create:
2689         neigh_release(n);
2690         return err;
2691 }
2692
2693 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
2694                                         struct mlxsw_sp_nexthop *nh)
2695 {
2696         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2697         struct neighbour *n;
2698
2699         if (!neigh_entry)
2700                 return;
2701         n = neigh_entry->key.n;
2702
2703         __mlxsw_sp_nexthop_neigh_update(nh, true);
2704         list_del(&nh->neigh_list_node);
2705         nh->neigh_entry = NULL;
2706
2707         /* If that is the last nexthop connected to that neigh, remove from
2708          * nexthop_neighs_list
2709          */
2710         if (list_empty(&neigh_entry->nexthop_list))
2711                 list_del(&neigh_entry->nexthop_neighs_list_node);
2712
2713         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2714                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2715
2716         neigh_release(n);
2717 }
2718
2719 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2720                                       const struct net_device *dev,
2721                                       enum mlxsw_sp_ipip_type *p_type)
2722 {
2723         struct mlxsw_sp_router *router = mlxsw_sp->router;
2724         const struct mlxsw_sp_ipip_ops *ipip_ops;
2725         enum mlxsw_sp_ipip_type ipipt;
2726
2727         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
2728                 ipip_ops = router->ipip_ops_arr[ipipt];
2729                 if (dev->type == ipip_ops->dev_type) {
2730                         if (p_type)
2731                                 *p_type = ipipt;
2732                         return true;
2733                 }
2734         }
2735         return false;
2736 }
2737
2738 static int mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
2739                                       enum mlxsw_sp_ipip_type ipipt,
2740                                       struct mlxsw_sp_nexthop *nh,
2741                                       struct net_device *ol_dev)
2742 {
2743         if (!nh->nh_grp->gateway || nh->ipip_entry)
2744                 return 0;
2745
2746         nh->ipip_entry = mlxsw_sp_ipip_entry_get(mlxsw_sp, ipipt, ol_dev);
2747         if (IS_ERR(nh->ipip_entry))
2748                 return PTR_ERR(nh->ipip_entry);
2749
2750         __mlxsw_sp_nexthop_neigh_update(nh, false);
2751         return 0;
2752 }
2753
2754 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
2755                                        struct mlxsw_sp_nexthop *nh)
2756 {
2757         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
2758
2759         if (!ipip_entry)
2760                 return;
2761
2762         __mlxsw_sp_nexthop_neigh_update(nh, true);
2763         mlxsw_sp_ipip_entry_put(mlxsw_sp, ipip_entry);
2764         nh->ipip_entry = NULL;
2765 }
2766
2767 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2768                                         const struct fib_nh *fib_nh,
2769                                         enum mlxsw_sp_ipip_type *p_ipipt)
2770 {
2771         struct net_device *dev = fib_nh->nh_dev;
2772
2773         return dev &&
2774                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
2775                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
2776 }
2777
2778 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
2779                                        struct mlxsw_sp_nexthop *nh)
2780 {
2781         switch (nh->type) {
2782         case MLXSW_SP_NEXTHOP_TYPE_ETH:
2783                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2784                 mlxsw_sp_nexthop_rif_fini(nh);
2785                 break;
2786         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2787                 mlxsw_sp_nexthop_rif_fini(nh);
2788                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
2789                 break;
2790         }
2791 }
2792
2793 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp,
2794                                        struct mlxsw_sp_nexthop *nh,
2795                                        struct fib_nh *fib_nh)
2796 {
2797         struct mlxsw_sp_router *router = mlxsw_sp->router;
2798         struct net_device *dev = fib_nh->nh_dev;
2799         enum mlxsw_sp_ipip_type ipipt;
2800         struct mlxsw_sp_rif *rif;
2801         int err;
2802
2803         if (mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fib_nh, &ipipt) &&
2804             router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
2805                                                      MLXSW_SP_L3_PROTO_IPV4)) {
2806                 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
2807                 err = mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
2808                 if (err)
2809                         return err;
2810                 mlxsw_sp_nexthop_rif_init(nh, &nh->ipip_entry->ol_lb->common);
2811                 return 0;
2812         }
2813
2814         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
2815         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
2816         if (!rif)
2817                 return 0;
2818
2819         mlxsw_sp_nexthop_rif_init(nh, rif);
2820         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2821         if (err)
2822                 goto err_neigh_init;
2823
2824         return 0;
2825
2826 err_neigh_init:
2827         mlxsw_sp_nexthop_rif_fini(nh);
2828         return err;
2829 }
2830
2831 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp,
2832                                         struct mlxsw_sp_nexthop *nh)
2833 {
2834         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2835 }
2836
2837 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
2838                                   struct mlxsw_sp_nexthop_group *nh_grp,
2839                                   struct mlxsw_sp_nexthop *nh,
2840                                   struct fib_nh *fib_nh)
2841 {
2842         struct net_device *dev = fib_nh->nh_dev;
2843         struct in_device *in_dev;
2844         int err;
2845
2846         nh->nh_grp = nh_grp;
2847         nh->key.fib_nh = fib_nh;
2848         memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
2849         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
2850         if (err)
2851                 return err;
2852
2853         if (!dev)
2854                 return 0;
2855
2856         in_dev = __in_dev_get_rtnl(dev);
2857         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
2858             fib_nh->nh_flags & RTNH_F_LINKDOWN)
2859                 return 0;
2860
2861         err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2862         if (err)
2863                 goto err_nexthop_neigh_init;
2864
2865         return 0;
2866
2867 err_nexthop_neigh_init:
2868         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2869         return err;
2870 }
2871
2872 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
2873                                    struct mlxsw_sp_nexthop *nh)
2874 {
2875         mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2876         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2877 }
2878
2879 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
2880                                     unsigned long event, struct fib_nh *fib_nh)
2881 {
2882         struct mlxsw_sp_nexthop_key key;
2883         struct mlxsw_sp_nexthop *nh;
2884
2885         if (mlxsw_sp->router->aborted)
2886                 return;
2887
2888         key.fib_nh = fib_nh;
2889         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
2890         if (WARN_ON_ONCE(!nh))
2891                 return;
2892
2893         switch (event) {
2894         case FIB_EVENT_NH_ADD:
2895                 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2896                 break;
2897         case FIB_EVENT_NH_DEL:
2898                 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2899                 break;
2900         }
2901
2902         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2903 }
2904
2905 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2906                                            struct mlxsw_sp_rif *rif)
2907 {
2908         struct mlxsw_sp_nexthop *nh, *tmp;
2909
2910         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
2911                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2912                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2913         }
2914 }
2915
2916 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
2917                                    const struct fib_info *fi)
2918 {
2919         return fi->fib_nh->nh_scope == RT_SCOPE_LINK ||
2920                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL);
2921 }
2922
2923 static struct mlxsw_sp_nexthop_group *
2924 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
2925 {
2926         struct mlxsw_sp_nexthop_group *nh_grp;
2927         struct mlxsw_sp_nexthop *nh;
2928         struct fib_nh *fib_nh;
2929         size_t alloc_size;
2930         int i;
2931         int err;
2932
2933         alloc_size = sizeof(*nh_grp) +
2934                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
2935         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
2936         if (!nh_grp)
2937                 return ERR_PTR(-ENOMEM);
2938         nh_grp->priv = fi;
2939         INIT_LIST_HEAD(&nh_grp->fib_list);
2940         nh_grp->neigh_tbl = &arp_tbl;
2941
2942         nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
2943         nh_grp->count = fi->fib_nhs;
2944         fib_info_hold(fi);
2945         for (i = 0; i < nh_grp->count; i++) {
2946                 nh = &nh_grp->nexthops[i];
2947                 fib_nh = &fi->fib_nh[i];
2948                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
2949                 if (err)
2950                         goto err_nexthop4_init;
2951         }
2952         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
2953         if (err)
2954                 goto err_nexthop_group_insert;
2955         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2956         return nh_grp;
2957
2958 err_nexthop_group_insert:
2959 err_nexthop4_init:
2960         for (i--; i >= 0; i--) {
2961                 nh = &nh_grp->nexthops[i];
2962                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2963         }
2964         fib_info_put(fi);
2965         kfree(nh_grp);
2966         return ERR_PTR(err);
2967 }
2968
2969 static void
2970 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
2971                                 struct mlxsw_sp_nexthop_group *nh_grp)
2972 {
2973         struct mlxsw_sp_nexthop *nh;
2974         int i;
2975
2976         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
2977         for (i = 0; i < nh_grp->count; i++) {
2978                 nh = &nh_grp->nexthops[i];
2979                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2980         }
2981         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2982         WARN_ON_ONCE(nh_grp->adj_index_valid);
2983         fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
2984         kfree(nh_grp);
2985 }
2986
2987 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
2988                                        struct mlxsw_sp_fib_entry *fib_entry,
2989                                        struct fib_info *fi)
2990 {
2991         struct mlxsw_sp_nexthop_group *nh_grp;
2992
2993         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
2994         if (!nh_grp) {
2995                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
2996                 if (IS_ERR(nh_grp))
2997                         return PTR_ERR(nh_grp);
2998         }
2999         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
3000         fib_entry->nh_group = nh_grp;
3001         return 0;
3002 }
3003
3004 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
3005                                         struct mlxsw_sp_fib_entry *fib_entry)
3006 {
3007         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3008
3009         list_del(&fib_entry->nexthop_group_node);
3010         if (!list_empty(&nh_grp->fib_list))
3011                 return;
3012         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
3013 }
3014
3015 static bool
3016 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
3017 {
3018         struct mlxsw_sp_fib4_entry *fib4_entry;
3019
3020         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
3021                                   common);
3022         return !fib4_entry->tos;
3023 }
3024
3025 static bool
3026 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
3027 {
3028         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
3029
3030         switch (fib_entry->fib_node->fib->proto) {
3031         case MLXSW_SP_L3_PROTO_IPV4:
3032                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
3033                         return false;
3034                 break;
3035         case MLXSW_SP_L3_PROTO_IPV6:
3036                 break;
3037         }
3038
3039         switch (fib_entry->type) {
3040         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
3041                 return !!nh_group->adj_index_valid;
3042         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
3043                 return !!nh_group->nh_rif;
3044         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
3045                 return true;
3046         default:
3047                 return false;
3048         }
3049 }
3050
3051 static struct mlxsw_sp_nexthop *
3052 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3053                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3054 {
3055         int i;
3056
3057         for (i = 0; i < nh_grp->count; i++) {
3058                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3059                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3060
3061                 if (nh->rif && nh->rif->dev == rt->dst.dev &&
3062                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
3063                                     &rt->rt6i_gateway))
3064                         return nh;
3065                 continue;
3066         }
3067
3068         return NULL;
3069 }
3070
3071 static void
3072 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3073 {
3074         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3075         int i;
3076
3077         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL ||
3078             fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) {
3079                 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3080                 return;
3081         }
3082
3083         for (i = 0; i < nh_grp->count; i++) {
3084                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3085
3086                 if (nh->offloaded)
3087                         nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3088                 else
3089                         nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3090         }
3091 }
3092
3093 static void
3094 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3095 {
3096         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3097         int i;
3098
3099         if (!list_is_singular(&nh_grp->fib_list))
3100                 return;
3101
3102         for (i = 0; i < nh_grp->count; i++) {
3103                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3104
3105                 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3106         }
3107 }
3108
3109 static void
3110 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3111 {
3112         struct mlxsw_sp_fib6_entry *fib6_entry;
3113         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3114
3115         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3116                                   common);
3117
3118         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
3119                 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3120                                  list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
3121                 return;
3122         }
3123
3124         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3125                 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3126                 struct mlxsw_sp_nexthop *nh;
3127
3128                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3129                 if (nh && nh->offloaded)
3130                         mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
3131                 else
3132                         mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
3133         }
3134 }
3135
3136 static void
3137 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3138 {
3139         struct mlxsw_sp_fib6_entry *fib6_entry;
3140         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3141
3142         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3143                                   common);
3144         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3145                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3146
3147                 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
3148         }
3149 }
3150
3151 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3152 {
3153         switch (fib_entry->fib_node->fib->proto) {
3154         case MLXSW_SP_L3_PROTO_IPV4:
3155                 mlxsw_sp_fib4_entry_offload_set(fib_entry);
3156                 break;
3157         case MLXSW_SP_L3_PROTO_IPV6:
3158                 mlxsw_sp_fib6_entry_offload_set(fib_entry);
3159                 break;
3160         }
3161 }
3162
3163 static void
3164 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3165 {
3166         switch (fib_entry->fib_node->fib->proto) {
3167         case MLXSW_SP_L3_PROTO_IPV4:
3168                 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
3169                 break;
3170         case MLXSW_SP_L3_PROTO_IPV6:
3171                 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
3172                 break;
3173         }
3174 }
3175
3176 static void
3177 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
3178                                    enum mlxsw_reg_ralue_op op, int err)
3179 {
3180         switch (op) {
3181         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
3182                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
3183         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
3184                 if (err)
3185                         return;
3186                 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
3187                         mlxsw_sp_fib_entry_offload_set(fib_entry);
3188                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
3189                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
3190                 return;
3191         default:
3192                 return;
3193         }
3194 }
3195
3196 static void
3197 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
3198                               const struct mlxsw_sp_fib_entry *fib_entry,
3199                               enum mlxsw_reg_ralue_op op)
3200 {
3201         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
3202         enum mlxsw_reg_ralxx_protocol proto;
3203         u32 *p_dip;
3204
3205         proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
3206
3207         switch (fib->proto) {
3208         case MLXSW_SP_L3_PROTO_IPV4:
3209                 p_dip = (u32 *) fib_entry->fib_node->key.addr;
3210                 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
3211                                       fib_entry->fib_node->key.prefix_len,
3212                                       *p_dip);
3213                 break;
3214         case MLXSW_SP_L3_PROTO_IPV6:
3215                 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
3216                                       fib_entry->fib_node->key.prefix_len,
3217                                       fib_entry->fib_node->key.addr);
3218                 break;
3219         }
3220 }
3221
3222 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
3223                                         struct mlxsw_sp_fib_entry *fib_entry,
3224                                         enum mlxsw_reg_ralue_op op)
3225 {
3226         char ralue_pl[MLXSW_REG_RALUE_LEN];
3227         enum mlxsw_reg_ralue_trap_action trap_action;
3228         u16 trap_id = 0;
3229         u32 adjacency_index = 0;
3230         u16 ecmp_size = 0;
3231
3232         /* In case the nexthop group adjacency index is valid, use it
3233          * with provided ECMP size. Otherwise, setup trap and pass
3234          * traffic to kernel.
3235          */
3236         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
3237                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
3238                 adjacency_index = fib_entry->nh_group->adj_index;
3239                 ecmp_size = fib_entry->nh_group->ecmp_size;
3240         } else {
3241                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
3242                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
3243         }
3244
3245         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3246         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
3247                                         adjacency_index, ecmp_size);
3248         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3249 }
3250
3251 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
3252                                        struct mlxsw_sp_fib_entry *fib_entry,
3253                                        enum mlxsw_reg_ralue_op op)
3254 {
3255         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
3256         enum mlxsw_reg_ralue_trap_action trap_action;
3257         char ralue_pl[MLXSW_REG_RALUE_LEN];
3258         u16 trap_id = 0;
3259         u16 rif_index = 0;
3260
3261         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
3262                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
3263                 rif_index = rif->rif_index;
3264         } else {
3265                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
3266                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
3267         }
3268
3269         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3270         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
3271                                        rif_index);
3272         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3273 }
3274
3275 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
3276                                       struct mlxsw_sp_fib_entry *fib_entry,
3277                                       enum mlxsw_reg_ralue_op op)
3278 {
3279         char ralue_pl[MLXSW_REG_RALUE_LEN];
3280
3281         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3282         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
3283         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3284 }
3285
3286 static int
3287 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
3288                                  struct mlxsw_sp_fib_entry *fib_entry,
3289                                  enum mlxsw_reg_ralue_op op)
3290 {
3291         struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
3292         const struct mlxsw_sp_ipip_ops *ipip_ops;
3293
3294         if (WARN_ON(!ipip_entry))
3295                 return -EINVAL;
3296
3297         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3298         return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op,
3299                                       fib_entry->decap.tunnel_index);
3300 }
3301
3302 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3303                                    struct mlxsw_sp_fib_entry *fib_entry,
3304                                    enum mlxsw_reg_ralue_op op)
3305 {
3306         switch (fib_entry->type) {
3307         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
3308                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
3309         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
3310                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
3311         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
3312                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
3313         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
3314                 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
3315                                                         fib_entry, op);
3316         }
3317         return -EINVAL;
3318 }
3319
3320 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3321                                  struct mlxsw_sp_fib_entry *fib_entry,
3322                                  enum mlxsw_reg_ralue_op op)
3323 {
3324         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
3325
3326         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
3327
3328         return err;
3329 }
3330
3331 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
3332                                      struct mlxsw_sp_fib_entry *fib_entry)
3333 {
3334         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3335                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
3336 }
3337
3338 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
3339                                   struct mlxsw_sp_fib_entry *fib_entry)
3340 {
3341         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3342                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
3343 }
3344
3345 static int
3346 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
3347                              const struct fib_entry_notifier_info *fen_info,
3348                              struct mlxsw_sp_fib_entry *fib_entry)
3349 {
3350         union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
3351         struct net_device *dev = fen_info->fi->fib_dev;
3352         struct mlxsw_sp_ipip_entry *ipip_entry;
3353         struct fib_info *fi = fen_info->fi;
3354
3355         switch (fen_info->type) {
3356         case RTN_LOCAL:
3357                 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev,
3358                                                  MLXSW_SP_L3_PROTO_IPV4, dip);
3359                 if (ipip_entry) {
3360                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
3361                         return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
3362                                                              fib_entry,
3363                                                              ipip_entry);
3364                 }
3365                 /* fall through */
3366         case RTN_BROADCAST:
3367                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
3368                 return 0;
3369         case RTN_UNREACHABLE: /* fall through */
3370         case RTN_BLACKHOLE: /* fall through */
3371         case RTN_PROHIBIT:
3372                 /* Packets hitting these routes need to be trapped, but
3373                  * can do so with a lower priority than packets directed
3374                  * at the host, so use action type local instead of trap.
3375                  */
3376                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3377                 return 0;
3378         case RTN_UNICAST:
3379                 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
3380                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
3381                 else
3382                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3383                 return 0;
3384         default:
3385                 return -EINVAL;
3386         }
3387 }
3388
3389 static struct mlxsw_sp_fib4_entry *
3390 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
3391                            struct mlxsw_sp_fib_node *fib_node,
3392                            const struct fib_entry_notifier_info *fen_info)
3393 {
3394         struct mlxsw_sp_fib4_entry *fib4_entry;
3395         struct mlxsw_sp_fib_entry *fib_entry;
3396         int err;
3397
3398         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
3399         if (!fib4_entry)
3400                 return ERR_PTR(-ENOMEM);
3401         fib_entry = &fib4_entry->common;
3402
3403         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
3404         if (err)
3405                 goto err_fib4_entry_type_set;
3406
3407         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
3408         if (err)
3409                 goto err_nexthop4_group_get;
3410
3411         fib4_entry->prio = fen_info->fi->fib_priority;
3412         fib4_entry->tb_id = fen_info->tb_id;
3413         fib4_entry->type = fen_info->type;
3414         fib4_entry->tos = fen_info->tos;
3415
3416         fib_entry->fib_node = fib_node;
3417
3418         return fib4_entry;
3419
3420 err_nexthop4_group_get:
3421 err_fib4_entry_type_set:
3422         kfree(fib4_entry);
3423         return ERR_PTR(err);
3424 }
3425
3426 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
3427                                         struct mlxsw_sp_fib4_entry *fib4_entry)
3428 {
3429         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
3430         kfree(fib4_entry);
3431 }
3432
3433 static struct mlxsw_sp_fib4_entry *
3434 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
3435                            const struct fib_entry_notifier_info *fen_info)
3436 {
3437         struct mlxsw_sp_fib4_entry *fib4_entry;
3438         struct mlxsw_sp_fib_node *fib_node;
3439         struct mlxsw_sp_fib *fib;
3440         struct mlxsw_sp_vr *vr;
3441
3442         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
3443         if (!vr)
3444                 return NULL;
3445         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
3446
3447         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
3448                                             sizeof(fen_info->dst),
3449                                             fen_info->dst_len);
3450         if (!fib_node)
3451                 return NULL;
3452
3453         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3454                 if (fib4_entry->tb_id == fen_info->tb_id &&
3455                     fib4_entry->tos == fen_info->tos &&
3456                     fib4_entry->type == fen_info->type &&
3457                     mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
3458                     fen_info->fi) {
3459                         return fib4_entry;
3460                 }
3461         }
3462
3463         return NULL;
3464 }
3465
3466 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
3467         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
3468         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
3469         .key_len = sizeof(struct mlxsw_sp_fib_key),
3470         .automatic_shrinking = true,
3471 };
3472
3473 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
3474                                     struct mlxsw_sp_fib_node *fib_node)
3475 {
3476         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
3477                                       mlxsw_sp_fib_ht_params);
3478 }
3479
3480 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
3481                                      struct mlxsw_sp_fib_node *fib_node)
3482 {
3483         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
3484                                mlxsw_sp_fib_ht_params);
3485 }
3486
3487 static struct mlxsw_sp_fib_node *
3488 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
3489                          size_t addr_len, unsigned char prefix_len)
3490 {
3491         struct mlxsw_sp_fib_key key;
3492
3493         memset(&key, 0, sizeof(key));
3494         memcpy(key.addr, addr, addr_len);
3495         key.prefix_len = prefix_len;
3496         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
3497 }
3498
3499 static struct mlxsw_sp_fib_node *
3500 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
3501                          size_t addr_len, unsigned char prefix_len)
3502 {
3503         struct mlxsw_sp_fib_node *fib_node;
3504
3505         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
3506         if (!fib_node)
3507                 return NULL;
3508
3509         INIT_LIST_HEAD(&fib_node->entry_list);
3510         list_add(&fib_node->list, &fib->node_list);
3511         memcpy(fib_node->key.addr, addr, addr_len);
3512         fib_node->key.prefix_len = prefix_len;
3513
3514         return fib_node;
3515 }
3516
3517 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
3518 {
3519         list_del(&fib_node->list);
3520         WARN_ON(!list_empty(&fib_node->entry_list));
3521         kfree(fib_node);
3522 }
3523
3524 static bool
3525 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
3526                                  const struct mlxsw_sp_fib_entry *fib_entry)
3527 {
3528         return list_first_entry(&fib_node->entry_list,
3529                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
3530 }
3531
3532 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
3533                                       struct mlxsw_sp_fib *fib,
3534                                       struct mlxsw_sp_fib_node *fib_node)
3535 {
3536         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3537         struct mlxsw_sp_lpm_tree *lpm_tree;
3538         int err;
3539
3540         /* Since the tree is shared between all virtual routers we must
3541          * make sure it contains all the required prefix lengths. This
3542          * can be computed by either adding the new prefix length to the
3543          * existing prefix usage of a bound tree, or by aggregating the
3544          * prefix lengths across all virtual routers and adding the new
3545          * one as well.
3546          */
3547         if (fib->lpm_tree)
3548                 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
3549                                           &fib->lpm_tree->prefix_usage);
3550         else
3551                 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3552         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
3553
3554         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3555                                          fib->proto);
3556         if (IS_ERR(lpm_tree))
3557                 return PTR_ERR(lpm_tree);
3558
3559         if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
3560                 return 0;
3561
3562         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3563         if (err)
3564                 return err;
3565
3566         return 0;
3567 }
3568
3569 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
3570                                          struct mlxsw_sp_fib *fib)
3571 {
3572         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
3573                 return;
3574         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
3575         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
3576         fib->lpm_tree = NULL;
3577 }
3578
3579 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
3580 {
3581         unsigned char prefix_len = fib_node->key.prefix_len;
3582         struct mlxsw_sp_fib *fib = fib_node->fib;
3583
3584         if (fib->prefix_ref_count[prefix_len]++ == 0)
3585                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
3586 }
3587
3588 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
3589 {
3590         unsigned char prefix_len = fib_node->key.prefix_len;
3591         struct mlxsw_sp_fib *fib = fib_node->fib;
3592
3593         if (--fib->prefix_ref_count[prefix_len] == 0)
3594                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
3595 }
3596
3597 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
3598                                   struct mlxsw_sp_fib_node *fib_node,
3599                                   struct mlxsw_sp_fib *fib)
3600 {
3601         int err;
3602
3603         err = mlxsw_sp_fib_node_insert(fib, fib_node);
3604         if (err)
3605                 return err;
3606         fib_node->fib = fib;
3607
3608         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
3609         if (err)
3610                 goto err_fib_lpm_tree_link;
3611
3612         mlxsw_sp_fib_node_prefix_inc(fib_node);
3613
3614         return 0;
3615
3616 err_fib_lpm_tree_link:
3617         fib_node->fib = NULL;
3618         mlxsw_sp_fib_node_remove(fib, fib_node);
3619         return err;
3620 }
3621
3622 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
3623                                    struct mlxsw_sp_fib_node *fib_node)
3624 {
3625         struct mlxsw_sp_fib *fib = fib_node->fib;
3626
3627         mlxsw_sp_fib_node_prefix_dec(fib_node);
3628         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
3629         fib_node->fib = NULL;
3630         mlxsw_sp_fib_node_remove(fib, fib_node);
3631 }
3632
3633 static struct mlxsw_sp_fib_node *
3634 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
3635                       size_t addr_len, unsigned char prefix_len,
3636                       enum mlxsw_sp_l3proto proto)
3637 {
3638         struct mlxsw_sp_fib_node *fib_node;
3639         struct mlxsw_sp_fib *fib;
3640         struct mlxsw_sp_vr *vr;
3641         int err;
3642
3643         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
3644         if (IS_ERR(vr))
3645                 return ERR_CAST(vr);
3646         fib = mlxsw_sp_vr_fib(vr, proto);
3647
3648         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
3649         if (fib_node)
3650                 return fib_node;
3651
3652         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
3653         if (!fib_node) {
3654                 err = -ENOMEM;
3655                 goto err_fib_node_create;
3656         }
3657
3658         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
3659         if (err)
3660                 goto err_fib_node_init;
3661
3662         return fib_node;
3663
3664 err_fib_node_init:
3665         mlxsw_sp_fib_node_destroy(fib_node);
3666 err_fib_node_create:
3667         mlxsw_sp_vr_put(vr);
3668         return ERR_PTR(err);
3669 }
3670
3671 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
3672                                   struct mlxsw_sp_fib_node *fib_node)
3673 {
3674         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
3675
3676         if (!list_empty(&fib_node->entry_list))
3677                 return;
3678         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
3679         mlxsw_sp_fib_node_destroy(fib_node);
3680         mlxsw_sp_vr_put(vr);
3681 }
3682
3683 static struct mlxsw_sp_fib4_entry *
3684 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3685                               const struct mlxsw_sp_fib4_entry *new4_entry)
3686 {
3687         struct mlxsw_sp_fib4_entry *fib4_entry;
3688
3689         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3690                 if (fib4_entry->tb_id > new4_entry->tb_id)
3691                         continue;
3692                 if (fib4_entry->tb_id != new4_entry->tb_id)
3693                         break;
3694                 if (fib4_entry->tos > new4_entry->tos)
3695                         continue;
3696                 if (fib4_entry->prio >= new4_entry->prio ||
3697                     fib4_entry->tos < new4_entry->tos)
3698                         return fib4_entry;
3699         }
3700
3701         return NULL;
3702 }
3703
3704 static int
3705 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
3706                                struct mlxsw_sp_fib4_entry *new4_entry)
3707 {
3708         struct mlxsw_sp_fib_node *fib_node;
3709
3710         if (WARN_ON(!fib4_entry))
3711                 return -EINVAL;
3712
3713         fib_node = fib4_entry->common.fib_node;
3714         list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
3715                                  common.list) {
3716                 if (fib4_entry->tb_id != new4_entry->tb_id ||
3717                     fib4_entry->tos != new4_entry->tos ||
3718                     fib4_entry->prio != new4_entry->prio)
3719                         break;
3720         }
3721
3722         list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
3723         return 0;
3724 }
3725
3726 static int
3727 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
3728                                bool replace, bool append)
3729 {
3730         struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
3731         struct mlxsw_sp_fib4_entry *fib4_entry;
3732
3733         fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
3734
3735         if (append)
3736                 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
3737         if (replace && WARN_ON(!fib4_entry))
3738                 return -EINVAL;
3739
3740         /* Insert new entry before replaced one, so that we can later
3741          * remove the second.
3742          */
3743         if (fib4_entry) {
3744                 list_add_tail(&new4_entry->common.list,
3745                               &fib4_entry->common.list);
3746         } else {
3747                 struct mlxsw_sp_fib4_entry *last;
3748
3749                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3750                         if (new4_entry->tb_id > last->tb_id)
3751                                 break;
3752                         fib4_entry = last;
3753                 }
3754
3755                 if (fib4_entry)
3756                         list_add(&new4_entry->common.list,
3757                                  &fib4_entry->common.list);
3758                 else
3759                         list_add(&new4_entry->common.list,
3760                                  &fib_node->entry_list);
3761         }
3762
3763         return 0;
3764 }
3765
3766 static void
3767 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
3768 {
3769         list_del(&fib4_entry->common.list);
3770 }
3771
3772 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
3773                                        struct mlxsw_sp_fib_entry *fib_entry)
3774 {
3775         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3776
3777         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3778                 return 0;
3779
3780         /* To prevent packet loss, overwrite the previously offloaded
3781          * entry.
3782          */
3783         if (!list_is_singular(&fib_node->entry_list)) {
3784                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3785                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3786
3787                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
3788         }
3789
3790         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3791 }
3792
3793 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
3794                                         struct mlxsw_sp_fib_entry *fib_entry)
3795 {
3796         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3797
3798         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3799                 return;
3800
3801         /* Promote the next entry by overwriting the deleted entry */
3802         if (!list_is_singular(&fib_node->entry_list)) {
3803                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3804                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3805
3806                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
3807                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3808                 return;
3809         }
3810
3811         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
3812 }
3813
3814 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3815                                          struct mlxsw_sp_fib4_entry *fib4_entry,
3816                                          bool replace, bool append)
3817 {
3818         int err;
3819
3820         err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
3821         if (err)
3822                 return err;
3823
3824         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
3825         if (err)
3826                 goto err_fib_node_entry_add;
3827
3828         return 0;
3829
3830 err_fib_node_entry_add:
3831         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3832         return err;
3833 }
3834
3835 static void
3836 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3837                                 struct mlxsw_sp_fib4_entry *fib4_entry)
3838 {
3839         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
3840         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3841
3842         if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP)
3843                 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common);
3844 }
3845
3846 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
3847                                         struct mlxsw_sp_fib4_entry *fib4_entry,
3848                                         bool replace)
3849 {
3850         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
3851         struct mlxsw_sp_fib4_entry *replaced;
3852
3853         if (!replace)
3854                 return;
3855
3856         /* We inserted the new entry before replaced one */
3857         replaced = list_next_entry(fib4_entry, common.list);
3858
3859         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
3860         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
3861         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3862 }
3863
3864 static int
3865 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
3866                          const struct fib_entry_notifier_info *fen_info,
3867                          bool replace, bool append)
3868 {
3869         struct mlxsw_sp_fib4_entry *fib4_entry;
3870         struct mlxsw_sp_fib_node *fib_node;
3871         int err;
3872
3873         if (mlxsw_sp->router->aborted)
3874                 return 0;
3875
3876         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
3877                                          &fen_info->dst, sizeof(fen_info->dst),
3878                                          fen_info->dst_len,
3879                                          MLXSW_SP_L3_PROTO_IPV4);
3880         if (IS_ERR(fib_node)) {
3881                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
3882                 return PTR_ERR(fib_node);
3883         }
3884
3885         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
3886         if (IS_ERR(fib4_entry)) {
3887                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
3888                 err = PTR_ERR(fib4_entry);
3889                 goto err_fib4_entry_create;
3890         }
3891
3892         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
3893                                             append);
3894         if (err) {
3895                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
3896                 goto err_fib4_node_entry_link;
3897         }
3898
3899         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
3900
3901         return 0;
3902
3903 err_fib4_node_entry_link:
3904         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3905 err_fib4_entry_create:
3906         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3907         return err;
3908 }
3909
3910 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
3911                                      struct fib_entry_notifier_info *fen_info)
3912 {
3913         struct mlxsw_sp_fib4_entry *fib4_entry;
3914         struct mlxsw_sp_fib_node *fib_node;
3915
3916         if (mlxsw_sp->router->aborted)
3917                 return;
3918
3919         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
3920         if (WARN_ON(!fib4_entry))
3921                 return;
3922         fib_node = fib4_entry->common.fib_node;
3923
3924         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
3925         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3926         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3927 }
3928
3929 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
3930 {
3931         /* Packets with link-local destination IP arriving to the router
3932          * are trapped to the CPU, so no need to program specific routes
3933          * for them.
3934          */
3935         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
3936                 return true;
3937
3938         /* Multicast routes aren't supported, so ignore them. Neighbour
3939          * Discovery packets are specifically trapped.
3940          */
3941         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
3942                 return true;
3943
3944         /* Cloned routes are irrelevant in the forwarding path. */
3945         if (rt->rt6i_flags & RTF_CACHE)
3946                 return true;
3947
3948         return false;
3949 }
3950
3951 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
3952 {
3953         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3954
3955         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
3956         if (!mlxsw_sp_rt6)
3957                 return ERR_PTR(-ENOMEM);
3958
3959         /* In case of route replace, replaced route is deleted with
3960          * no notification. Take reference to prevent accessing freed
3961          * memory.
3962          */
3963         mlxsw_sp_rt6->rt = rt;
3964         rt6_hold(rt);
3965
3966         return mlxsw_sp_rt6;
3967 }
3968
3969 #if IS_ENABLED(CONFIG_IPV6)
3970 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3971 {
3972         rt6_release(rt);
3973 }
3974 #else
3975 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3976 {
3977 }
3978 #endif
3979
3980 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3981 {
3982         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
3983         kfree(mlxsw_sp_rt6);
3984 }
3985
3986 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
3987 {
3988         /* RTF_CACHE routes are ignored */
3989         return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
3990 }
3991
3992 static struct rt6_info *
3993 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
3994 {
3995         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3996                                 list)->rt;
3997 }
3998
3999 static struct mlxsw_sp_fib6_entry *
4000 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4001                                  const struct rt6_info *nrt, bool replace)
4002 {
4003         struct mlxsw_sp_fib6_entry *fib6_entry;
4004
4005         if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
4006                 return NULL;
4007
4008         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4009                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4010
4011                 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
4012                  * virtual router.
4013                  */
4014                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
4015                         continue;
4016                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
4017                         break;
4018                 if (rt->rt6i_metric < nrt->rt6i_metric)
4019                         continue;
4020                 if (rt->rt6i_metric == nrt->rt6i_metric &&
4021                     mlxsw_sp_fib6_rt_can_mp(rt))
4022                         return fib6_entry;
4023                 if (rt->rt6i_metric > nrt->rt6i_metric)
4024                         break;
4025         }
4026
4027         return NULL;
4028 }
4029
4030 static struct mlxsw_sp_rt6 *
4031 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
4032                             const struct rt6_info *rt)
4033 {
4034         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4035
4036         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
4037                 if (mlxsw_sp_rt6->rt == rt)
4038                         return mlxsw_sp_rt6;
4039         }
4040
4041         return NULL;
4042 }
4043
4044 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
4045                                         const struct rt6_info *rt,
4046                                         enum mlxsw_sp_ipip_type *ret)
4047 {
4048         return rt->dst.dev &&
4049                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->dst.dev, ret);
4050 }
4051
4052 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
4053                                        struct mlxsw_sp_nexthop_group *nh_grp,
4054                                        struct mlxsw_sp_nexthop *nh,
4055                                        const struct rt6_info *rt)
4056 {
4057         struct mlxsw_sp_router *router = mlxsw_sp->router;
4058         struct net_device *dev = rt->dst.dev;
4059         enum mlxsw_sp_ipip_type ipipt;
4060         struct mlxsw_sp_rif *rif;
4061         int err;
4062
4063         if (mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, &ipipt) &&
4064             router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
4065                                                      MLXSW_SP_L3_PROTO_IPV6)) {
4066                 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4067                 err = mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
4068                 if (err)
4069                         return err;
4070                 mlxsw_sp_nexthop_rif_init(nh, &nh->ipip_entry->ol_lb->common);
4071                 return 0;
4072         }
4073
4074         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4075         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4076         if (!rif)
4077                 return 0;
4078         mlxsw_sp_nexthop_rif_init(nh, rif);
4079
4080         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4081         if (err)
4082                 goto err_nexthop_neigh_init;
4083
4084         return 0;
4085
4086 err_nexthop_neigh_init:
4087         mlxsw_sp_nexthop_rif_fini(nh);
4088         return err;
4089 }
4090
4091 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
4092                                         struct mlxsw_sp_nexthop *nh)
4093 {
4094         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4095 }
4096
4097 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
4098                                   struct mlxsw_sp_nexthop_group *nh_grp,
4099                                   struct mlxsw_sp_nexthop *nh,
4100                                   const struct rt6_info *rt)
4101 {
4102         struct net_device *dev = rt->dst.dev;
4103
4104         nh->nh_grp = nh_grp;
4105         memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
4106
4107         if (!dev)
4108                 return 0;
4109         nh->ifindex = dev->ifindex;
4110
4111         return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt);
4112 }
4113
4114 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
4115                                    struct mlxsw_sp_nexthop *nh)
4116 {
4117         mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh);
4118 }
4119
4120 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4121                                     const struct rt6_info *rt)
4122 {
4123         return rt->rt6i_flags & RTF_GATEWAY ||
4124                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
4125 }
4126
4127 static struct mlxsw_sp_nexthop_group *
4128 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
4129                                struct mlxsw_sp_fib6_entry *fib6_entry)
4130 {
4131         struct mlxsw_sp_nexthop_group *nh_grp;
4132         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4133         struct mlxsw_sp_nexthop *nh;
4134         size_t alloc_size;
4135         int i = 0;
4136         int err;
4137
4138         alloc_size = sizeof(*nh_grp) +
4139                      fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
4140         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
4141         if (!nh_grp)
4142                 return ERR_PTR(-ENOMEM);
4143         INIT_LIST_HEAD(&nh_grp->fib_list);
4144 #if IS_ENABLED(CONFIG_IPV6)
4145         nh_grp->neigh_tbl = &nd_tbl;
4146 #endif
4147         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
4148                                         struct mlxsw_sp_rt6, list);
4149         nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
4150         nh_grp->count = fib6_entry->nrt6;
4151         for (i = 0; i < nh_grp->count; i++) {
4152                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
4153
4154                 nh = &nh_grp->nexthops[i];
4155                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
4156                 if (err)
4157                         goto err_nexthop6_init;
4158                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
4159         }
4160
4161         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4162         if (err)
4163                 goto err_nexthop_group_insert;
4164
4165         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4166         return nh_grp;
4167
4168 err_nexthop_group_insert:
4169 err_nexthop6_init:
4170         for (i--; i >= 0; i--) {
4171                 nh = &nh_grp->nexthops[i];
4172                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4173         }
4174         kfree(nh_grp);
4175         return ERR_PTR(err);
4176 }
4177
4178 static void
4179 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
4180                                 struct mlxsw_sp_nexthop_group *nh_grp)
4181 {
4182         struct mlxsw_sp_nexthop *nh;
4183         int i = nh_grp->count;
4184
4185         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4186         for (i--; i >= 0; i--) {
4187                 nh = &nh_grp->nexthops[i];
4188                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4189         }
4190         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4191         WARN_ON(nh_grp->adj_index_valid);
4192         kfree(nh_grp);
4193 }
4194
4195 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
4196                                        struct mlxsw_sp_fib6_entry *fib6_entry)
4197 {
4198         struct mlxsw_sp_nexthop_group *nh_grp;
4199
4200         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
4201         if (!nh_grp) {
4202                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
4203                 if (IS_ERR(nh_grp))
4204                         return PTR_ERR(nh_grp);
4205         }
4206
4207         list_add_tail(&fib6_entry->common.nexthop_group_node,
4208                       &nh_grp->fib_list);
4209         fib6_entry->common.nh_group = nh_grp;
4210
4211         return 0;
4212 }
4213
4214 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
4215                                         struct mlxsw_sp_fib_entry *fib_entry)
4216 {
4217         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4218
4219         list_del(&fib_entry->nexthop_group_node);
4220         if (!list_empty(&nh_grp->fib_list))
4221                 return;
4222         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
4223 }
4224
4225 static int
4226 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
4227                                struct mlxsw_sp_fib6_entry *fib6_entry)
4228 {
4229         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
4230         int err;
4231
4232         fib6_entry->common.nh_group = NULL;
4233         list_del(&fib6_entry->common.nexthop_group_node);
4234
4235         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4236         if (err)
4237                 goto err_nexthop6_group_get;
4238
4239         /* In case this entry is offloaded, then the adjacency index
4240          * currently associated with it in the device's table is that
4241          * of the old group. Start using the new one instead.
4242          */
4243         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4244         if (err)
4245                 goto err_fib_node_entry_add;
4246
4247         if (list_empty(&old_nh_grp->fib_list))
4248                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
4249
4250         return 0;
4251
4252 err_fib_node_entry_add:
4253         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4254 err_nexthop6_group_get:
4255         list_add_tail(&fib6_entry->common.nexthop_group_node,
4256                       &old_nh_grp->fib_list);
4257         fib6_entry->common.nh_group = old_nh_grp;
4258         return err;
4259 }
4260
4261 static int
4262 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
4263                                 struct mlxsw_sp_fib6_entry *fib6_entry,
4264                                 struct rt6_info *rt)
4265 {
4266         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4267         int err;
4268
4269         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4270         if (IS_ERR(mlxsw_sp_rt6))
4271                 return PTR_ERR(mlxsw_sp_rt6);
4272
4273         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4274         fib6_entry->nrt6++;
4275
4276         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4277         if (err)
4278                 goto err_nexthop6_group_update;
4279
4280         return 0;
4281
4282 err_nexthop6_group_update:
4283         fib6_entry->nrt6--;
4284         list_del(&mlxsw_sp_rt6->list);
4285         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4286         return err;
4287 }
4288
4289 static void
4290 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
4291                                 struct mlxsw_sp_fib6_entry *fib6_entry,
4292                                 struct rt6_info *rt)
4293 {
4294         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4295
4296         mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
4297         if (WARN_ON(!mlxsw_sp_rt6))
4298                 return;
4299
4300         fib6_entry->nrt6--;
4301         list_del(&mlxsw_sp_rt6->list);
4302         mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4303         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4304 }
4305
4306 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4307                                          struct mlxsw_sp_fib_entry *fib_entry,
4308                                          const struct rt6_info *rt)
4309 {
4310         /* Packets hitting RTF_REJECT routes need to be discarded by the
4311          * stack. We can rely on their destination device not having a
4312          * RIF (it's the loopback device) and can thus use action type
4313          * local, which will cause them to be trapped with a lower
4314          * priority than packets that need to be locally received.
4315          */
4316         if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
4317                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4318         else if (rt->rt6i_flags & RTF_REJECT)
4319                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4320         else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
4321                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4322         else
4323                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4324 }
4325
4326 static void
4327 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
4328 {
4329         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
4330
4331         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
4332                                  list) {
4333                 fib6_entry->nrt6--;
4334                 list_del(&mlxsw_sp_rt6->list);
4335                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4336         }
4337 }
4338
4339 static struct mlxsw_sp_fib6_entry *
4340 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
4341                            struct mlxsw_sp_fib_node *fib_node,
4342                            struct rt6_info *rt)
4343 {
4344         struct mlxsw_sp_fib6_entry *fib6_entry;
4345         struct mlxsw_sp_fib_entry *fib_entry;
4346         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4347         int err;
4348
4349         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
4350         if (!fib6_entry)
4351                 return ERR_PTR(-ENOMEM);
4352         fib_entry = &fib6_entry->common;
4353
4354         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4355         if (IS_ERR(mlxsw_sp_rt6)) {
4356                 err = PTR_ERR(mlxsw_sp_rt6);
4357                 goto err_rt6_create;
4358         }
4359
4360         mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt);
4361
4362         INIT_LIST_HEAD(&fib6_entry->rt6_list);
4363         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4364         fib6_entry->nrt6 = 1;
4365         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4366         if (err)
4367                 goto err_nexthop6_group_get;
4368
4369         fib_entry->fib_node = fib_node;
4370
4371         return fib6_entry;
4372
4373 err_nexthop6_group_get:
4374         list_del(&mlxsw_sp_rt6->list);
4375         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4376 err_rt6_create:
4377         kfree(fib6_entry);
4378         return ERR_PTR(err);
4379 }
4380
4381 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4382                                         struct mlxsw_sp_fib6_entry *fib6_entry)
4383 {
4384         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4385         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
4386         WARN_ON(fib6_entry->nrt6);
4387         kfree(fib6_entry);
4388 }
4389
4390 static struct mlxsw_sp_fib6_entry *
4391 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4392                               const struct rt6_info *nrt, bool replace)
4393 {
4394         struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
4395
4396         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4397                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4398
4399                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
4400                         continue;
4401                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
4402                         break;
4403                 if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
4404                         if (mlxsw_sp_fib6_rt_can_mp(rt) ==
4405                             mlxsw_sp_fib6_rt_can_mp(nrt))
4406                                 return fib6_entry;
4407                         if (mlxsw_sp_fib6_rt_can_mp(nrt))
4408                                 fallback = fallback ?: fib6_entry;
4409                 }
4410                 if (rt->rt6i_metric > nrt->rt6i_metric)
4411                         return fallback ?: fib6_entry;
4412         }
4413
4414         return fallback;
4415 }
4416
4417 static int
4418 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
4419                                bool replace)
4420 {
4421         struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
4422         struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
4423         struct mlxsw_sp_fib6_entry *fib6_entry;
4424
4425         fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
4426
4427         if (replace && WARN_ON(!fib6_entry))
4428                 return -EINVAL;
4429
4430         if (fib6_entry) {
4431                 list_add_tail(&new6_entry->common.list,
4432                               &fib6_entry->common.list);
4433         } else {
4434                 struct mlxsw_sp_fib6_entry *last;
4435
4436                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
4437                         struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
4438
4439                         if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
4440                                 break;
4441                         fib6_entry = last;
4442                 }
4443
4444                 if (fib6_entry)
4445                         list_add(&new6_entry->common.list,
4446                                  &fib6_entry->common.list);
4447                 else
4448                         list_add(&new6_entry->common.list,
4449                                  &fib_node->entry_list);
4450         }
4451
4452         return 0;
4453 }
4454
4455 static void
4456 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
4457 {
4458         list_del(&fib6_entry->common.list);
4459 }
4460
4461 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4462                                          struct mlxsw_sp_fib6_entry *fib6_entry,
4463                                          bool replace)
4464 {
4465         int err;
4466
4467         err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
4468         if (err)
4469                 return err;
4470
4471         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4472         if (err)
4473                 goto err_fib_node_entry_add;
4474
4475         return 0;
4476
4477 err_fib_node_entry_add:
4478         mlxsw_sp_fib6_node_list_remove(fib6_entry);
4479         return err;
4480 }
4481
4482 static void
4483 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4484                                 struct mlxsw_sp_fib6_entry *fib6_entry)
4485 {
4486         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
4487         mlxsw_sp_fib6_node_list_remove(fib6_entry);
4488 }
4489
4490 static struct mlxsw_sp_fib6_entry *
4491 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4492                            const struct rt6_info *rt)
4493 {
4494         struct mlxsw_sp_fib6_entry *fib6_entry;
4495         struct mlxsw_sp_fib_node *fib_node;
4496         struct mlxsw_sp_fib *fib;
4497         struct mlxsw_sp_vr *vr;
4498
4499         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
4500         if (!vr)
4501                 return NULL;
4502         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
4503
4504         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
4505                                             sizeof(rt->rt6i_dst.addr),
4506                                             rt->rt6i_dst.plen);
4507         if (!fib_node)
4508                 return NULL;
4509
4510         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4511                 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4512
4513                 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
4514                     rt->rt6i_metric == iter_rt->rt6i_metric &&
4515                     mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
4516                         return fib6_entry;
4517         }
4518
4519         return NULL;
4520 }
4521
4522 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
4523                                         struct mlxsw_sp_fib6_entry *fib6_entry,
4524                                         bool replace)
4525 {
4526         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
4527         struct mlxsw_sp_fib6_entry *replaced;
4528
4529         if (!replace)
4530                 return;
4531
4532         replaced = list_next_entry(fib6_entry, common.list);
4533
4534         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
4535         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
4536         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4537 }
4538
4539 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
4540                                     struct rt6_info *rt, bool replace)
4541 {
4542         struct mlxsw_sp_fib6_entry *fib6_entry;
4543         struct mlxsw_sp_fib_node *fib_node;
4544         int err;
4545
4546         if (mlxsw_sp->router->aborted)
4547                 return 0;
4548
4549         if (rt->rt6i_src.plen)
4550                 return -EINVAL;
4551
4552         if (mlxsw_sp_fib6_rt_should_ignore(rt))
4553                 return 0;
4554
4555         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
4556                                          &rt->rt6i_dst.addr,
4557                                          sizeof(rt->rt6i_dst.addr),
4558                                          rt->rt6i_dst.plen,
4559                                          MLXSW_SP_L3_PROTO_IPV6);
4560         if (IS_ERR(fib_node))
4561                 return PTR_ERR(fib_node);
4562
4563         /* Before creating a new entry, try to append route to an existing
4564          * multipath entry.
4565          */
4566         fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
4567         if (fib6_entry) {
4568                 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
4569                 if (err)
4570                         goto err_fib6_entry_nexthop_add;
4571                 return 0;
4572         }
4573
4574         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
4575         if (IS_ERR(fib6_entry)) {
4576                 err = PTR_ERR(fib6_entry);
4577                 goto err_fib6_entry_create;
4578         }
4579
4580         err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
4581         if (err)
4582                 goto err_fib6_node_entry_link;
4583
4584         mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
4585
4586         return 0;
4587
4588 err_fib6_node_entry_link:
4589         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4590 err_fib6_entry_create:
4591 err_fib6_entry_nexthop_add:
4592         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4593         return err;
4594 }
4595
4596 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
4597                                      struct rt6_info *rt)
4598 {
4599         struct mlxsw_sp_fib6_entry *fib6_entry;
4600         struct mlxsw_sp_fib_node *fib_node;
4601
4602         if (mlxsw_sp->router->aborted)
4603                 return;
4604
4605         if (mlxsw_sp_fib6_rt_should_ignore(rt))
4606                 return;
4607
4608         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
4609         if (WARN_ON(!fib6_entry))
4610                 return;
4611
4612         /* If route is part of a multipath entry, but not the last one
4613          * removed, then only reduce its nexthop group.
4614          */
4615         if (!list_is_singular(&fib6_entry->rt6_list)) {
4616                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
4617                 return;
4618         }
4619
4620         fib_node = fib6_entry->common.fib_node;
4621
4622         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4623         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4624         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4625 }
4626
4627 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
4628                                             enum mlxsw_reg_ralxx_protocol proto,
4629                                             u8 tree_id)
4630 {
4631         char ralta_pl[MLXSW_REG_RALTA_LEN];
4632         char ralst_pl[MLXSW_REG_RALST_LEN];
4633         int i, err;
4634
4635         mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
4636         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
4637         if (err)
4638                 return err;
4639
4640         mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
4641         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
4642         if (err)
4643                 return err;
4644
4645         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4646                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4647                 char raltb_pl[MLXSW_REG_RALTB_LEN];
4648                 char ralue_pl[MLXSW_REG_RALUE_LEN];
4649
4650                 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
4651                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
4652                                       raltb_pl);
4653                 if (err)
4654                         return err;
4655
4656                 mlxsw_reg_ralue_pack(ralue_pl, proto,
4657                                      MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
4658                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4659                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4660                                       ralue_pl);
4661                 if (err)
4662                         return err;
4663         }
4664
4665         return 0;
4666 }
4667
4668 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
4669 {
4670         enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
4671         int err;
4672
4673         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4674                                                MLXSW_SP_LPM_TREE_MIN);
4675         if (err)
4676                 return err;
4677
4678         proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
4679         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4680                                                 MLXSW_SP_LPM_TREE_MIN + 1);
4681 }
4682
4683 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
4684                                      struct mlxsw_sp_fib_node *fib_node)
4685 {
4686         struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
4687
4688         list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
4689                                  common.list) {
4690                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4691
4692                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4693                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4694                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4695                 /* Break when entry list is empty and node was freed.
4696                  * Otherwise, we'll access freed memory in the next
4697                  * iteration.
4698                  */
4699                 if (do_break)
4700                         break;
4701         }
4702 }
4703
4704 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
4705                                      struct mlxsw_sp_fib_node *fib_node)
4706 {
4707         struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
4708
4709         list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
4710                                  common.list) {
4711                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4712
4713                 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4714                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4715                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4716                 if (do_break)
4717                         break;
4718         }
4719 }
4720
4721 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
4722                                     struct mlxsw_sp_fib_node *fib_node)
4723 {
4724         switch (fib_node->fib->proto) {
4725         case MLXSW_SP_L3_PROTO_IPV4:
4726                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
4727                 break;
4728         case MLXSW_SP_L3_PROTO_IPV6:
4729                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
4730                 break;
4731         }
4732 }
4733
4734 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
4735                                   struct mlxsw_sp_vr *vr,
4736                                   enum mlxsw_sp_l3proto proto)
4737 {
4738         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
4739         struct mlxsw_sp_fib_node *fib_node, *tmp;
4740
4741         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
4742                 bool do_break = &tmp->list == &fib->node_list;
4743
4744                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
4745                 if (do_break)
4746                         break;
4747         }
4748 }
4749
4750 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
4751 {
4752         int i;
4753
4754         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4755                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4756
4757                 if (!mlxsw_sp_vr_is_used(vr))
4758                         continue;
4759                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
4760
4761                 /* If virtual router was only used for IPv4, then it's no
4762                  * longer used.
4763                  */
4764                 if (!mlxsw_sp_vr_is_used(vr))
4765                         continue;
4766                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
4767         }
4768 }
4769
4770 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
4771 {
4772         int err;
4773
4774         if (mlxsw_sp->router->aborted)
4775                 return;
4776         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
4777         mlxsw_sp_router_fib_flush(mlxsw_sp);
4778         mlxsw_sp->router->aborted = true;
4779         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
4780         if (err)
4781                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
4782 }
4783
4784 struct mlxsw_sp_fib_event_work {
4785         struct work_struct work;
4786         union {
4787                 struct fib6_entry_notifier_info fen6_info;
4788                 struct fib_entry_notifier_info fen_info;
4789                 struct fib_rule_notifier_info fr_info;
4790                 struct fib_nh_notifier_info fnh_info;
4791         };
4792         struct mlxsw_sp *mlxsw_sp;
4793         unsigned long event;
4794 };
4795
4796 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
4797 {
4798         struct mlxsw_sp_fib_event_work *fib_work =
4799                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4800         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4801         struct fib_rule *rule;
4802         bool replace, append;
4803         int err;
4804
4805         /* Protect internal structures from changes */
4806         rtnl_lock();
4807         switch (fib_work->event) {
4808         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4809         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4810         case FIB_EVENT_ENTRY_ADD:
4811                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4812                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
4813                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
4814                                                replace, append);
4815                 if (err)
4816                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4817                 fib_info_put(fib_work->fen_info.fi);
4818                 break;
4819         case FIB_EVENT_ENTRY_DEL:
4820                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
4821                 fib_info_put(fib_work->fen_info.fi);
4822                 break;
4823         case FIB_EVENT_RULE_ADD: /* fall through */
4824         case FIB_EVENT_RULE_DEL:
4825                 rule = fib_work->fr_info.rule;
4826                 if (!fib4_rule_default(rule) && !rule->l3mdev)
4827                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4828                 fib_rule_put(rule);
4829                 break;
4830         case FIB_EVENT_NH_ADD: /* fall through */
4831         case FIB_EVENT_NH_DEL:
4832                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
4833                                         fib_work->fnh_info.fib_nh);
4834                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
4835                 break;
4836         }
4837         rtnl_unlock();
4838         kfree(fib_work);
4839 }
4840
4841 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
4842 {
4843         struct mlxsw_sp_fib_event_work *fib_work =
4844                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4845         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4846         struct fib_rule *rule;
4847         bool replace;
4848         int err;
4849
4850         rtnl_lock();
4851         switch (fib_work->event) {
4852         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4853         case FIB_EVENT_ENTRY_ADD:
4854                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4855                 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
4856                                                fib_work->fen6_info.rt, replace);
4857                 if (err)
4858                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4859                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4860                 break;
4861         case FIB_EVENT_ENTRY_DEL:
4862                 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
4863                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4864                 break;
4865         case FIB_EVENT_RULE_ADD: /* fall through */
4866         case FIB_EVENT_RULE_DEL:
4867                 rule = fib_work->fr_info.rule;
4868                 if (!fib6_rule_default(rule) && !rule->l3mdev)
4869                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4870                 fib_rule_put(rule);
4871                 break;
4872         }
4873         rtnl_unlock();
4874         kfree(fib_work);
4875 }
4876
4877 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
4878                                        struct fib_notifier_info *info)
4879 {
4880         switch (fib_work->event) {
4881         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4882         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4883         case FIB_EVENT_ENTRY_ADD: /* fall through */
4884         case FIB_EVENT_ENTRY_DEL:
4885                 memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
4886                 /* Take referece on fib_info to prevent it from being
4887                  * freed while work is queued. Release it afterwards.
4888                  */
4889                 fib_info_hold(fib_work->fen_info.fi);
4890                 break;
4891         case FIB_EVENT_RULE_ADD: /* fall through */
4892         case FIB_EVENT_RULE_DEL:
4893                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4894                 fib_rule_get(fib_work->fr_info.rule);
4895                 break;
4896         case FIB_EVENT_NH_ADD: /* fall through */
4897         case FIB_EVENT_NH_DEL:
4898                 memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
4899                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
4900                 break;
4901         }
4902 }
4903
4904 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
4905                                        struct fib_notifier_info *info)
4906 {
4907         switch (fib_work->event) {
4908         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4909         case FIB_EVENT_ENTRY_ADD: /* fall through */
4910         case FIB_EVENT_ENTRY_DEL:
4911                 memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
4912                 rt6_hold(fib_work->fen6_info.rt);
4913                 break;
4914         case FIB_EVENT_RULE_ADD: /* fall through */
4915         case FIB_EVENT_RULE_DEL:
4916                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4917                 fib_rule_get(fib_work->fr_info.rule);
4918                 break;
4919         }
4920 }
4921
4922 /* Called with rcu_read_lock() */
4923 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
4924                                      unsigned long event, void *ptr)
4925 {
4926         struct mlxsw_sp_fib_event_work *fib_work;
4927         struct fib_notifier_info *info = ptr;
4928         struct mlxsw_sp_router *router;
4929
4930         if (!net_eq(info->net, &init_net) ||
4931             (info->family != AF_INET && info->family != AF_INET6))
4932                 return NOTIFY_DONE;
4933
4934         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
4935         if (!fib_work)
4936                 return NOTIFY_BAD;
4937
4938         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
4939         fib_work->mlxsw_sp = router->mlxsw_sp;
4940         fib_work->event = event;
4941
4942         switch (info->family) {
4943         case AF_INET:
4944                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
4945                 mlxsw_sp_router_fib4_event(fib_work, info);
4946                 break;
4947         case AF_INET6:
4948                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
4949                 mlxsw_sp_router_fib6_event(fib_work, info);
4950                 break;
4951         }
4952
4953         mlxsw_core_schedule_work(&fib_work->work);
4954
4955         return NOTIFY_DONE;
4956 }
4957
4958 static struct mlxsw_sp_rif *
4959 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
4960                          const struct net_device *dev)
4961 {
4962         int i;
4963
4964         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
4965                 if (mlxsw_sp->router->rifs[i] &&
4966                     mlxsw_sp->router->rifs[i]->dev == dev)
4967                         return mlxsw_sp->router->rifs[i];
4968
4969         return NULL;
4970 }
4971
4972 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
4973 {
4974         char ritr_pl[MLXSW_REG_RITR_LEN];
4975         int err;
4976
4977         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
4978         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4979         if (WARN_ON_ONCE(err))
4980                 return err;
4981
4982         mlxsw_reg_ritr_enable_set(ritr_pl, false);
4983         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4984 }
4985
4986 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4987                                           struct mlxsw_sp_rif *rif)
4988 {
4989         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
4990         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
4991         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
4992 }
4993
4994 static bool
4995 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
4996                            unsigned long event)
4997 {
4998         struct inet6_dev *inet6_dev;
4999         bool addr_list_empty = true;
5000         struct in_device *idev;
5001
5002         switch (event) {
5003         case NETDEV_UP:
5004                 return rif == NULL;
5005         case NETDEV_DOWN:
5006                 idev = __in_dev_get_rtnl(dev);
5007                 if (idev && idev->ifa_list)
5008                         addr_list_empty = false;
5009
5010                 inet6_dev = __in6_dev_get(dev);
5011                 if (addr_list_empty && inet6_dev &&
5012                     !list_empty(&inet6_dev->addr_list))
5013                         addr_list_empty = false;
5014
5015                 if (rif && addr_list_empty &&
5016                     !netif_is_l3_slave(rif->dev))
5017                         return true;
5018                 /* It is possible we already removed the RIF ourselves
5019                  * if it was assigned to a netdev that is now a bridge
5020                  * or LAG slave.
5021                  */
5022                 return false;
5023         }
5024
5025         return false;
5026 }
5027
5028 static enum mlxsw_sp_rif_type
5029 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
5030                       const struct net_device *dev)
5031 {
5032         enum mlxsw_sp_fid_type type;
5033
5034         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
5035                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
5036
5037         /* Otherwise RIF type is derived from the type of the underlying FID. */
5038         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
5039                 type = MLXSW_SP_FID_TYPE_8021Q;
5040         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
5041                 type = MLXSW_SP_FID_TYPE_8021Q;
5042         else if (netif_is_bridge_master(dev))
5043                 type = MLXSW_SP_FID_TYPE_8021D;
5044         else
5045                 type = MLXSW_SP_FID_TYPE_RFID;
5046
5047         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
5048 }
5049
5050 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
5051 {
5052         int i;
5053
5054         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
5055                 if (!mlxsw_sp->router->rifs[i]) {
5056                         *p_rif_index = i;
5057                         return 0;
5058                 }
5059         }
5060
5061         return -ENOBUFS;
5062 }
5063
5064 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
5065                                                u16 vr_id,
5066                                                struct net_device *l3_dev)
5067 {
5068         struct mlxsw_sp_rif *rif;
5069
5070         rif = kzalloc(rif_size, GFP_KERNEL);
5071         if (!rif)
5072                 return NULL;
5073
5074         INIT_LIST_HEAD(&rif->nexthop_list);
5075         INIT_LIST_HEAD(&rif->neigh_list);
5076         ether_addr_copy(rif->addr, l3_dev->dev_addr);
5077         rif->mtu = l3_dev->mtu;
5078         rif->vr_id = vr_id;
5079         rif->dev = l3_dev;
5080         rif->rif_index = rif_index;
5081
5082         return rif;
5083 }
5084
5085 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
5086                                            u16 rif_index)
5087 {
5088         return mlxsw_sp->router->rifs[rif_index];
5089 }
5090
5091 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
5092 {
5093         return rif->rif_index;
5094 }
5095
5096 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
5097 {
5098         return lb_rif->common.rif_index;
5099 }
5100
5101 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
5102 {
5103         return lb_rif->ul_vr_id;
5104 }
5105
5106 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
5107 {
5108         return rif->dev->ifindex;
5109 }
5110
5111 static struct mlxsw_sp_rif *
5112 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
5113                     const struct mlxsw_sp_rif_params *params)
5114 {
5115         u32 tb_id = l3mdev_fib_table(params->dev);
5116         const struct mlxsw_sp_rif_ops *ops;
5117         struct mlxsw_sp_fid *fid = NULL;
5118         enum mlxsw_sp_rif_type type;
5119         struct mlxsw_sp_rif *rif;
5120         struct mlxsw_sp_vr *vr;
5121         u16 rif_index;
5122         int err;
5123
5124         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
5125         ops = mlxsw_sp->router->rif_ops_arr[type];
5126
5127         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
5128         if (IS_ERR(vr))
5129                 return ERR_CAST(vr);
5130         vr->rif_count++;
5131
5132         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
5133         if (err)
5134                 goto err_rif_index_alloc;
5135
5136         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
5137         if (!rif) {
5138                 err = -ENOMEM;
5139                 goto err_rif_alloc;
5140         }
5141         rif->mlxsw_sp = mlxsw_sp;
5142         rif->ops = ops;
5143
5144         if (ops->fid_get) {
5145                 fid = ops->fid_get(rif);
5146                 if (IS_ERR(fid)) {
5147                         err = PTR_ERR(fid);
5148                         goto err_fid_get;
5149                 }
5150                 rif->fid = fid;
5151         }
5152
5153         if (ops->setup)
5154                 ops->setup(rif, params);
5155
5156         err = ops->configure(rif);
5157         if (err)
5158                 goto err_configure;
5159
5160         mlxsw_sp_rif_counters_alloc(rif);
5161         mlxsw_sp->router->rifs[rif_index] = rif;
5162
5163         return rif;
5164
5165 err_configure:
5166         if (fid)
5167                 mlxsw_sp_fid_put(fid);
5168 err_fid_get:
5169         kfree(rif);
5170 err_rif_alloc:
5171 err_rif_index_alloc:
5172         vr->rif_count--;
5173         mlxsw_sp_vr_put(vr);
5174         return ERR_PTR(err);
5175 }
5176
5177 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
5178 {
5179         const struct mlxsw_sp_rif_ops *ops = rif->ops;
5180         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5181         struct mlxsw_sp_fid *fid = rif->fid;
5182         struct mlxsw_sp_vr *vr;
5183
5184         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
5185         vr = &mlxsw_sp->router->vrs[rif->vr_id];
5186
5187         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
5188         mlxsw_sp_rif_counters_free(rif);
5189         ops->deconfigure(rif);
5190         if (fid)
5191                 /* Loopback RIFs are not associated with a FID. */
5192                 mlxsw_sp_fid_put(fid);
5193         kfree(rif);
5194         vr->rif_count--;
5195         mlxsw_sp_vr_put(vr);
5196 }
5197
5198 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
5199                                  struct net_device *dev)
5200 {
5201         struct mlxsw_sp_rif *rif;
5202
5203         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5204         if (!rif)
5205                 return;
5206         mlxsw_sp_rif_destroy(rif);
5207 }
5208
5209 static void
5210 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
5211                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
5212 {
5213         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5214
5215         params->vid = mlxsw_sp_port_vlan->vid;
5216         params->lag = mlxsw_sp_port->lagged;
5217         if (params->lag)
5218                 params->lag_id = mlxsw_sp_port->lag_id;
5219         else
5220                 params->system_port = mlxsw_sp_port->local_port;
5221 }
5222
5223 static int
5224 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
5225                                struct net_device *l3_dev)
5226 {
5227         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5228         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
5229         u16 vid = mlxsw_sp_port_vlan->vid;
5230         struct mlxsw_sp_rif *rif;
5231         struct mlxsw_sp_fid *fid;
5232         int err;
5233
5234         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5235         if (!rif) {
5236                 struct mlxsw_sp_rif_params params = {
5237                         .dev = l3_dev,
5238                 };
5239
5240                 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
5241                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
5242                 if (IS_ERR(rif))
5243                         return PTR_ERR(rif);
5244         }
5245
5246         /* FID was already created, just take a reference */
5247         fid = rif->ops->fid_get(rif);
5248         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
5249         if (err)
5250                 goto err_fid_port_vid_map;
5251
5252         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
5253         if (err)
5254                 goto err_port_vid_learning_set;
5255
5256         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
5257                                         BR_STATE_FORWARDING);
5258         if (err)
5259                 goto err_port_vid_stp_set;
5260
5261         mlxsw_sp_port_vlan->fid = fid;
5262
5263         return 0;
5264
5265 err_port_vid_stp_set:
5266         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
5267 err_port_vid_learning_set:
5268         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
5269 err_fid_port_vid_map:
5270         mlxsw_sp_fid_put(fid);
5271         return err;
5272 }
5273
5274 void
5275 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
5276 {
5277         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5278         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
5279         u16 vid = mlxsw_sp_port_vlan->vid;
5280
5281         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
5282                 return;
5283
5284         mlxsw_sp_port_vlan->fid = NULL;
5285         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
5286         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
5287         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
5288         /* If router port holds the last reference on the rFID, then the
5289          * associated Sub-port RIF will be destroyed.
5290          */
5291         mlxsw_sp_fid_put(fid);
5292 }
5293
5294 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
5295                                              struct net_device *port_dev,
5296                                              unsigned long event, u16 vid)
5297 {
5298         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
5299         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
5300
5301         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
5302         if (WARN_ON(!mlxsw_sp_port_vlan))
5303                 return -EINVAL;
5304
5305         switch (event) {
5306         case NETDEV_UP:
5307                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
5308                                                       l3_dev);
5309         case NETDEV_DOWN:
5310                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
5311                 break;
5312         }
5313
5314         return 0;
5315 }
5316
5317 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
5318                                         unsigned long event)
5319 {
5320         if (netif_is_bridge_port(port_dev) ||
5321             netif_is_lag_port(port_dev) ||
5322             netif_is_ovs_port(port_dev))
5323                 return 0;
5324
5325         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
5326 }
5327
5328 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
5329                                          struct net_device *lag_dev,
5330                                          unsigned long event, u16 vid)
5331 {
5332         struct net_device *port_dev;
5333         struct list_head *iter;
5334         int err;
5335
5336         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
5337                 if (mlxsw_sp_port_dev_check(port_dev)) {
5338                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
5339                                                                 port_dev,
5340                                                                 event, vid);
5341                         if (err)
5342                                 return err;
5343                 }
5344         }
5345
5346         return 0;
5347 }
5348
5349 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
5350                                        unsigned long event)
5351 {
5352         if (netif_is_bridge_port(lag_dev))
5353                 return 0;
5354
5355         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
5356 }
5357
5358 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
5359                                           unsigned long event)
5360 {
5361         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5362         struct mlxsw_sp_rif_params params = {
5363                 .dev = l3_dev,
5364         };
5365         struct mlxsw_sp_rif *rif;
5366
5367         switch (event) {
5368         case NETDEV_UP:
5369                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
5370                 if (IS_ERR(rif))
5371                         return PTR_ERR(rif);
5372                 break;
5373         case NETDEV_DOWN:
5374                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5375                 mlxsw_sp_rif_destroy(rif);
5376                 break;
5377         }
5378
5379         return 0;
5380 }
5381
5382 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
5383                                         unsigned long event)
5384 {
5385         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
5386         u16 vid = vlan_dev_vlan_id(vlan_dev);
5387
5388         if (netif_is_bridge_port(vlan_dev))
5389                 return 0;
5390
5391         if (mlxsw_sp_port_dev_check(real_dev))
5392                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
5393                                                          event, vid);
5394         else if (netif_is_lag_master(real_dev))
5395                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
5396                                                      vid);
5397         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
5398                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
5399
5400         return 0;
5401 }
5402
5403 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
5404                                      unsigned long event)
5405 {
5406         if (mlxsw_sp_port_dev_check(dev))
5407                 return mlxsw_sp_inetaddr_port_event(dev, event);
5408         else if (netif_is_lag_master(dev))
5409                 return mlxsw_sp_inetaddr_lag_event(dev, event);
5410         else if (netif_is_bridge_master(dev))
5411                 return mlxsw_sp_inetaddr_bridge_event(dev, event);
5412         else if (is_vlan_dev(dev))
5413                 return mlxsw_sp_inetaddr_vlan_event(dev, event);
5414         else
5415                 return 0;
5416 }
5417
5418 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
5419                             unsigned long event, void *ptr)
5420 {
5421         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
5422         struct net_device *dev = ifa->ifa_dev->dev;
5423         struct mlxsw_sp *mlxsw_sp;
5424         struct mlxsw_sp_rif *rif;
5425         int err = 0;
5426
5427         mlxsw_sp = mlxsw_sp_lower_get(dev);
5428         if (!mlxsw_sp)
5429                 goto out;
5430
5431         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5432         if (!mlxsw_sp_rif_should_config(rif, dev, event))
5433                 goto out;
5434
5435         err = __mlxsw_sp_inetaddr_event(dev, event);
5436 out:
5437         return notifier_from_errno(err);
5438 }
5439
5440 struct mlxsw_sp_inet6addr_event_work {
5441         struct work_struct work;
5442         struct net_device *dev;
5443         unsigned long event;
5444 };
5445
5446 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
5447 {
5448         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
5449                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
5450         struct net_device *dev = inet6addr_work->dev;
5451         unsigned long event = inet6addr_work->event;
5452         struct mlxsw_sp *mlxsw_sp;
5453         struct mlxsw_sp_rif *rif;
5454
5455         rtnl_lock();
5456         mlxsw_sp = mlxsw_sp_lower_get(dev);
5457         if (!mlxsw_sp)
5458                 goto out;
5459
5460         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5461         if (!mlxsw_sp_rif_should_config(rif, dev, event))
5462                 goto out;
5463
5464         __mlxsw_sp_inetaddr_event(dev, event);
5465 out:
5466         rtnl_unlock();
5467         dev_put(dev);
5468         kfree(inet6addr_work);
5469 }
5470
5471 /* Called with rcu_read_lock() */
5472 int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
5473                              unsigned long event, void *ptr)
5474 {
5475         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
5476         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
5477         struct net_device *dev = if6->idev->dev;
5478
5479         if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
5480                 return NOTIFY_DONE;
5481
5482         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
5483         if (!inet6addr_work)
5484                 return NOTIFY_BAD;
5485
5486         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
5487         inet6addr_work->dev = dev;
5488         inet6addr_work->event = event;
5489         dev_hold(dev);
5490         mlxsw_core_schedule_work(&inet6addr_work->work);
5491
5492         return NOTIFY_DONE;
5493 }
5494
5495 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
5496                              const char *mac, int mtu)
5497 {
5498         char ritr_pl[MLXSW_REG_RITR_LEN];
5499         int err;
5500
5501         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
5502         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5503         if (err)
5504                 return err;
5505
5506         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
5507         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
5508         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
5509         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5510 }
5511
5512 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
5513 {
5514         struct mlxsw_sp *mlxsw_sp;
5515         struct mlxsw_sp_rif *rif;
5516         u16 fid_index;
5517         int err;
5518
5519         mlxsw_sp = mlxsw_sp_lower_get(dev);
5520         if (!mlxsw_sp)
5521                 return 0;
5522
5523         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5524         if (!rif)
5525                 return 0;
5526         fid_index = mlxsw_sp_fid_index(rif->fid);
5527
5528         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
5529         if (err)
5530                 return err;
5531
5532         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
5533                                 dev->mtu);
5534         if (err)
5535                 goto err_rif_edit;
5536
5537         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
5538         if (err)
5539                 goto err_rif_fdb_op;
5540
5541         ether_addr_copy(rif->addr, dev->dev_addr);
5542         rif->mtu = dev->mtu;
5543
5544         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
5545
5546         return 0;
5547
5548 err_rif_fdb_op:
5549         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
5550 err_rif_edit:
5551         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
5552         return err;
5553 }
5554
5555 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
5556                                   struct net_device *l3_dev)
5557 {
5558         struct mlxsw_sp_rif *rif;
5559
5560         /* If netdev is already associated with a RIF, then we need to
5561          * destroy it and create a new one with the new virtual router ID.
5562          */
5563         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5564         if (rif)
5565                 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5566
5567         return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
5568 }
5569
5570 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
5571                                     struct net_device *l3_dev)
5572 {
5573         struct mlxsw_sp_rif *rif;
5574
5575         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5576         if (!rif)
5577                 return;
5578         __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5579 }
5580
5581 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
5582                                  struct netdev_notifier_changeupper_info *info)
5583 {
5584         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5585         int err = 0;
5586
5587         if (!mlxsw_sp)
5588                 return 0;
5589
5590         switch (event) {
5591         case NETDEV_PRECHANGEUPPER:
5592                 return 0;
5593         case NETDEV_CHANGEUPPER:
5594                 if (info->linking)
5595                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
5596                 else
5597                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
5598                 break;
5599         }
5600
5601         return err;
5602 }
5603
5604 static struct mlxsw_sp_rif_subport *
5605 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
5606 {
5607         return container_of(rif, struct mlxsw_sp_rif_subport, common);
5608 }
5609
5610 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
5611                                        const struct mlxsw_sp_rif_params *params)
5612 {
5613         struct mlxsw_sp_rif_subport *rif_subport;
5614
5615         rif_subport = mlxsw_sp_rif_subport_rif(rif);
5616         rif_subport->vid = params->vid;
5617         rif_subport->lag = params->lag;
5618         if (params->lag)
5619                 rif_subport->lag_id = params->lag_id;
5620         else
5621                 rif_subport->system_port = params->system_port;
5622 }
5623
5624 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
5625 {
5626         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5627         struct mlxsw_sp_rif_subport *rif_subport;
5628         char ritr_pl[MLXSW_REG_RITR_LEN];
5629
5630         rif_subport = mlxsw_sp_rif_subport_rif(rif);
5631         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
5632                             rif->rif_index, rif->vr_id, rif->dev->mtu);
5633         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5634         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
5635                                   rif_subport->lag ? rif_subport->lag_id :
5636                                                      rif_subport->system_port,
5637                                   rif_subport->vid);
5638
5639         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5640 }
5641
5642 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
5643 {
5644         int err;
5645
5646         err = mlxsw_sp_rif_subport_op(rif, true);
5647         if (err)
5648                 return err;
5649
5650         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5651                                   mlxsw_sp_fid_index(rif->fid), true);
5652         if (err)
5653                 goto err_rif_fdb_op;
5654
5655         mlxsw_sp_fid_rif_set(rif->fid, rif);
5656         return 0;
5657
5658 err_rif_fdb_op:
5659         mlxsw_sp_rif_subport_op(rif, false);
5660         return err;
5661 }
5662
5663 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
5664 {
5665         struct mlxsw_sp_fid *fid = rif->fid;
5666
5667         mlxsw_sp_fid_rif_set(fid, NULL);
5668         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5669                             mlxsw_sp_fid_index(fid), false);
5670         mlxsw_sp_rif_subport_op(rif, false);
5671 }
5672
5673 static struct mlxsw_sp_fid *
5674 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
5675 {
5676         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
5677 }
5678
5679 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
5680         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
5681         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
5682         .setup                  = mlxsw_sp_rif_subport_setup,
5683         .configure              = mlxsw_sp_rif_subport_configure,
5684         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
5685         .fid_get                = mlxsw_sp_rif_subport_fid_get,
5686 };
5687
5688 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
5689                                     enum mlxsw_reg_ritr_if_type type,
5690                                     u16 vid_fid, bool enable)
5691 {
5692         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5693         char ritr_pl[MLXSW_REG_RITR_LEN];
5694
5695         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
5696                             rif->dev->mtu);
5697         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5698         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
5699
5700         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5701 }
5702
5703 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
5704 {
5705         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
5706 }
5707
5708 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
5709 {
5710         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5711         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5712         int err;
5713
5714         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
5715         if (err)
5716                 return err;
5717
5718         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5719                                      mlxsw_sp_router_port(mlxsw_sp), true);
5720         if (err)
5721                 goto err_fid_mc_flood_set;
5722
5723         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5724                                      mlxsw_sp_router_port(mlxsw_sp), true);
5725         if (err)
5726                 goto err_fid_bc_flood_set;
5727
5728         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5729                                   mlxsw_sp_fid_index(rif->fid), true);
5730         if (err)
5731                 goto err_rif_fdb_op;
5732
5733         mlxsw_sp_fid_rif_set(rif->fid, rif);
5734         return 0;
5735
5736 err_rif_fdb_op:
5737         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5738                                mlxsw_sp_router_port(mlxsw_sp), false);
5739 err_fid_bc_flood_set:
5740         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5741                                mlxsw_sp_router_port(mlxsw_sp), false);
5742 err_fid_mc_flood_set:
5743         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5744         return err;
5745 }
5746
5747 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
5748 {
5749         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5750         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5751         struct mlxsw_sp_fid *fid = rif->fid;
5752
5753         mlxsw_sp_fid_rif_set(fid, NULL);
5754         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5755                             mlxsw_sp_fid_index(fid), false);
5756         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5757                                mlxsw_sp_router_port(mlxsw_sp), false);
5758         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5759                                mlxsw_sp_router_port(mlxsw_sp), false);
5760         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5761 }
5762
5763 static struct mlxsw_sp_fid *
5764 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
5765 {
5766         u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
5767
5768         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
5769 }
5770
5771 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
5772         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
5773         .rif_size               = sizeof(struct mlxsw_sp_rif),
5774         .configure              = mlxsw_sp_rif_vlan_configure,
5775         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
5776         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
5777 };
5778
5779 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
5780 {
5781         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5782         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5783         int err;
5784
5785         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
5786                                        true);
5787         if (err)
5788                 return err;
5789
5790         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5791                                      mlxsw_sp_router_port(mlxsw_sp), true);
5792         if (err)
5793                 goto err_fid_mc_flood_set;
5794
5795         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5796                                      mlxsw_sp_router_port(mlxsw_sp), true);
5797         if (err)
5798                 goto err_fid_bc_flood_set;
5799
5800         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5801                                   mlxsw_sp_fid_index(rif->fid), true);
5802         if (err)
5803                 goto err_rif_fdb_op;
5804
5805         mlxsw_sp_fid_rif_set(rif->fid, rif);
5806         return 0;
5807
5808 err_rif_fdb_op:
5809         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5810                                mlxsw_sp_router_port(mlxsw_sp), false);
5811 err_fid_bc_flood_set:
5812         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5813                                mlxsw_sp_router_port(mlxsw_sp), false);
5814 err_fid_mc_flood_set:
5815         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5816         return err;
5817 }
5818
5819 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
5820 {
5821         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5822         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5823         struct mlxsw_sp_fid *fid = rif->fid;
5824
5825         mlxsw_sp_fid_rif_set(fid, NULL);
5826         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5827                             mlxsw_sp_fid_index(fid), false);
5828         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5829                                mlxsw_sp_router_port(mlxsw_sp), false);
5830         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5831                                mlxsw_sp_router_port(mlxsw_sp), false);
5832         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5833 }
5834
5835 static struct mlxsw_sp_fid *
5836 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
5837 {
5838         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
5839 }
5840
5841 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
5842         .type                   = MLXSW_SP_RIF_TYPE_FID,
5843         .rif_size               = sizeof(struct mlxsw_sp_rif),
5844         .configure              = mlxsw_sp_rif_fid_configure,
5845         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
5846         .fid_get                = mlxsw_sp_rif_fid_fid_get,
5847 };
5848
5849 static struct mlxsw_sp_rif_ipip_lb *
5850 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
5851 {
5852         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
5853 }
5854
5855 static void
5856 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
5857                            const struct mlxsw_sp_rif_params *params)
5858 {
5859         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
5860         struct mlxsw_sp_rif_ipip_lb *rif_lb;
5861
5862         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
5863                                  common);
5864         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
5865         rif_lb->lb_config = params_lb->lb_config;
5866 }
5867
5868 static int
5869 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
5870                         struct mlxsw_sp_vr *ul_vr, bool enable)
5871 {
5872         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
5873         struct mlxsw_sp_rif *rif = &lb_rif->common;
5874         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5875         char ritr_pl[MLXSW_REG_RITR_LEN];
5876         u32 saddr4;
5877
5878         switch (lb_cf.ul_protocol) {
5879         case MLXSW_SP_L3_PROTO_IPV4:
5880                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
5881                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
5882                                     rif->rif_index, rif->vr_id, rif->dev->mtu);
5883                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
5884                             MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
5885                             ul_vr->id, saddr4, lb_cf.okey);
5886                 break;
5887
5888         case MLXSW_SP_L3_PROTO_IPV6:
5889                 return -EAFNOSUPPORT;
5890         }
5891
5892         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5893 }
5894
5895 static int
5896 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
5897 {
5898         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5899         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
5900         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5901         struct mlxsw_sp_vr *ul_vr;
5902         int err;
5903
5904         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id);
5905         if (IS_ERR(ul_vr))
5906                 return PTR_ERR(ul_vr);
5907
5908         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
5909         if (err)
5910                 goto err_loopback_op;
5911
5912         lb_rif->ul_vr_id = ul_vr->id;
5913         ++ul_vr->rif_count;
5914         return 0;
5915
5916 err_loopback_op:
5917         mlxsw_sp_vr_put(ul_vr);
5918         return err;
5919 }
5920
5921 static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
5922 {
5923         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5924         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5925         struct mlxsw_sp_vr *ul_vr;
5926
5927         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
5928         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
5929
5930         --ul_vr->rif_count;
5931         mlxsw_sp_vr_put(ul_vr);
5932 }
5933
5934 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
5935         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
5936         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
5937         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
5938         .configure              = mlxsw_sp_rif_ipip_lb_configure,
5939         .deconfigure            = mlxsw_sp_rif_ipip_lb_deconfigure,
5940 };
5941
5942 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
5943         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
5944         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_ops,
5945         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
5946         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp_rif_ipip_lb_ops,
5947 };
5948
5949 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
5950 {
5951         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5952
5953         mlxsw_sp->router->rifs = kcalloc(max_rifs,
5954                                          sizeof(struct mlxsw_sp_rif *),
5955                                          GFP_KERNEL);
5956         if (!mlxsw_sp->router->rifs)
5957                 return -ENOMEM;
5958
5959         mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
5960
5961         return 0;
5962 }
5963
5964 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
5965 {
5966         int i;
5967
5968         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5969                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
5970
5971         kfree(mlxsw_sp->router->rifs);
5972 }
5973
5974 static int
5975 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
5976 {
5977         char tigcr_pl[MLXSW_REG_TIGCR_LEN];
5978
5979         mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
5980         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
5981 }
5982
5983 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
5984 {
5985         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
5986         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
5987         return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
5988 }
5989
5990 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
5991 {
5992         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
5993 }
5994
5995 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
5996 {
5997         struct mlxsw_sp_router *router;
5998
5999         /* Flush pending FIB notifications and then flush the device's
6000          * table before requesting another dump. The FIB notification
6001          * block is unregistered, so no need to take RTNL.
6002          */
6003         mlxsw_core_flush_owq();
6004         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
6005         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
6006 }
6007
6008 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
6009 {
6010         char rgcr_pl[MLXSW_REG_RGCR_LEN];
6011         u64 max_rifs;
6012         int err;
6013
6014         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
6015                 return -EIO;
6016         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
6017
6018         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
6019         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
6020         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
6021         if (err)
6022                 return err;
6023         return 0;
6024 }
6025
6026 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
6027 {
6028         char rgcr_pl[MLXSW_REG_RGCR_LEN];
6029
6030         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
6031         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
6032 }
6033
6034 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
6035 {
6036         struct mlxsw_sp_router *router;
6037         int err;
6038
6039         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
6040         if (!router)
6041                 return -ENOMEM;
6042         mlxsw_sp->router = router;
6043         router->mlxsw_sp = mlxsw_sp;
6044
6045         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
6046         err = __mlxsw_sp_router_init(mlxsw_sp);
6047         if (err)
6048                 goto err_router_init;
6049
6050         err = mlxsw_sp_rifs_init(mlxsw_sp);
6051         if (err)
6052                 goto err_rifs_init;
6053
6054         err = mlxsw_sp_ipips_init(mlxsw_sp);
6055         if (err)
6056                 goto err_ipips_init;
6057
6058         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
6059                               &mlxsw_sp_nexthop_ht_params);
6060         if (err)
6061                 goto err_nexthop_ht_init;
6062
6063         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
6064                               &mlxsw_sp_nexthop_group_ht_params);
6065         if (err)
6066                 goto err_nexthop_group_ht_init;
6067
6068         err = mlxsw_sp_lpm_init(mlxsw_sp);
6069         if (err)
6070                 goto err_lpm_init;
6071
6072         err = mlxsw_sp_vrs_init(mlxsw_sp);
6073         if (err)
6074                 goto err_vrs_init;
6075
6076         err = mlxsw_sp_neigh_init(mlxsw_sp);
6077         if (err)
6078                 goto err_neigh_init;
6079
6080         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
6081         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
6082                                     mlxsw_sp_router_fib_dump_flush);
6083         if (err)
6084                 goto err_register_fib_notifier;
6085
6086         return 0;
6087
6088 err_register_fib_notifier:
6089         mlxsw_sp_neigh_fini(mlxsw_sp);
6090 err_neigh_init:
6091         mlxsw_sp_vrs_fini(mlxsw_sp);
6092 err_vrs_init:
6093         mlxsw_sp_lpm_fini(mlxsw_sp);
6094 err_lpm_init:
6095         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
6096 err_nexthop_group_ht_init:
6097         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
6098 err_nexthop_ht_init:
6099         mlxsw_sp_ipips_fini(mlxsw_sp);
6100 err_ipips_init:
6101         mlxsw_sp_rifs_fini(mlxsw_sp);
6102 err_rifs_init:
6103         __mlxsw_sp_router_fini(mlxsw_sp);
6104 err_router_init:
6105         kfree(mlxsw_sp->router);
6106         return err;
6107 }
6108
6109 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
6110 {
6111         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
6112         mlxsw_sp_neigh_fini(mlxsw_sp);
6113         mlxsw_sp_vrs_fini(mlxsw_sp);
6114         mlxsw_sp_lpm_fini(mlxsw_sp);
6115         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
6116         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
6117         mlxsw_sp_ipips_fini(mlxsw_sp);
6118         mlxsw_sp_rifs_fini(mlxsw_sp);
6119         __mlxsw_sp_router_fini(mlxsw_sp);
6120         kfree(mlxsw_sp->router);
6121 }