drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c

   1 /*
   2  * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <crypto/internal/geniv.h>
  35 #include <crypto/aead.h>
  36 #include <linux/inetdevice.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/module.h>
  39
  40 #include "en.h"
  41 #include "en_accel/ipsec.h"
  42 #include "en_accel/ipsec_rxtx.h"
  43
  44
  45 static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x)
  46 {
  47         struct mlx5e_ipsec_sa_entry *sa;
  48
  49         if (!x)
  50                 return NULL;
  51
  52         sa = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;
  53         if (!sa)
  54                 return NULL;
  55
  56         WARN_ON(sa->x != x);
  57         return sa;
  58 }
  59
  60 struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec,
  61                                               unsigned int handle)
  62 {
  63         struct mlx5e_ipsec_sa_entry *sa_entry;
  64         struct xfrm_state *ret = NULL;
  65
  66         rcu_read_lock();
  67         hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle)
  68                 if (sa_entry->handle == handle) {
  69                         ret = sa_entry->x;
  70                         xfrm_state_hold(ret);
  71                         break;
  72                 }
  73         rcu_read_unlock();
  74
  75         return ret;
  76 }
  77
  78 static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry)
  79 {
  80         struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
  81         unsigned long flags;
  82         int ret;
  83
  84         ret = ida_simple_get(&ipsec->halloc, 1, 0, GFP_KERNEL);
  85         if (ret < 0)
  86                 return ret;
  87
  88         spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
  89         sa_entry->handle = ret;
  90         hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle);
  91         spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);
  92
  93         return 0;
  94 }
  95
  96 static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry)
  97 {
  98         struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
  99         unsigned long flags;
 100
 101         spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
 102         hash_del_rcu(&sa_entry->hlist);
 103         spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);
 104 }
 105
 106 static void mlx5e_ipsec_sadb_rx_free(struct mlx5e_ipsec_sa_entry *sa_entry)
 107 {
 108         struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
 109
 110         /* xfrm already doing sync rcu between del and free callbacks */
 111
 112         ida_simple_remove(&ipsec->halloc, sa_entry->handle);
 113 }
 114
 115 static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry)
 116 {
 117         struct xfrm_replay_state_esn *replay_esn;
 118         u32 seq_bottom;
 119         u8 overlap;
 120
 121         if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) {
 122                 sa_entry->esn_state.trigger = 0;
 123                 return false;
 124         }
 125
 126         replay_esn = sa_entry->x->replay_esn;
 127         seq_bottom = replay_esn->seq - replay_esn->replay_window + 1;
 128         overlap = sa_entry->esn_state.overlap;
 129
 130         sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x,
 131                                                     htonl(seq_bottom));
 132
 133         sa_entry->esn_state.trigger = 1;
 134         if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) {
 135                 sa_entry->esn_state.overlap = 0;
 136                 return true;
 137         } else if (unlikely(!overlap &&
 138                             (seq_bottom >= MLX5E_IPSEC_ESN_SCOPE_MID))) {
 139                 sa_entry->esn_state.overlap = 1;
 140                 return true;
 141         }
 142
 143         return false;
 144 }
 145
 146 static void
 147 mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
 148                                    struct mlx5_accel_esp_xfrm_attrs *attrs)
 149 {
 150         struct xfrm_state *x = sa_entry->x;
 151         struct aes_gcm_keymat *aes_gcm = &attrs->keymat.aes_gcm;
 152         struct aead_geniv_ctx *geniv_ctx;
 153         struct crypto_aead *aead;
 154         unsigned int crypto_data_len, key_len;
 155         int ivsize;
 156
 157         memset(attrs, 0, sizeof(*attrs));
 158
 159         /* key */
 160         crypto_data_len = (x->aead->alg_key_len + 7) / 8;
 161         key_len = crypto_data_len - 4; /* 4 bytes salt at end */
 162
 163         memcpy(aes_gcm->aes_key, x->aead->alg_key, key_len);
 164         aes_gcm->key_len = key_len * 8;
 165
 166         /* salt and seq_iv */
 167         aead = x->data;
 168         geniv_ctx = crypto_aead_ctx(aead);
 169         ivsize = crypto_aead_ivsize(aead);
 170         memcpy(&aes_gcm->seq_iv, &geniv_ctx->salt, ivsize);
 171         memcpy(&aes_gcm->salt, x->aead->alg_key + key_len,
 172                sizeof(aes_gcm->salt));
 173
 174         /* iv len */
 175         aes_gcm->icv_len = x->aead->alg_icv_len;
 176
 177         /* esn */
 178         if (sa_entry->esn_state.trigger) {
 179                 attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
 180                 attrs->esn = sa_entry->esn_state.esn;
 181                 if (sa_entry->esn_state.overlap)
 182                         attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
 183         }
 184
 185         /* rx handle */
 186         attrs->sa_handle = sa_entry->handle;
 187
 188         /* algo type */
 189         attrs->keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;
 190
 191         /* action */
 192         attrs->action = (!(x->xso.flags & XFRM_OFFLOAD_INBOUND)) ?
 193                         MLX5_ACCEL_ESP_ACTION_ENCRYPT :
 194                         MLX5_ACCEL_ESP_ACTION_DECRYPT;
 195         /* flags */
 196         attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ?
 197                         MLX5_ACCEL_ESP_FLAGS_TRANSPORT :
 198                         MLX5_ACCEL_ESP_FLAGS_TUNNEL;
 199 }
 200
 201 static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x)
 202 {
 203         struct net_device *netdev = x->xso.dev;
 204         struct mlx5e_priv *priv;
 205
 206         priv = netdev_priv(netdev);
 207
 208         if (x->props.aalgo != SADB_AALG_NONE) {
 209                 netdev_info(netdev, "Cannot offload authenticated xfrm states\n");
 210                 return -EINVAL;
 211         }
 212         if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) {
 213                 netdev_info(netdev, "Only AES-GCM-ICV16 xfrm state may be offloaded\n");
 214                 return -EINVAL;
 215         }
 216         if (x->props.calgo != SADB_X_CALG_NONE) {
 217                 netdev_info(netdev, "Cannot offload compressed xfrm states\n");
 218                 return -EINVAL;
 219         }
 220         if (x->props.flags & XFRM_STATE_ESN &&
 221             !(mlx5_accel_ipsec_device_caps(priv->mdev) &
 222             MLX5_ACCEL_IPSEC_CAP_ESN)) {
 223                 netdev_info(netdev, "Cannot offload ESN xfrm states\n");
 224                 return -EINVAL;
 225         }
 226         if (x->props.family != AF_INET &&
 227             x->props.family != AF_INET6) {
 228                 netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n");
 229                 return -EINVAL;
 230         }
 231         if (x->props.mode != XFRM_MODE_TRANSPORT &&
 232             x->props.mode != XFRM_MODE_TUNNEL) {
 233                 dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n");
 234                 return -EINVAL;
 235         }
 236         if (x->id.proto != IPPROTO_ESP) {
 237                 netdev_info(netdev, "Only ESP xfrm state may be offloaded\n");
 238                 return -EINVAL;
 239         }
 240         if (x->encap) {
 241                 netdev_info(netdev, "Encapsulated xfrm state may not be offloaded\n");
 242                 return -EINVAL;
 243         }
 244         if (!x->aead) {
 245                 netdev_info(netdev, "Cannot offload xfrm states without aead\n");
 246                 return -EINVAL;
 247         }
 248         if (x->aead->alg_icv_len != 128) {
 249                 netdev_info(netdev, "Cannot offload xfrm states with AEAD ICV length other than 128bit\n");
 250                 return -EINVAL;
 251         }
 252         if ((x->aead->alg_key_len != 128 + 32) &&
 253             (x->aead->alg_key_len != 256 + 32)) {
 254                 netdev_info(netdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
 255                 return -EINVAL;
 256         }
 257         if (x->tfcpad) {
 258                 netdev_info(netdev, "Cannot offload xfrm states with tfc padding\n");
 259                 return -EINVAL;
 260         }
 261         if (!x->geniv) {
 262                 netdev_info(netdev, "Cannot offload xfrm states without geniv\n");
 263                 return -EINVAL;
 264         }
 265         if (strcmp(x->geniv, "seqiv")) {
 266                 netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n");
 267                 return -EINVAL;
 268         }
 269         if (x->props.family == AF_INET6 &&
 270             !(mlx5_accel_ipsec_device_caps(priv->mdev) &
 271              MLX5_ACCEL_IPSEC_CAP_IPV6)) {
 272                 netdev_info(netdev, "IPv6 xfrm state offload is not supported by this device\n");
 273                 return -EINVAL;
 274         }
 275         return 0;
 276 }
 277
 278 static int mlx5e_xfrm_add_state(struct xfrm_state *x)
 279 {
 280         struct mlx5e_ipsec_sa_entry *sa_entry = NULL;
 281         struct net_device *netdev = x->xso.dev;
 282         struct mlx5_accel_esp_xfrm_attrs attrs;
 283         struct mlx5e_priv *priv;
 284         __be32 saddr[4] = {0}, daddr[4] = {0}, spi;
 285         bool is_ipv6 = false;
 286         int err;
 287
 288         priv = netdev_priv(netdev);
 289
 290         err = mlx5e_xfrm_validate_state(x);
 291         if (err)
 292                 return err;
 293
 294         sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
 295         if (!sa_entry) {
 296                 err = -ENOMEM;
 297                 goto out;
 298         }
 299
 300         sa_entry->x = x;
 301         sa_entry->ipsec = priv->ipsec;
 302
 303         /* Add the SA to handle processed incoming packets before the add SA
 304          * completion was received
 305          */
 306         if (x->xso.flags & XFRM_OFFLOAD_INBOUND) {
 307                 err = mlx5e_ipsec_sadb_rx_add(sa_entry);
 308                 if (err) {
 309                         netdev_info(netdev, "Failed adding to SADB_RX: %d\n", err);
 310                         goto err_entry;
 311                 }
 312         } else {
 313                 sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ?
 314                                 mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv;
 315         }
 316
 317         /* check esn */
 318         mlx5e_ipsec_update_esn_state(sa_entry);
 319
 320         /* create xfrm */
 321         mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs);
 322         sa_entry->xfrm =
 323                 mlx5_accel_esp_create_xfrm(priv->mdev, &attrs,
 324                                            MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA);
 325         if (IS_ERR(sa_entry->xfrm)) {
 326                 err = PTR_ERR(sa_entry->xfrm);
 327                 goto err_sadb_rx;
 328         }
 329
 330         /* create hw context */
 331         if (x->props.family == AF_INET) {
 332                 saddr[3] = x->props.saddr.a4;
 333                 daddr[3] = x->id.daddr.a4;
 334         } else {
 335                 memcpy(saddr, x->props.saddr.a6, sizeof(saddr));
 336                 memcpy(daddr, x->id.daddr.a6, sizeof(daddr));
 337                 is_ipv6 = true;
 338         }
 339         spi = x->id.spi;
 340         sa_entry->hw_context =
 341                         mlx5_accel_esp_create_hw_context(priv->mdev,
 342                                                          sa_entry->xfrm,
 343                                                          saddr, daddr, spi,
 344                                                          is_ipv6);
 345         if (IS_ERR(sa_entry->hw_context)) {
 346                 err = PTR_ERR(sa_entry->hw_context);
 347                 goto err_xfrm;
 348         }
 349
 350         x->xso.offload_handle = (unsigned long)sa_entry;
 351         goto out;
 352
 353 err_xfrm:
 354         mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm);
 355 err_sadb_rx:
 356         if (x->xso.flags & XFRM_OFFLOAD_INBOUND) {
 357                 mlx5e_ipsec_sadb_rx_del(sa_entry);
 358                 mlx5e_ipsec_sadb_rx_free(sa_entry);
 359         }
 360 err_entry:
 361         kfree(sa_entry);
 362 out:
 363         return err;
 364 }
 365
 366 static void mlx5e_xfrm_del_state(struct xfrm_state *x)
 367 {
 368         struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
 369
 370         if (!sa_entry)
 371                 return;
 372
 373         if (x->xso.flags & XFRM_OFFLOAD_INBOUND)
 374                 mlx5e_ipsec_sadb_rx_del(sa_entry);
 375 }
 376
 377 static void mlx5e_xfrm_free_state(struct xfrm_state *x)
 378 {
 379         struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
 380
 381         if (!sa_entry)
 382                 return;
 383
 384         if (sa_entry->hw_context) {
 385                 flush_workqueue(sa_entry->ipsec->wq);
 386                 mlx5_accel_esp_free_hw_context(sa_entry->hw_context);
 387                 mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm);
 388         }
 389
 390         if (x->xso.flags & XFRM_OFFLOAD_INBOUND)
 391                 mlx5e_ipsec_sadb_rx_free(sa_entry);
 392
 393         kfree(sa_entry);
 394 }
 395
 396 int mlx5e_ipsec_init(struct mlx5e_priv *priv)
 397 {
 398         struct mlx5e_ipsec *ipsec = NULL;
 399
 400         if (!MLX5_IPSEC_DEV(priv->mdev)) {
 401                 netdev_dbg(priv->netdev, "Not an IPSec offload device\n");
 402                 return 0;
 403         }
 404
 405         ipsec = kzalloc(sizeof(*ipsec), GFP_KERNEL);
 406         if (!ipsec)
 407                 return -ENOMEM;
 408
 409         hash_init(ipsec->sadb_rx);
 410         spin_lock_init(&ipsec->sadb_rx_lock);
 411         ida_init(&ipsec->halloc);
 412         ipsec->en_priv = priv;
 413         ipsec->en_priv->ipsec = ipsec;
 414         ipsec->no_trailer = !!(mlx5_accel_ipsec_device_caps(priv->mdev) &
 415                                MLX5_ACCEL_IPSEC_CAP_RX_NO_TRAILER);
 416         ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0,
 417                                             priv->netdev->name);
 418         if (!ipsec->wq) {
 419                 kfree(ipsec);
 420                 return -ENOMEM;
 421         }
 422         netdev_dbg(priv->netdev, "IPSec attached to netdevice\n");
 423         return 0;
 424 }
 425
 426 void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv)
 427 {
 428         struct mlx5e_ipsec *ipsec = priv->ipsec;
 429
 430         if (!ipsec)
 431                 return;
 432
 433         drain_workqueue(ipsec->wq);
 434         destroy_workqueue(ipsec->wq);
 435
 436         ida_destroy(&ipsec->halloc);
 437         kfree(ipsec);
 438         priv->ipsec = NULL;
 439 }
 440
 441 static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
 442 {
 443         if (x->props.family == AF_INET) {
 444                 /* Offload with IPv4 options is not supported yet */
 445                 if (ip_hdr(skb)->ihl > 5)
 446                         return false;
 447         } else {
 448                 /* Offload with IPv6 extension headers is not support yet */
 449                 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
 450                         return false;
 451         }
 452
 453         return true;
 454 }
 455
 456 struct mlx5e_ipsec_modify_state_work {
 457         struct work_struct              work;
 458         struct mlx5_accel_esp_xfrm_attrs attrs;
 459         struct mlx5e_ipsec_sa_entry     *sa_entry;
 460 };
 461
 462 static void _update_xfrm_state(struct work_struct *work)
 463 {
 464         int ret;
 465         struct mlx5e_ipsec_modify_state_work *modify_work =
 466                 container_of(work, struct mlx5e_ipsec_modify_state_work, work);
 467         struct mlx5e_ipsec_sa_entry *sa_entry = modify_work->sa_entry;
 468
 469         ret = mlx5_accel_esp_modify_xfrm(sa_entry->xfrm,
 470                                          &modify_work->attrs);
 471         if (ret)
 472                 netdev_warn(sa_entry->ipsec->en_priv->netdev,
 473                             "Not an IPSec offload device\n");
 474
 475         kfree(modify_work);
 476 }
 477
 478 static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x)
 479 {
 480         struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
 481         struct mlx5e_ipsec_modify_state_work *modify_work;
 482         bool need_update;
 483
 484         if (!sa_entry)
 485                 return;
 486
 487         need_update = mlx5e_ipsec_update_esn_state(sa_entry);
 488         if (!need_update)
 489                 return;
 490
 491         modify_work = kzalloc(sizeof(*modify_work), GFP_ATOMIC);
 492         if (!modify_work)
 493                 return;
 494
 495         mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs);
 496         modify_work->sa_entry = sa_entry;
 497
 498         INIT_WORK(&modify_work->work, _update_xfrm_state);
 499         WARN_ON(!queue_work(sa_entry->ipsec->wq, &modify_work->work));
 500 }
 501
 502 static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = {
 503         .xdo_dev_state_add      = mlx5e_xfrm_add_state,
 504         .xdo_dev_state_delete   = mlx5e_xfrm_del_state,
 505         .xdo_dev_state_free     = mlx5e_xfrm_free_state,
 506         .xdo_dev_offload_ok     = mlx5e_ipsec_offload_ok,
 507         .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state,
 508 };
 509
 510 void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv)
 511 {
 512         struct mlx5_core_dev *mdev = priv->mdev;
 513         struct net_device *netdev = priv->netdev;
 514
 515         if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_ESP) ||
 516             !MLX5_CAP_ETH(mdev, swp)) {
 517                 mlx5_core_dbg(mdev, "mlx5e: ESP and SWP offload not supported\n");
 518                 return;
 519         }
 520
 521         mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n");
 522         netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops;
 523         netdev->features |= NETIF_F_HW_ESP;
 524         netdev->hw_enc_features |= NETIF_F_HW_ESP;
 525
 526         if (!MLX5_CAP_ETH(mdev, swp_csum)) {
 527                 mlx5_core_dbg(mdev, "mlx5e: SWP checksum not supported\n");
 528                 return;
 529         }
 530
 531         netdev->features |= NETIF_F_HW_ESP_TX_CSUM;
 532         netdev->hw_enc_features |= NETIF_F_HW_ESP_TX_CSUM;
 533
 534         if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_LSO) ||
 535             !MLX5_CAP_ETH(mdev, swp_lso)) {
 536                 mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n");
 537                 return;
 538         }
 539
 540         mlx5_core_dbg(mdev, "mlx5e: ESP GSO capability turned on\n");
 541         netdev->features |= NETIF_F_GSO_ESP;
 542         netdev->hw_features |= NETIF_F_GSO_ESP;
 543         netdev->hw_enc_features |= NETIF_F_GSO_ESP;
 544 }