drivers/net/ethernet/netronome/nfp/bpf/cmsg.c

   1 /*
   2  * Copyright (C) 2017-2018 Netronome Systems, Inc.
   3  *
   4  * This software is dual licensed under the GNU General License Version 2,
   5  * June 1991 as shown in the file COPYING in the top-level directory of this
   6  * source tree or the BSD 2-Clause License provided below.  You have the
   7  * option to license this software under the complete terms of either license.
   8  *
   9  * The BSD 2-Clause License:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      1. Redistributions of source code must retain the above
  16  *         copyright notice, this list of conditions and the following
  17  *         disclaimer.
  18  *
  19  *      2. Redistributions in binary form must reproduce the above
  20  *         copyright notice, this list of conditions and the following
  21  *         disclaimer in the documentation and/or other materials
  22  *         provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/bpf.h>
  35 #include <linux/bitops.h>
  36 #include <linux/bug.h>
  37 #include <linux/jiffies.h>
  38 #include <linux/skbuff.h>
  39 #include <linux/wait.h>
  40
  41 #include "../nfp_app.h"
  42 #include "../nfp_net.h"
  43 #include "fw.h"
  44 #include "main.h"
  45
  46 #define NFP_BPF_TAG_ALLOC_SPAN  (U16_MAX / 4)
  47
  48 static bool nfp_bpf_all_tags_busy(struct nfp_app_bpf *bpf)
  49 {
  50         u16 used_tags;
  51
  52         used_tags = bpf->tag_alloc_next - bpf->tag_alloc_last;
  53
  54         return used_tags > NFP_BPF_TAG_ALLOC_SPAN;
  55 }
  56
  57 static int nfp_bpf_alloc_tag(struct nfp_app_bpf *bpf)
  58 {
  59         /* All FW communication for BPF is request-reply.  To make sure we
  60          * don't reuse the message ID too early after timeout - limit the
  61          * number of requests in flight.
  62          */
  63         if (nfp_bpf_all_tags_busy(bpf)) {
  64                 cmsg_warn(bpf, "all FW request contexts busy!\n");
  65                 return -EAGAIN;
  66         }
  67
  68         WARN_ON(__test_and_set_bit(bpf->tag_alloc_next, bpf->tag_allocator));
  69         return bpf->tag_alloc_next++;
  70 }
  71
  72 static void nfp_bpf_free_tag(struct nfp_app_bpf *bpf, u16 tag)
  73 {
  74         WARN_ON(!__test_and_clear_bit(tag, bpf->tag_allocator));
  75
  76         while (!test_bit(bpf->tag_alloc_last, bpf->tag_allocator) &&
  77                bpf->tag_alloc_last != bpf->tag_alloc_next)
  78                 bpf->tag_alloc_last++;
  79 }
  80
  81 static struct sk_buff *
  82 nfp_bpf_cmsg_alloc(struct nfp_app_bpf *bpf, unsigned int size)
  83 {
  84         struct sk_buff *skb;
  85
  86         skb = nfp_app_ctrl_msg_alloc(bpf->app, size, GFP_KERNEL);
  87         skb_put(skb, size);
  88
  89         return skb;
  90 }
  91
  92 static struct sk_buff *
  93 nfp_bpf_cmsg_map_req_alloc(struct nfp_app_bpf *bpf, unsigned int n)
  94 {
  95         unsigned int size;
  96
  97         size = sizeof(struct cmsg_req_map_op);
  98         size += sizeof(struct cmsg_key_value_pair) * n;
  99
 100         return nfp_bpf_cmsg_alloc(bpf, size);
 101 }
 102
 103 static u8 nfp_bpf_cmsg_get_type(struct sk_buff *skb)
 104 {
 105         struct cmsg_hdr *hdr;
 106
 107         hdr = (struct cmsg_hdr *)skb->data;
 108
 109         return hdr->type;
 110 }
 111
 112 static unsigned int nfp_bpf_cmsg_get_tag(struct sk_buff *skb)
 113 {
 114         struct cmsg_hdr *hdr;
 115
 116         hdr = (struct cmsg_hdr *)skb->data;
 117
 118         return be16_to_cpu(hdr->tag);
 119 }
 120
 121 static struct sk_buff *__nfp_bpf_reply(struct nfp_app_bpf *bpf, u16 tag)
 122 {
 123         unsigned int msg_tag;
 124         struct sk_buff *skb;
 125
 126         skb_queue_walk(&bpf->cmsg_replies, skb) {
 127                 msg_tag = nfp_bpf_cmsg_get_tag(skb);
 128                 if (msg_tag == tag) {
 129                         nfp_bpf_free_tag(bpf, tag);
 130                         __skb_unlink(skb, &bpf->cmsg_replies);
 131                         return skb;
 132                 }
 133         }
 134
 135         return NULL;
 136 }
 137
 138 static struct sk_buff *nfp_bpf_reply(struct nfp_app_bpf *bpf, u16 tag)
 139 {
 140         struct sk_buff *skb;
 141
 142         nfp_ctrl_lock(bpf->app->ctrl);
 143         skb = __nfp_bpf_reply(bpf, tag);
 144         nfp_ctrl_unlock(bpf->app->ctrl);
 145
 146         return skb;
 147 }
 148
 149 static struct sk_buff *nfp_bpf_reply_drop_tag(struct nfp_app_bpf *bpf, u16 tag)
 150 {
 151         struct sk_buff *skb;
 152
 153         nfp_ctrl_lock(bpf->app->ctrl);
 154         skb = __nfp_bpf_reply(bpf, tag);
 155         if (!skb)
 156                 nfp_bpf_free_tag(bpf, tag);
 157         nfp_ctrl_unlock(bpf->app->ctrl);
 158
 159         return skb;
 160 }
 161
 162 static struct sk_buff *
 163 nfp_bpf_cmsg_wait_reply(struct nfp_app_bpf *bpf, enum nfp_bpf_cmsg_type type,
 164                         int tag)
 165 {
 166         struct sk_buff *skb;
 167         int i, err;
 168
 169         for (i = 0; i < 50; i++) {
 170                 udelay(4);
 171                 skb = nfp_bpf_reply(bpf, tag);
 172                 if (skb)
 173                         return skb;
 174         }
 175
 176         err = wait_event_interruptible_timeout(bpf->cmsg_wq,
 177                                                skb = nfp_bpf_reply(bpf, tag),
 178                                                msecs_to_jiffies(5000));
 179         /* We didn't get a response - try last time and atomically drop
 180          * the tag even if no response is matched.
 181          */
 182         if (!skb)
 183                 skb = nfp_bpf_reply_drop_tag(bpf, tag);
 184         if (err < 0) {
 185                 cmsg_warn(bpf, "%s waiting for response to 0x%02x: %d\n",
 186                           err == ERESTARTSYS ? "interrupted" : "error",
 187                           type, err);
 188                 return ERR_PTR(err);
 189         }
 190         if (!skb) {
 191                 cmsg_warn(bpf, "timeout waiting for response to 0x%02x\n",
 192                           type);
 193                 return ERR_PTR(-ETIMEDOUT);
 194         }
 195
 196         return skb;
 197 }
 198
 199 static struct sk_buff *
 200 nfp_bpf_cmsg_communicate(struct nfp_app_bpf *bpf, struct sk_buff *skb,
 201                          enum nfp_bpf_cmsg_type type, unsigned int reply_size)
 202 {
 203         struct cmsg_hdr *hdr;
 204         int tag;
 205
 206         nfp_ctrl_lock(bpf->app->ctrl);
 207         tag = nfp_bpf_alloc_tag(bpf);
 208         if (tag < 0) {
 209                 nfp_ctrl_unlock(bpf->app->ctrl);
 210                 dev_kfree_skb_any(skb);
 211                 return ERR_PTR(tag);
 212         }
 213
 214         hdr = (void *)skb->data;
 215         hdr->ver = CMSG_MAP_ABI_VERSION;
 216         hdr->type = type;
 217         hdr->tag = cpu_to_be16(tag);
 218
 219         __nfp_app_ctrl_tx(bpf->app, skb);
 220
 221         nfp_ctrl_unlock(bpf->app->ctrl);
 222
 223         skb = nfp_bpf_cmsg_wait_reply(bpf, type, tag);
 224         if (IS_ERR(skb))
 225                 return skb;
 226
 227         hdr = (struct cmsg_hdr *)skb->data;
 228         if (hdr->type != __CMSG_REPLY(type)) {
 229                 cmsg_warn(bpf, "cmsg drop - wrong type 0x%02x != 0x%02lx!\n",
 230                           hdr->type, __CMSG_REPLY(type));
 231                 goto err_free;
 232         }
 233         /* 0 reply_size means caller will do the validation */
 234         if (reply_size && skb->len != reply_size) {
 235                 cmsg_warn(bpf, "cmsg drop - type 0x%02x wrong size %d != %d!\n",
 236                           type, skb->len, reply_size);
 237                 goto err_free;
 238         }
 239
 240         return skb;
 241 err_free:
 242         dev_kfree_skb_any(skb);
 243         return ERR_PTR(-EIO);
 244 }
 245
 246 static int
 247 nfp_bpf_ctrl_rc_to_errno(struct nfp_app_bpf *bpf,
 248                          struct cmsg_reply_map_simple *reply)
 249 {
 250         static const int res_table[] = {
 251                 [CMSG_RC_SUCCESS]       = 0,
 252                 [CMSG_RC_ERR_MAP_FD]    = -EBADFD,
 253                 [CMSG_RC_ERR_MAP_NOENT] = -ENOENT,
 254                 [CMSG_RC_ERR_MAP_ERR]   = -EINVAL,
 255                 [CMSG_RC_ERR_MAP_PARSE] = -EIO,
 256                 [CMSG_RC_ERR_MAP_EXIST] = -EEXIST,
 257                 [CMSG_RC_ERR_MAP_NOMEM] = -ENOMEM,
 258                 [CMSG_RC_ERR_MAP_E2BIG] = -E2BIG,
 259         };
 260         u32 rc;
 261
 262         rc = be32_to_cpu(reply->rc);
 263         if (rc >= ARRAY_SIZE(res_table)) {
 264                 cmsg_warn(bpf, "FW responded with invalid status: %u\n", rc);
 265                 return -EIO;
 266         }
 267
 268         return res_table[rc];
 269 }
 270
 271 long long int
 272 nfp_bpf_ctrl_alloc_map(struct nfp_app_bpf *bpf, struct bpf_map *map)
 273 {
 274         struct cmsg_reply_map_alloc_tbl *reply;
 275         struct cmsg_req_map_alloc_tbl *req;
 276         struct sk_buff *skb;
 277         u32 tid;
 278         int err;
 279
 280         skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
 281         if (!skb)
 282                 return -ENOMEM;
 283
 284         req = (void *)skb->data;
 285         req->key_size = cpu_to_be32(map->key_size);
 286         req->value_size = cpu_to_be32(map->value_size);
 287         req->max_entries = cpu_to_be32(map->max_entries);
 288         req->map_type = cpu_to_be32(map->map_type);
 289         req->map_flags = 0;
 290
 291         skb = nfp_bpf_cmsg_communicate(bpf, skb, CMSG_TYPE_MAP_ALLOC,
 292                                        sizeof(*reply));
 293         if (IS_ERR(skb))
 294                 return PTR_ERR(skb);
 295
 296         reply = (void *)skb->data;
 297         err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
 298         if (err)
 299                 goto err_free;
 300
 301         tid = be32_to_cpu(reply->tid);
 302         dev_consume_skb_any(skb);
 303
 304         return tid;
 305 err_free:
 306         dev_kfree_skb_any(skb);
 307         return err;
 308 }
 309
 310 void nfp_bpf_ctrl_free_map(struct nfp_app_bpf *bpf, struct nfp_bpf_map *nfp_map)
 311 {
 312         struct cmsg_reply_map_free_tbl *reply;
 313         struct cmsg_req_map_free_tbl *req;
 314         struct sk_buff *skb;
 315         int err;
 316
 317         skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
 318         if (!skb) {
 319                 cmsg_warn(bpf, "leaking map - failed to allocate msg\n");
 320                 return;
 321         }
 322
 323         req = (void *)skb->data;
 324         req->tid = cpu_to_be32(nfp_map->tid);
 325
 326         skb = nfp_bpf_cmsg_communicate(bpf, skb, CMSG_TYPE_MAP_FREE,
 327                                        sizeof(*reply));
 328         if (IS_ERR(skb)) {
 329                 cmsg_warn(bpf, "leaking map - I/O error\n");
 330                 return;
 331         }
 332
 333         reply = (void *)skb->data;
 334         err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
 335         if (err)
 336                 cmsg_warn(bpf, "leaking map - FW responded with: %d\n", err);
 337
 338         dev_consume_skb_any(skb);
 339 }
 340
 341 static int
 342 nfp_bpf_ctrl_entry_op(struct bpf_offloaded_map *offmap,
 343                       enum nfp_bpf_cmsg_type op,
 344                       u8 *key, u8 *value, u64 flags, u8 *out_key, u8 *out_value)
 345 {
 346         struct nfp_bpf_map *nfp_map = offmap->dev_priv;
 347         struct nfp_app_bpf *bpf = nfp_map->bpf;
 348         struct bpf_map *map = &offmap->map;
 349         struct cmsg_reply_map_op *reply;
 350         struct cmsg_req_map_op *req;
 351         struct sk_buff *skb;
 352         int err;
 353
 354         /* FW messages have no space for more than 32 bits of flags */
 355         if (flags >> 32)
 356                 return -EOPNOTSUPP;
 357
 358         skb = nfp_bpf_cmsg_map_req_alloc(bpf, 1);
 359         if (!skb)
 360                 return -ENOMEM;
 361
 362         req = (void *)skb->data;
 363         req->tid = cpu_to_be32(nfp_map->tid);
 364         req->count = cpu_to_be32(1);
 365         req->flags = cpu_to_be32(flags);
 366
 367         /* Copy inputs */
 368         if (key)
 369                 memcpy(&req->elem[0].key, key, map->key_size);
 370         if (value)
 371                 memcpy(&req->elem[0].value, value, map->value_size);
 372
 373         skb = nfp_bpf_cmsg_communicate(bpf, skb, op,
 374                                        sizeof(*reply) + sizeof(*reply->elem));
 375         if (IS_ERR(skb))
 376                 return PTR_ERR(skb);
 377
 378         reply = (void *)skb->data;
 379         err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
 380         if (err)
 381                 goto err_free;
 382
 383         /* Copy outputs */
 384         if (out_key)
 385                 memcpy(out_key, &reply->elem[0].key, map->key_size);
 386         if (out_value)
 387                 memcpy(out_value, &reply->elem[0].value, map->value_size);
 388
 389         dev_consume_skb_any(skb);
 390
 391         return 0;
 392 err_free:
 393         dev_kfree_skb_any(skb);
 394         return err;
 395 }
 396
 397 int nfp_bpf_ctrl_update_entry(struct bpf_offloaded_map *offmap,
 398                               void *key, void *value, u64 flags)
 399 {
 400         return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_UPDATE,
 401                                      key, value, flags, NULL, NULL);
 402 }
 403
 404 int nfp_bpf_ctrl_del_entry(struct bpf_offloaded_map *offmap, void *key)
 405 {
 406         return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_DELETE,
 407                                      key, NULL, 0, NULL, NULL);
 408 }
 409
 410 int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap,
 411                               void *key, void *value)
 412 {
 413         return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_LOOKUP,
 414                                      key, NULL, 0, NULL, value);
 415 }
 416
 417 int nfp_bpf_ctrl_getfirst_entry(struct bpf_offloaded_map *offmap,
 418                                 void *next_key)
 419 {
 420         return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_GETFIRST,
 421                                      NULL, NULL, 0, next_key, NULL);
 422 }
 423
 424 int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap,
 425                                void *key, void *next_key)
 426 {
 427         return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_GETNEXT,
 428                                      key, NULL, 0, next_key, NULL);
 429 }
 430
 431 void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb)
 432 {
 433         struct nfp_app_bpf *bpf = app->priv;
 434         unsigned int tag;
 435
 436         if (unlikely(skb->len < sizeof(struct cmsg_reply_map_simple))) {
 437                 cmsg_warn(bpf, "cmsg drop - too short %d!\n", skb->len);
 438                 goto err_free;
 439         }
 440
 441         if (nfp_bpf_cmsg_get_type(skb) == CMSG_TYPE_BPF_EVENT) {
 442                 if (!nfp_bpf_event_output(bpf, skb->data, skb->len))
 443                         dev_consume_skb_any(skb);
 444                 else
 445                         dev_kfree_skb_any(skb);
 446                 return;
 447         }
 448
 449         nfp_ctrl_lock(bpf->app->ctrl);
 450
 451         tag = nfp_bpf_cmsg_get_tag(skb);
 452         if (unlikely(!test_bit(tag, bpf->tag_allocator))) {
 453                 cmsg_warn(bpf, "cmsg drop - no one is waiting for tag %u!\n",
 454                           tag);
 455                 goto err_unlock;
 456         }
 457
 458         __skb_queue_tail(&bpf->cmsg_replies, skb);
 459         wake_up_interruptible_all(&bpf->cmsg_wq);
 460
 461         nfp_ctrl_unlock(bpf->app->ctrl);
 462
 463         return;
 464 err_unlock:
 465         nfp_ctrl_unlock(bpf->app->ctrl);
 466 err_free:
 467         dev_kfree_skb_any(skb);
 468 }
 469
 470 void
 471 nfp_bpf_ctrl_msg_rx_raw(struct nfp_app *app, const void *data, unsigned int len)
 472 {
 473         struct nfp_app_bpf *bpf = app->priv;
 474         const struct cmsg_hdr *hdr = data;
 475
 476         if (unlikely(len < sizeof(struct cmsg_reply_map_simple))) {
 477                 cmsg_warn(bpf, "cmsg drop - too short %d!\n", len);
 478                 return;
 479         }
 480
 481         if (hdr->type == CMSG_TYPE_BPF_EVENT)
 482                 nfp_bpf_event_output(bpf, data, len);
 483         else
 484                 cmsg_warn(bpf, "cmsg drop - msg type %d with raw buffer!\n",
 485                           hdr->type);
 486 }