net/tls/tls_sw.c

   1 /*
   2  * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
   3  * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
   4  * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
   5  * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
   6  * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  */
  36
  37 #include <linux/module.h>
  38 #include <crypto/aead.h>
  39
  40 #include <net/tls.h>
  41
  42 static inline void tls_make_aad(int recv,
  43                                 char *buf,
  44                                 size_t size,
  45                                 char *record_sequence,
  46                                 int record_sequence_size,
  47                                 unsigned char record_type)
  48 {
  49         memcpy(buf, record_sequence, record_sequence_size);
  50
  51         buf[8] = record_type;
  52         buf[9] = TLS_1_2_VERSION_MAJOR;
  53         buf[10] = TLS_1_2_VERSION_MINOR;
  54         buf[11] = size >> 8;
  55         buf[12] = size & 0xFF;
  56 }
  57
  58 static void trim_sg(struct sock *sk, struct scatterlist *sg,
  59                     int *sg_num_elem, unsigned int *sg_size, int target_size)
  60 {
  61         int i = *sg_num_elem - 1;
  62         int trim = *sg_size - target_size;
  63
  64         if (trim <= 0) {
  65                 WARN_ON(trim < 0);
  66                 return;
  67         }
  68
  69         *sg_size = target_size;
  70         while (trim >= sg[i].length) {
  71                 trim -= sg[i].length;
  72                 sk_mem_uncharge(sk, sg[i].length);
  73                 put_page(sg_page(&sg[i]));
  74                 i--;
  75
  76                 if (i < 0)
  77                         goto out;
  78         }
  79
  80         sg[i].length -= trim;
  81         sk_mem_uncharge(sk, trim);
  82
  83 out:
  84         *sg_num_elem = i + 1;
  85 }
  86
  87 static void trim_both_sgl(struct sock *sk, int target_size)
  88 {
  89         struct tls_context *tls_ctx = tls_get_ctx(sk);
  90         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
  91
  92         trim_sg(sk, ctx->sg_plaintext_data,
  93                 &ctx->sg_plaintext_num_elem,
  94                 &ctx->sg_plaintext_size,
  95                 target_size);
  96
  97         if (target_size > 0)
  98                 target_size += tls_ctx->overhead_size;
  99
 100         trim_sg(sk, ctx->sg_encrypted_data,
 101                 &ctx->sg_encrypted_num_elem,
 102                 &ctx->sg_encrypted_size,
 103                 target_size);
 104 }
 105
 106 static int alloc_sg(struct sock *sk, int len, struct scatterlist *sg,
 107                     int *sg_num_elem, unsigned int *sg_size,
 108                     int first_coalesce)
 109 {
 110         struct page_frag *pfrag;
 111         unsigned int size = *sg_size;
 112         int num_elem = *sg_num_elem, use = 0, rc = 0;
 113         struct scatterlist *sge;
 114         unsigned int orig_offset;
 115
 116         len -= size;
 117         pfrag = sk_page_frag(sk);
 118
 119         while (len > 0) {
 120                 if (!sk_page_frag_refill(sk, pfrag)) {
 121                         rc = -ENOMEM;
 122                         goto out;
 123                 }
 124
 125                 use = min_t(int, len, pfrag->size - pfrag->offset);
 126
 127                 if (!sk_wmem_schedule(sk, use)) {
 128                         rc = -ENOMEM;
 129                         goto out;
 130                 }
 131
 132                 sk_mem_charge(sk, use);
 133                 size += use;
 134                 orig_offset = pfrag->offset;
 135                 pfrag->offset += use;
 136
 137                 sge = sg + num_elem - 1;
 138
 139                 if (num_elem > first_coalesce && sg_page(sge) == pfrag->page &&
 140                     sge->offset + sge->length == orig_offset) {
 141                         sge->length += use;
 142                 } else {
 143                         sge++;
 144                         sg_unmark_end(sge);
 145                         sg_set_page(sge, pfrag->page, use, orig_offset);
 146                         get_page(pfrag->page);
 147                         ++num_elem;
 148                         if (num_elem == MAX_SKB_FRAGS) {
 149                                 rc = -ENOSPC;
 150                                 break;
 151                         }
 152                 }
 153
 154                 len -= use;
 155         }
 156         goto out;
 157
 158 out:
 159         *sg_size = size;
 160         *sg_num_elem = num_elem;
 161         return rc;
 162 }
 163
 164 static int alloc_encrypted_sg(struct sock *sk, int len)
 165 {
 166         struct tls_context *tls_ctx = tls_get_ctx(sk);
 167         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 168         int rc = 0;
 169
 170         rc = alloc_sg(sk, len, ctx->sg_encrypted_data,
 171                       &ctx->sg_encrypted_num_elem, &ctx->sg_encrypted_size, 0);
 172
 173         if (rc == -ENOSPC)
 174                 ctx->sg_encrypted_num_elem = ARRAY_SIZE(ctx->sg_encrypted_data);
 175
 176         return rc;
 177 }
 178
 179 static int alloc_plaintext_sg(struct sock *sk, int len)
 180 {
 181         struct tls_context *tls_ctx = tls_get_ctx(sk);
 182         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 183         int rc = 0;
 184
 185         rc = alloc_sg(sk, len, ctx->sg_plaintext_data,
 186                       &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size,
 187                       tls_ctx->pending_open_record_frags);
 188
 189         if (rc == -ENOSPC)
 190                 ctx->sg_plaintext_num_elem = ARRAY_SIZE(ctx->sg_plaintext_data);
 191
 192         return rc;
 193 }
 194
 195 static void free_sg(struct sock *sk, struct scatterlist *sg,
 196                     int *sg_num_elem, unsigned int *sg_size)
 197 {
 198         int i, n = *sg_num_elem;
 199
 200         for (i = 0; i < n; ++i) {
 201                 sk_mem_uncharge(sk, sg[i].length);
 202                 put_page(sg_page(&sg[i]));
 203         }
 204         *sg_num_elem = 0;
 205         *sg_size = 0;
 206 }
 207
 208 static void tls_free_both_sg(struct sock *sk)
 209 {
 210         struct tls_context *tls_ctx = tls_get_ctx(sk);
 211         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 212
 213         free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem,
 214                 &ctx->sg_encrypted_size);
 215
 216         free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
 217                 &ctx->sg_plaintext_size);
 218 }
 219
 220 static int tls_do_encryption(struct tls_context *tls_ctx,
 221                              struct tls_sw_context *ctx,
 222                              struct aead_request *aead_req,
 223                              size_t data_len)
 224 {
 225         int rc;
 226
 227         ctx->sg_encrypted_data[0].offset += tls_ctx->prepend_size;
 228         ctx->sg_encrypted_data[0].length -= tls_ctx->prepend_size;
 229
 230         aead_request_set_tfm(aead_req, ctx->aead_send);
 231         aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
 232         aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out,
 233                                data_len, tls_ctx->iv);
 234         rc = crypto_aead_encrypt(aead_req);
 235
 236         ctx->sg_encrypted_data[0].offset -= tls_ctx->prepend_size;
 237         ctx->sg_encrypted_data[0].length += tls_ctx->prepend_size;
 238
 239         return rc;
 240 }
 241
 242 static int tls_push_record(struct sock *sk, int flags,
 243                            unsigned char record_type)
 244 {
 245         struct tls_context *tls_ctx = tls_get_ctx(sk);
 246         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 247         struct aead_request *req;
 248         int rc;
 249
 250         req = kzalloc(sizeof(struct aead_request) +
 251                       crypto_aead_reqsize(ctx->aead_send), sk->sk_allocation);
 252         if (!req)
 253                 return -ENOMEM;
 254
 255         sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1);
 256         sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1);
 257
 258         tls_make_aad(0, ctx->aad_space, ctx->sg_plaintext_size,
 259                      tls_ctx->rec_seq, tls_ctx->rec_seq_size,
 260                      record_type);
 261
 262         tls_fill_prepend(tls_ctx,
 263                          page_address(sg_page(&ctx->sg_encrypted_data[0])) +
 264                          ctx->sg_encrypted_data[0].offset,
 265                          ctx->sg_plaintext_size, record_type);
 266
 267         tls_ctx->pending_open_record_frags = 0;
 268         set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags);
 269
 270         rc = tls_do_encryption(tls_ctx, ctx, req, ctx->sg_plaintext_size);
 271         if (rc < 0) {
 272                 /* If we are called from write_space and
 273                  * we fail, we need to set this SOCK_NOSPACE
 274                  * to trigger another write_space in the future.
 275                  */
 276                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 277                 goto out_req;
 278         }
 279
 280         free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
 281                 &ctx->sg_plaintext_size);
 282
 283         ctx->sg_encrypted_num_elem = 0;
 284         ctx->sg_encrypted_size = 0;
 285
 286         /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */
 287         rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags);
 288         if (rc < 0 && rc != -EAGAIN)
 289                 tls_err_abort(sk);
 290
 291         tls_advance_record_sn(sk, tls_ctx);
 292 out_req:
 293         kfree(req);
 294         return rc;
 295 }
 296
 297 static int tls_sw_push_pending_record(struct sock *sk, int flags)
 298 {
 299         return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA);
 300 }
 301
 302 static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
 303                               int length)
 304 {
 305         struct tls_context *tls_ctx = tls_get_ctx(sk);
 306         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 307         struct page *pages[MAX_SKB_FRAGS];
 308
 309         size_t offset;
 310         ssize_t copied, use;
 311         int i = 0;
 312         unsigned int size = ctx->sg_plaintext_size;
 313         int num_elem = ctx->sg_plaintext_num_elem;
 314         int rc = 0;
 315         int maxpages;
 316
 317         while (length > 0) {
 318                 i = 0;
 319                 maxpages = ARRAY_SIZE(ctx->sg_plaintext_data) - num_elem;
 320                 if (maxpages == 0) {
 321                         rc = -EFAULT;
 322                         goto out;
 323                 }
 324                 copied = iov_iter_get_pages(from, pages,
 325                                             length,
 326                                             maxpages, &offset);
 327                 if (copied <= 0) {
 328                         rc = -EFAULT;
 329                         goto out;
 330                 }
 331
 332                 iov_iter_advance(from, copied);
 333
 334                 length -= copied;
 335                 size += copied;
 336                 while (copied) {
 337                         use = min_t(int, copied, PAGE_SIZE - offset);
 338
 339                         sg_set_page(&ctx->sg_plaintext_data[num_elem],
 340                                     pages[i], use, offset);
 341                         sg_unmark_end(&ctx->sg_plaintext_data[num_elem]);
 342                         sk_mem_charge(sk, use);
 343
 344                         offset = 0;
 345                         copied -= use;
 346
 347                         ++i;
 348                         ++num_elem;
 349                 }
 350         }
 351
 352 out:
 353         ctx->sg_plaintext_size = size;
 354         ctx->sg_plaintext_num_elem = num_elem;
 355         return rc;
 356 }
 357
 358 static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
 359                              int bytes)
 360 {
 361         struct tls_context *tls_ctx = tls_get_ctx(sk);
 362         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 363         struct scatterlist *sg = ctx->sg_plaintext_data;
 364         int copy, i, rc = 0;
 365
 366         for (i = tls_ctx->pending_open_record_frags;
 367              i < ctx->sg_plaintext_num_elem; ++i) {
 368                 copy = sg[i].length;
 369                 if (copy_from_iter(
 370                                 page_address(sg_page(&sg[i])) + sg[i].offset,
 371                                 copy, from) != copy) {
 372                         rc = -EFAULT;
 373                         goto out;
 374                 }
 375                 bytes -= copy;
 376
 377                 ++tls_ctx->pending_open_record_frags;
 378
 379                 if (!bytes)
 380                         break;
 381         }
 382
 383 out:
 384         return rc;
 385 }
 386
 387 int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 388 {
 389         struct tls_context *tls_ctx = tls_get_ctx(sk);
 390         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 391         int ret;
 392         int required_size;
 393         long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
 394         bool eor = !(msg->msg_flags & MSG_MORE);
 395         size_t try_to_copy, copied = 0;
 396         unsigned char record_type = TLS_RECORD_TYPE_DATA;
 397         int record_room;
 398         bool full_record;
 399         int orig_size;
 400
 401         if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
 402                 return -ENOTSUPP;
 403
 404         lock_sock(sk);
 405
 406         ret = tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo);
 407         if (ret)
 408                 goto send_end;
 409
 410         if (unlikely(msg->msg_controllen)) {
 411                 ret = tls_proccess_cmsg(sk, msg, &record_type);
 412                 if (ret)
 413                         goto send_end;
 414         }
 415
 416         while (msg_data_left(msg)) {
 417                 if (sk->sk_err) {
 418                         ret = -sk->sk_err;
 419                         goto send_end;
 420                 }
 421
 422                 orig_size = ctx->sg_plaintext_size;
 423                 full_record = false;
 424                 try_to_copy = msg_data_left(msg);
 425                 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
 426                 if (try_to_copy >= record_room) {
 427                         try_to_copy = record_room;
 428                         full_record = true;
 429                 }
 430
 431                 required_size = ctx->sg_plaintext_size + try_to_copy +
 432                                 tls_ctx->overhead_size;
 433
 434                 if (!sk_stream_memory_free(sk))
 435                         goto wait_for_sndbuf;
 436 alloc_encrypted:
 437                 ret = alloc_encrypted_sg(sk, required_size);
 438                 if (ret) {
 439                         if (ret != -ENOSPC)
 440                                 goto wait_for_memory;
 441
 442                         /* Adjust try_to_copy according to the amount that was
 443                          * actually allocated. The difference is due
 444                          * to max sg elements limit
 445                          */
 446                         try_to_copy -= required_size - ctx->sg_encrypted_size;
 447                         full_record = true;
 448                 }
 449
 450                 if (full_record || eor) {
 451                         ret = zerocopy_from_iter(sk, &msg->msg_iter,
 452                                                  try_to_copy);
 453                         if (ret)
 454                                 goto fallback_to_reg_send;
 455
 456                         copied += try_to_copy;
 457                         ret = tls_push_record(sk, msg->msg_flags, record_type);
 458                         if (!ret)
 459                                 continue;
 460                         if (ret < 0)
 461                                 goto send_end;
 462
 463                         copied -= try_to_copy;
 464 fallback_to_reg_send:
 465                         iov_iter_revert(&msg->msg_iter,
 466                                         ctx->sg_plaintext_size - orig_size);
 467                         trim_sg(sk, ctx->sg_plaintext_data,
 468                                 &ctx->sg_plaintext_num_elem,
 469                                 &ctx->sg_plaintext_size,
 470                                 orig_size);
 471                 }
 472
 473                 required_size = ctx->sg_plaintext_size + try_to_copy;
 474 alloc_plaintext:
 475                 ret = alloc_plaintext_sg(sk, required_size);
 476                 if (ret) {
 477                         if (ret != -ENOSPC)
 478                                 goto wait_for_memory;
 479
 480                         /* Adjust try_to_copy according to the amount that was
 481                          * actually allocated. The difference is due
 482                          * to max sg elements limit
 483                          */
 484                         try_to_copy -= required_size - ctx->sg_plaintext_size;
 485                         full_record = true;
 486
 487                         trim_sg(sk, ctx->sg_encrypted_data,
 488                                 &ctx->sg_encrypted_num_elem,
 489                                 &ctx->sg_encrypted_size,
 490                                 ctx->sg_plaintext_size +
 491                                 tls_ctx->overhead_size);
 492                 }
 493
 494                 ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy);
 495                 if (ret)
 496                         goto trim_sgl;
 497
 498                 copied += try_to_copy;
 499                 if (full_record || eor) {
 500 push_record:
 501                         ret = tls_push_record(sk, msg->msg_flags, record_type);
 502                         if (ret) {
 503                                 if (ret == -ENOMEM)
 504                                         goto wait_for_memory;
 505
 506                                 goto send_end;
 507                         }
 508                 }
 509
 510                 continue;
 511
 512 wait_for_sndbuf:
 513                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 514 wait_for_memory:
 515                 ret = sk_stream_wait_memory(sk, &timeo);
 516                 if (ret) {
 517 trim_sgl:
 518                         trim_both_sgl(sk, orig_size);
 519                         goto send_end;
 520                 }
 521
 522                 if (tls_is_pending_closed_record(tls_ctx))
 523                         goto push_record;
 524
 525                 if (ctx->sg_encrypted_size < required_size)
 526                         goto alloc_encrypted;
 527
 528                 goto alloc_plaintext;
 529         }
 530
 531 send_end:
 532         ret = sk_stream_error(sk, msg->msg_flags, ret);
 533
 534         release_sock(sk);
 535         return copied ? copied : ret;
 536 }
 537
 538 int tls_sw_sendpage(struct sock *sk, struct page *page,
 539                     int offset, size_t size, int flags)
 540 {
 541         struct tls_context *tls_ctx = tls_get_ctx(sk);
 542         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 543         int ret;
 544         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 545         bool eor;
 546         size_t orig_size = size;
 547         unsigned char record_type = TLS_RECORD_TYPE_DATA;
 548         struct scatterlist *sg;
 549         bool full_record;
 550         int record_room;
 551
 552         if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
 553                       MSG_SENDPAGE_NOTLAST))
 554                 return -ENOTSUPP;
 555
 556         /* No MSG_EOR from splice, only look at MSG_MORE */
 557         eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST));
 558
 559         lock_sock(sk);
 560
 561         sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 562
 563         ret = tls_complete_pending_work(sk, tls_ctx, flags, &timeo);
 564         if (ret)
 565                 goto sendpage_end;
 566
 567         /* Call the sk_stream functions to manage the sndbuf mem. */
 568         while (size > 0) {
 569                 size_t copy, required_size;
 570
 571                 if (sk->sk_err) {
 572                         ret = -sk->sk_err;
 573                         goto sendpage_end;
 574                 }
 575
 576                 full_record = false;
 577                 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
 578                 copy = size;
 579                 if (copy >= record_room) {
 580                         copy = record_room;
 581                         full_record = true;
 582                 }
 583                 required_size = ctx->sg_plaintext_size + copy +
 584                               tls_ctx->overhead_size;
 585
 586                 if (!sk_stream_memory_free(sk))
 587                         goto wait_for_sndbuf;
 588 alloc_payload:
 589                 ret = alloc_encrypted_sg(sk, required_size);
 590                 if (ret) {
 591                         if (ret != -ENOSPC)
 592                                 goto wait_for_memory;
 593
 594                         /* Adjust copy according to the amount that was
 595                          * actually allocated. The difference is due
 596                          * to max sg elements limit
 597                          */
 598                         copy -= required_size - ctx->sg_plaintext_size;
 599                         full_record = true;
 600                 }
 601
 602                 get_page(page);
 603                 sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem;
 604                 sg_set_page(sg, page, copy, offset);
 605                 ctx->sg_plaintext_num_elem++;
 606
 607                 sk_mem_charge(sk, copy);
 608                 offset += copy;
 609                 size -= copy;
 610                 ctx->sg_plaintext_size += copy;
 611                 tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem;
 612
 613                 if (full_record || eor ||
 614                     ctx->sg_plaintext_num_elem ==
 615                     ARRAY_SIZE(ctx->sg_plaintext_data)) {
 616 push_record:
 617                         ret = tls_push_record(sk, flags, record_type);
 618                         if (ret) {
 619                                 if (ret == -ENOMEM)
 620                                         goto wait_for_memory;
 621
 622                                 goto sendpage_end;
 623                         }
 624                 }
 625                 continue;
 626 wait_for_sndbuf:
 627                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 628 wait_for_memory:
 629                 ret = sk_stream_wait_memory(sk, &timeo);
 630                 if (ret) {
 631                         trim_both_sgl(sk, ctx->sg_plaintext_size);
 632                         goto sendpage_end;
 633                 }
 634
 635                 if (tls_is_pending_closed_record(tls_ctx))
 636                         goto push_record;
 637
 638                 goto alloc_payload;
 639         }
 640
 641 sendpage_end:
 642         if (orig_size > size)
 643                 ret = orig_size - size;
 644         else
 645                 ret = sk_stream_error(sk, flags, ret);
 646
 647         release_sock(sk);
 648         return ret;
 649 }
 650
 651 void tls_sw_free_tx_resources(struct sock *sk)
 652 {
 653         struct tls_context *tls_ctx = tls_get_ctx(sk);
 654         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 655
 656         if (ctx->aead_send)
 657                 crypto_free_aead(ctx->aead_send);
 658
 659         tls_free_both_sg(sk);
 660
 661         kfree(ctx);
 662 }
 663
 664 int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx)
 665 {
 666         struct tls_crypto_info *crypto_info;
 667         struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
 668         struct tls_sw_context *sw_ctx;
 669         u16 nonce_size, tag_size, iv_size, rec_seq_size;
 670         char *iv, *rec_seq;
 671         int rc = 0;
 672
 673         if (!ctx) {
 674                 rc = -EINVAL;
 675                 goto out;
 676         }
 677
 678         if (ctx->priv_ctx) {
 679                 rc = -EEXIST;
 680                 goto out;
 681         }
 682
 683         sw_ctx = kzalloc(sizeof(*sw_ctx), GFP_KERNEL);
 684         if (!sw_ctx) {
 685                 rc = -ENOMEM;
 686                 goto out;
 687         }
 688
 689         ctx->priv_ctx = (struct tls_offload_context *)sw_ctx;
 690
 691         crypto_info = &ctx->crypto_send.info;
 692         switch (crypto_info->cipher_type) {
 693         case TLS_CIPHER_AES_GCM_128: {
 694                 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
 695                 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
 696                 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
 697                 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
 698                 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
 699                 rec_seq =
 700                  ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
 701                 gcm_128_info =
 702                         (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
 703                 break;
 704         }
 705         default:
 706                 rc = -EINVAL;
 707                 goto free_priv;
 708         }
 709
 710         ctx->prepend_size = TLS_HEADER_SIZE + nonce_size;
 711         ctx->tag_size = tag_size;
 712         ctx->overhead_size = ctx->prepend_size + ctx->tag_size;
 713         ctx->iv_size = iv_size;
 714         ctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE, GFP_KERNEL);
 715         if (!ctx->iv) {
 716                 rc = -ENOMEM;
 717                 goto free_priv;
 718         }
 719         memcpy(ctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
 720         memcpy(ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
 721         ctx->rec_seq_size = rec_seq_size;
 722         ctx->rec_seq = kmalloc(rec_seq_size, GFP_KERNEL);
 723         if (!ctx->rec_seq) {
 724                 rc = -ENOMEM;
 725                 goto free_iv;
 726         }
 727         memcpy(ctx->rec_seq, rec_seq, rec_seq_size);
 728
 729         sg_init_table(sw_ctx->sg_encrypted_data,
 730                       ARRAY_SIZE(sw_ctx->sg_encrypted_data));
 731         sg_init_table(sw_ctx->sg_plaintext_data,
 732                       ARRAY_SIZE(sw_ctx->sg_plaintext_data));
 733
 734         sg_init_table(sw_ctx->sg_aead_in, 2);
 735         sg_set_buf(&sw_ctx->sg_aead_in[0], sw_ctx->aad_space,
 736                    sizeof(sw_ctx->aad_space));
 737         sg_unmark_end(&sw_ctx->sg_aead_in[1]);
 738         sg_chain(sw_ctx->sg_aead_in, 2, sw_ctx->sg_plaintext_data);
 739         sg_init_table(sw_ctx->sg_aead_out, 2);
 740         sg_set_buf(&sw_ctx->sg_aead_out[0], sw_ctx->aad_space,
 741                    sizeof(sw_ctx->aad_space));
 742         sg_unmark_end(&sw_ctx->sg_aead_out[1]);
 743         sg_chain(sw_ctx->sg_aead_out, 2, sw_ctx->sg_encrypted_data);
 744
 745         if (!sw_ctx->aead_send) {
 746                 sw_ctx->aead_send = crypto_alloc_aead("gcm(aes)", 0, 0);
 747                 if (IS_ERR(sw_ctx->aead_send)) {
 748                         rc = PTR_ERR(sw_ctx->aead_send);
 749                         sw_ctx->aead_send = NULL;
 750                         goto free_rec_seq;
 751                 }
 752         }
 753
 754         ctx->push_pending_record = tls_sw_push_pending_record;
 755
 756         rc = crypto_aead_setkey(sw_ctx->aead_send, gcm_128_info->key,
 757                                 TLS_CIPHER_AES_GCM_128_KEY_SIZE);
 758         if (rc)
 759                 goto free_aead;
 760
 761         rc = crypto_aead_setauthsize(sw_ctx->aead_send, ctx->tag_size);
 762         if (!rc)
 763                 return 0;
 764
 765 free_aead:
 766         crypto_free_aead(sw_ctx->aead_send);
 767         sw_ctx->aead_send = NULL;
 768 free_rec_seq:
 769         kfree(ctx->rec_seq);
 770         ctx->rec_seq = NULL;
 771 free_iv:
 772         kfree(ctx->iv);
 773         ctx->iv = NULL;
 774 free_priv:
 775         kfree(ctx->priv_ctx);
 776         ctx->priv_ctx = NULL;
 777 out:
 778         return rc;
 779 }