GNU Linux-libre 4.14.266-gnu1
[releases.git] / net / tls / tls_sw.c
1 /*
2  * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3  * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
4  * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
5  * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
6  * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
7  *
8  * This software is available to you under a choice of one of two
9  * licenses.  You may choose to be licensed under the terms of the GNU
10  * General Public License (GPL) Version 2, available from the file
11  * COPYING in the main directory of this source tree, or the
12  * OpenIB.org BSD license below:
13  *
14  *     Redistribution and use in source and binary forms, with or
15  *     without modification, are permitted provided that the following
16  *     conditions are met:
17  *
18  *      - Redistributions of source code must retain the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer.
21  *
22  *      - Redistributions in binary form must reproduce the above
23  *        copyright notice, this list of conditions and the following
24  *        disclaimer in the documentation and/or other materials
25  *        provided with the distribution.
26  *
27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34  * SOFTWARE.
35  */
36
37 #include <linux/module.h>
38 #include <crypto/aead.h>
39
40 #include <net/tls.h>
41
42 static inline void tls_make_aad(int recv,
43                                 char *buf,
44                                 size_t size,
45                                 char *record_sequence,
46                                 int record_sequence_size,
47                                 unsigned char record_type)
48 {
49         memcpy(buf, record_sequence, record_sequence_size);
50
51         buf[8] = record_type;
52         buf[9] = TLS_1_2_VERSION_MAJOR;
53         buf[10] = TLS_1_2_VERSION_MINOR;
54         buf[11] = size >> 8;
55         buf[12] = size & 0xFF;
56 }
57
58 static void trim_sg(struct sock *sk, struct scatterlist *sg,
59                     int *sg_num_elem, unsigned int *sg_size, int target_size)
60 {
61         int i = *sg_num_elem - 1;
62         int trim = *sg_size - target_size;
63
64         if (trim <= 0) {
65                 WARN_ON(trim < 0);
66                 return;
67         }
68
69         *sg_size = target_size;
70         while (trim >= sg[i].length) {
71                 trim -= sg[i].length;
72                 sk_mem_uncharge(sk, sg[i].length);
73                 put_page(sg_page(&sg[i]));
74                 i--;
75
76                 if (i < 0)
77                         goto out;
78         }
79
80         sg[i].length -= trim;
81         sk_mem_uncharge(sk, trim);
82
83 out:
84         *sg_num_elem = i + 1;
85 }
86
87 static void trim_both_sgl(struct sock *sk, int target_size)
88 {
89         struct tls_context *tls_ctx = tls_get_ctx(sk);
90         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
91
92         trim_sg(sk, ctx->sg_plaintext_data,
93                 &ctx->sg_plaintext_num_elem,
94                 &ctx->sg_plaintext_size,
95                 target_size);
96
97         if (target_size > 0)
98                 target_size += tls_ctx->overhead_size;
99
100         trim_sg(sk, ctx->sg_encrypted_data,
101                 &ctx->sg_encrypted_num_elem,
102                 &ctx->sg_encrypted_size,
103                 target_size);
104 }
105
106 static int alloc_sg(struct sock *sk, int len, struct scatterlist *sg,
107                     int *sg_num_elem, unsigned int *sg_size,
108                     int first_coalesce)
109 {
110         struct page_frag *pfrag;
111         unsigned int size = *sg_size;
112         int num_elem = *sg_num_elem, use = 0, rc = 0;
113         struct scatterlist *sge;
114         unsigned int orig_offset;
115
116         len -= size;
117         pfrag = sk_page_frag(sk);
118
119         while (len > 0) {
120                 if (!sk_page_frag_refill(sk, pfrag)) {
121                         rc = -ENOMEM;
122                         goto out;
123                 }
124
125                 use = min_t(int, len, pfrag->size - pfrag->offset);
126
127                 if (!sk_wmem_schedule(sk, use)) {
128                         rc = -ENOMEM;
129                         goto out;
130                 }
131
132                 sk_mem_charge(sk, use);
133                 size += use;
134                 orig_offset = pfrag->offset;
135                 pfrag->offset += use;
136
137                 sge = sg + num_elem - 1;
138
139                 if (num_elem > first_coalesce && sg_page(sge) == pfrag->page &&
140                     sge->offset + sge->length == orig_offset) {
141                         sge->length += use;
142                 } else {
143                         sge++;
144                         sg_unmark_end(sge);
145                         sg_set_page(sge, pfrag->page, use, orig_offset);
146                         get_page(pfrag->page);
147                         ++num_elem;
148                         if (num_elem == MAX_SKB_FRAGS) {
149                                 rc = -ENOSPC;
150                                 break;
151                         }
152                 }
153
154                 len -= use;
155         }
156         goto out;
157
158 out:
159         *sg_size = size;
160         *sg_num_elem = num_elem;
161         return rc;
162 }
163
164 static int alloc_encrypted_sg(struct sock *sk, int len)
165 {
166         struct tls_context *tls_ctx = tls_get_ctx(sk);
167         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
168         int rc = 0;
169
170         rc = alloc_sg(sk, len, ctx->sg_encrypted_data,
171                       &ctx->sg_encrypted_num_elem, &ctx->sg_encrypted_size, 0);
172
173         if (rc == -ENOSPC)
174                 ctx->sg_encrypted_num_elem = ARRAY_SIZE(ctx->sg_encrypted_data);
175
176         return rc;
177 }
178
179 static int alloc_plaintext_sg(struct sock *sk, int len)
180 {
181         struct tls_context *tls_ctx = tls_get_ctx(sk);
182         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
183         int rc = 0;
184
185         rc = alloc_sg(sk, len, ctx->sg_plaintext_data,
186                       &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size,
187                       tls_ctx->pending_open_record_frags);
188
189         if (rc == -ENOSPC)
190                 ctx->sg_plaintext_num_elem = ARRAY_SIZE(ctx->sg_plaintext_data);
191
192         return rc;
193 }
194
195 static void free_sg(struct sock *sk, struct scatterlist *sg,
196                     int *sg_num_elem, unsigned int *sg_size)
197 {
198         int i, n = *sg_num_elem;
199
200         for (i = 0; i < n; ++i) {
201                 sk_mem_uncharge(sk, sg[i].length);
202                 put_page(sg_page(&sg[i]));
203         }
204         *sg_num_elem = 0;
205         *sg_size = 0;
206 }
207
208 static void tls_free_both_sg(struct sock *sk)
209 {
210         struct tls_context *tls_ctx = tls_get_ctx(sk);
211         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
212
213         free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem,
214                 &ctx->sg_encrypted_size);
215
216         free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
217                 &ctx->sg_plaintext_size);
218 }
219
220 static int tls_do_encryption(struct tls_context *tls_ctx,
221                              struct tls_sw_context *ctx,
222                              struct aead_request *aead_req,
223                              size_t data_len)
224 {
225         int rc;
226
227         ctx->sg_encrypted_data[0].offset += tls_ctx->prepend_size;
228         ctx->sg_encrypted_data[0].length -= tls_ctx->prepend_size;
229
230         aead_request_set_tfm(aead_req, ctx->aead_send);
231         aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
232         aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out,
233                                data_len, tls_ctx->iv);
234         rc = crypto_aead_encrypt(aead_req);
235
236         ctx->sg_encrypted_data[0].offset -= tls_ctx->prepend_size;
237         ctx->sg_encrypted_data[0].length += tls_ctx->prepend_size;
238
239         return rc;
240 }
241
242 static int tls_push_record(struct sock *sk, int flags,
243                            unsigned char record_type)
244 {
245         struct tls_context *tls_ctx = tls_get_ctx(sk);
246         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
247         struct aead_request *req;
248         int rc;
249
250         req = kzalloc(sizeof(struct aead_request) +
251                       crypto_aead_reqsize(ctx->aead_send), sk->sk_allocation);
252         if (!req)
253                 return -ENOMEM;
254
255         sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1);
256         sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1);
257
258         tls_make_aad(0, ctx->aad_space, ctx->sg_plaintext_size,
259                      tls_ctx->rec_seq, tls_ctx->rec_seq_size,
260                      record_type);
261
262         tls_fill_prepend(tls_ctx,
263                          page_address(sg_page(&ctx->sg_encrypted_data[0])) +
264                          ctx->sg_encrypted_data[0].offset,
265                          ctx->sg_plaintext_size, record_type);
266
267         tls_ctx->pending_open_record_frags = 0;
268         set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags);
269
270         rc = tls_do_encryption(tls_ctx, ctx, req, ctx->sg_plaintext_size);
271         if (rc < 0) {
272                 /* If we are called from write_space and
273                  * we fail, we need to set this SOCK_NOSPACE
274                  * to trigger another write_space in the future.
275                  */
276                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
277                 goto out_req;
278         }
279
280         free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
281                 &ctx->sg_plaintext_size);
282
283         ctx->sg_encrypted_num_elem = 0;
284         ctx->sg_encrypted_size = 0;
285
286         /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */
287         rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags);
288         if (rc < 0 && rc != -EAGAIN)
289                 tls_err_abort(sk);
290
291         tls_advance_record_sn(sk, tls_ctx);
292 out_req:
293         kfree(req);
294         return rc;
295 }
296
297 static int tls_sw_push_pending_record(struct sock *sk, int flags)
298 {
299         return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA);
300 }
301
302 static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
303                               int length)
304 {
305         struct tls_context *tls_ctx = tls_get_ctx(sk);
306         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
307         struct page *pages[MAX_SKB_FRAGS];
308
309         size_t offset;
310         ssize_t copied, use;
311         int i = 0;
312         unsigned int size = ctx->sg_plaintext_size;
313         int num_elem = ctx->sg_plaintext_num_elem;
314         int rc = 0;
315         int maxpages;
316
317         while (length > 0) {
318                 i = 0;
319                 maxpages = ARRAY_SIZE(ctx->sg_plaintext_data) - num_elem;
320                 if (maxpages == 0) {
321                         rc = -EFAULT;
322                         goto out;
323                 }
324                 copied = iov_iter_get_pages(from, pages,
325                                             length,
326                                             maxpages, &offset);
327                 if (copied <= 0) {
328                         rc = -EFAULT;
329                         goto out;
330                 }
331
332                 iov_iter_advance(from, copied);
333
334                 length -= copied;
335                 size += copied;
336                 while (copied) {
337                         use = min_t(int, copied, PAGE_SIZE - offset);
338
339                         sg_set_page(&ctx->sg_plaintext_data[num_elem],
340                                     pages[i], use, offset);
341                         sg_unmark_end(&ctx->sg_plaintext_data[num_elem]);
342                         sk_mem_charge(sk, use);
343
344                         offset = 0;
345                         copied -= use;
346
347                         ++i;
348                         ++num_elem;
349                 }
350         }
351
352 out:
353         ctx->sg_plaintext_size = size;
354         ctx->sg_plaintext_num_elem = num_elem;
355         return rc;
356 }
357
358 static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
359                              int bytes)
360 {
361         struct tls_context *tls_ctx = tls_get_ctx(sk);
362         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
363         struct scatterlist *sg = ctx->sg_plaintext_data;
364         int copy, i, rc = 0;
365
366         for (i = tls_ctx->pending_open_record_frags;
367              i < ctx->sg_plaintext_num_elem; ++i) {
368                 copy = sg[i].length;
369                 if (copy_from_iter(
370                                 page_address(sg_page(&sg[i])) + sg[i].offset,
371                                 copy, from) != copy) {
372                         rc = -EFAULT;
373                         goto out;
374                 }
375                 bytes -= copy;
376
377                 ++tls_ctx->pending_open_record_frags;
378
379                 if (!bytes)
380                         break;
381         }
382
383 out:
384         return rc;
385 }
386
387 int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
388 {
389         struct tls_context *tls_ctx = tls_get_ctx(sk);
390         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
391         int ret;
392         int required_size;
393         long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
394         bool eor = !(msg->msg_flags & MSG_MORE);
395         size_t try_to_copy, copied = 0;
396         unsigned char record_type = TLS_RECORD_TYPE_DATA;
397         int record_room;
398         bool full_record;
399         int orig_size;
400
401         if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
402                 return -ENOTSUPP;
403
404         lock_sock(sk);
405
406         ret = tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo);
407         if (ret)
408                 goto send_end;
409
410         if (unlikely(msg->msg_controllen)) {
411                 ret = tls_proccess_cmsg(sk, msg, &record_type);
412                 if (ret)
413                         goto send_end;
414         }
415
416         while (msg_data_left(msg)) {
417                 if (sk->sk_err) {
418                         ret = -sk->sk_err;
419                         goto send_end;
420                 }
421
422                 orig_size = ctx->sg_plaintext_size;
423                 full_record = false;
424                 try_to_copy = msg_data_left(msg);
425                 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
426                 if (try_to_copy >= record_room) {
427                         try_to_copy = record_room;
428                         full_record = true;
429                 }
430
431                 required_size = ctx->sg_plaintext_size + try_to_copy +
432                                 tls_ctx->overhead_size;
433
434                 if (!sk_stream_memory_free(sk))
435                         goto wait_for_sndbuf;
436 alloc_encrypted:
437                 ret = alloc_encrypted_sg(sk, required_size);
438                 if (ret) {
439                         if (ret != -ENOSPC)
440                                 goto wait_for_memory;
441
442                         /* Adjust try_to_copy according to the amount that was
443                          * actually allocated. The difference is due
444                          * to max sg elements limit
445                          */
446                         try_to_copy -= required_size - ctx->sg_encrypted_size;
447                         full_record = true;
448                 }
449
450                 if (full_record || eor) {
451                         ret = zerocopy_from_iter(sk, &msg->msg_iter,
452                                                  try_to_copy);
453                         if (ret)
454                                 goto fallback_to_reg_send;
455
456                         copied += try_to_copy;
457                         ret = tls_push_record(sk, msg->msg_flags, record_type);
458                         if (!ret)
459                                 continue;
460                         if (ret < 0)
461                                 goto send_end;
462
463                         copied -= try_to_copy;
464 fallback_to_reg_send:
465                         iov_iter_revert(&msg->msg_iter,
466                                         ctx->sg_plaintext_size - orig_size);
467                         trim_sg(sk, ctx->sg_plaintext_data,
468                                 &ctx->sg_plaintext_num_elem,
469                                 &ctx->sg_plaintext_size,
470                                 orig_size);
471                 }
472
473                 required_size = ctx->sg_plaintext_size + try_to_copy;
474 alloc_plaintext:
475                 ret = alloc_plaintext_sg(sk, required_size);
476                 if (ret) {
477                         if (ret != -ENOSPC)
478                                 goto wait_for_memory;
479
480                         /* Adjust try_to_copy according to the amount that was
481                          * actually allocated. The difference is due
482                          * to max sg elements limit
483                          */
484                         try_to_copy -= required_size - ctx->sg_plaintext_size;
485                         full_record = true;
486
487                         trim_sg(sk, ctx->sg_encrypted_data,
488                                 &ctx->sg_encrypted_num_elem,
489                                 &ctx->sg_encrypted_size,
490                                 ctx->sg_plaintext_size +
491                                 tls_ctx->overhead_size);
492                 }
493
494                 ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy);
495                 if (ret)
496                         goto trim_sgl;
497
498                 copied += try_to_copy;
499                 if (full_record || eor) {
500 push_record:
501                         ret = tls_push_record(sk, msg->msg_flags, record_type);
502                         if (ret) {
503                                 if (ret == -ENOMEM)
504                                         goto wait_for_memory;
505
506                                 goto send_end;
507                         }
508                 }
509
510                 continue;
511
512 wait_for_sndbuf:
513                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
514 wait_for_memory:
515                 ret = sk_stream_wait_memory(sk, &timeo);
516                 if (ret) {
517 trim_sgl:
518                         trim_both_sgl(sk, orig_size);
519                         goto send_end;
520                 }
521
522                 if (tls_is_pending_closed_record(tls_ctx))
523                         goto push_record;
524
525                 if (ctx->sg_encrypted_size < required_size)
526                         goto alloc_encrypted;
527
528                 goto alloc_plaintext;
529         }
530
531 send_end:
532         ret = sk_stream_error(sk, msg->msg_flags, ret);
533
534         release_sock(sk);
535         return copied ? copied : ret;
536 }
537
538 int tls_sw_sendpage(struct sock *sk, struct page *page,
539                     int offset, size_t size, int flags)
540 {
541         struct tls_context *tls_ctx = tls_get_ctx(sk);
542         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
543         int ret;
544         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
545         bool eor;
546         size_t orig_size = size;
547         unsigned char record_type = TLS_RECORD_TYPE_DATA;
548         struct scatterlist *sg;
549         bool full_record;
550         int record_room;
551
552         if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
553                       MSG_SENDPAGE_NOTLAST))
554                 return -ENOTSUPP;
555
556         /* No MSG_EOR from splice, only look at MSG_MORE */
557         eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST));
558
559         lock_sock(sk);
560
561         sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
562
563         ret = tls_complete_pending_work(sk, tls_ctx, flags, &timeo);
564         if (ret)
565                 goto sendpage_end;
566
567         /* Call the sk_stream functions to manage the sndbuf mem. */
568         while (size > 0) {
569                 size_t copy, required_size;
570
571                 if (sk->sk_err) {
572                         ret = -sk->sk_err;
573                         goto sendpage_end;
574                 }
575
576                 full_record = false;
577                 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
578                 copy = size;
579                 if (copy >= record_room) {
580                         copy = record_room;
581                         full_record = true;
582                 }
583                 required_size = ctx->sg_plaintext_size + copy +
584                               tls_ctx->overhead_size;
585
586                 if (!sk_stream_memory_free(sk))
587                         goto wait_for_sndbuf;
588 alloc_payload:
589                 ret = alloc_encrypted_sg(sk, required_size);
590                 if (ret) {
591                         if (ret != -ENOSPC)
592                                 goto wait_for_memory;
593
594                         /* Adjust copy according to the amount that was
595                          * actually allocated. The difference is due
596                          * to max sg elements limit
597                          */
598                         copy -= required_size - ctx->sg_plaintext_size;
599                         full_record = true;
600                 }
601
602                 get_page(page);
603                 sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem;
604                 sg_set_page(sg, page, copy, offset);
605                 ctx->sg_plaintext_num_elem++;
606
607                 sk_mem_charge(sk, copy);
608                 offset += copy;
609                 size -= copy;
610                 ctx->sg_plaintext_size += copy;
611                 tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem;
612
613                 if (full_record || eor ||
614                     ctx->sg_plaintext_num_elem ==
615                     ARRAY_SIZE(ctx->sg_plaintext_data)) {
616 push_record:
617                         ret = tls_push_record(sk, flags, record_type);
618                         if (ret) {
619                                 if (ret == -ENOMEM)
620                                         goto wait_for_memory;
621
622                                 goto sendpage_end;
623                         }
624                 }
625                 continue;
626 wait_for_sndbuf:
627                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
628 wait_for_memory:
629                 ret = sk_stream_wait_memory(sk, &timeo);
630                 if (ret) {
631                         trim_both_sgl(sk, ctx->sg_plaintext_size);
632                         goto sendpage_end;
633                 }
634
635                 if (tls_is_pending_closed_record(tls_ctx))
636                         goto push_record;
637
638                 goto alloc_payload;
639         }
640
641 sendpage_end:
642         if (orig_size > size)
643                 ret = orig_size - size;
644         else
645                 ret = sk_stream_error(sk, flags, ret);
646
647         release_sock(sk);
648         return ret;
649 }
650
651 void tls_sw_free_tx_resources(struct sock *sk)
652 {
653         struct tls_context *tls_ctx = tls_get_ctx(sk);
654         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
655
656         if (ctx->aead_send)
657                 crypto_free_aead(ctx->aead_send);
658
659         tls_free_both_sg(sk);
660
661         kfree(ctx);
662 }
663
664 int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx)
665 {
666         struct tls_crypto_info *crypto_info;
667         struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
668         struct tls_sw_context *sw_ctx;
669         u16 nonce_size, tag_size, iv_size, rec_seq_size;
670         char *iv, *rec_seq;
671         int rc = 0;
672
673         if (!ctx) {
674                 rc = -EINVAL;
675                 goto out;
676         }
677
678         if (ctx->priv_ctx) {
679                 rc = -EEXIST;
680                 goto out;
681         }
682
683         sw_ctx = kzalloc(sizeof(*sw_ctx), GFP_KERNEL);
684         if (!sw_ctx) {
685                 rc = -ENOMEM;
686                 goto out;
687         }
688
689         ctx->priv_ctx = (struct tls_offload_context *)sw_ctx;
690
691         crypto_info = &ctx->crypto_send.info;
692         switch (crypto_info->cipher_type) {
693         case TLS_CIPHER_AES_GCM_128: {
694                 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
695                 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
696                 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
697                 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
698                 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
699                 rec_seq =
700                  ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
701                 gcm_128_info =
702                         (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
703                 break;
704         }
705         default:
706                 rc = -EINVAL;
707                 goto free_priv;
708         }
709
710         ctx->prepend_size = TLS_HEADER_SIZE + nonce_size;
711         ctx->tag_size = tag_size;
712         ctx->overhead_size = ctx->prepend_size + ctx->tag_size;
713         ctx->iv_size = iv_size;
714         ctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE, GFP_KERNEL);
715         if (!ctx->iv) {
716                 rc = -ENOMEM;
717                 goto free_priv;
718         }
719         memcpy(ctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
720         memcpy(ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
721         ctx->rec_seq_size = rec_seq_size;
722         ctx->rec_seq = kmalloc(rec_seq_size, GFP_KERNEL);
723         if (!ctx->rec_seq) {
724                 rc = -ENOMEM;
725                 goto free_iv;
726         }
727         memcpy(ctx->rec_seq, rec_seq, rec_seq_size);
728
729         sg_init_table(sw_ctx->sg_encrypted_data,
730                       ARRAY_SIZE(sw_ctx->sg_encrypted_data));
731         sg_init_table(sw_ctx->sg_plaintext_data,
732                       ARRAY_SIZE(sw_ctx->sg_plaintext_data));
733
734         sg_init_table(sw_ctx->sg_aead_in, 2);
735         sg_set_buf(&sw_ctx->sg_aead_in[0], sw_ctx->aad_space,
736                    sizeof(sw_ctx->aad_space));
737         sg_unmark_end(&sw_ctx->sg_aead_in[1]);
738         sg_chain(sw_ctx->sg_aead_in, 2, sw_ctx->sg_plaintext_data);
739         sg_init_table(sw_ctx->sg_aead_out, 2);
740         sg_set_buf(&sw_ctx->sg_aead_out[0], sw_ctx->aad_space,
741                    sizeof(sw_ctx->aad_space));
742         sg_unmark_end(&sw_ctx->sg_aead_out[1]);
743         sg_chain(sw_ctx->sg_aead_out, 2, sw_ctx->sg_encrypted_data);
744
745         if (!sw_ctx->aead_send) {
746                 sw_ctx->aead_send = crypto_alloc_aead("gcm(aes)", 0, 0);
747                 if (IS_ERR(sw_ctx->aead_send)) {
748                         rc = PTR_ERR(sw_ctx->aead_send);
749                         sw_ctx->aead_send = NULL;
750                         goto free_rec_seq;
751                 }
752         }
753
754         ctx->push_pending_record = tls_sw_push_pending_record;
755
756         rc = crypto_aead_setkey(sw_ctx->aead_send, gcm_128_info->key,
757                                 TLS_CIPHER_AES_GCM_128_KEY_SIZE);
758         if (rc)
759                 goto free_aead;
760
761         rc = crypto_aead_setauthsize(sw_ctx->aead_send, ctx->tag_size);
762         if (!rc)
763                 return 0;
764
765 free_aead:
766         crypto_free_aead(sw_ctx->aead_send);
767         sw_ctx->aead_send = NULL;
768 free_rec_seq:
769         kfree(ctx->rec_seq);
770         ctx->rec_seq = NULL;
771 free_iv:
772         kfree(ctx->iv);
773         ctx->iv = NULL;
774 free_priv:
775         kfree(ctx->priv_ctx);
776         ctx->priv_ctx = NULL;
777 out:
778         return rc;
779 }