blob: 3793885f928dd33e8fa25b6f9fa2ee5aa1ef3ced [file] [log] [blame]
Thomas Gleixner64d85cc2019-05-29 07:18:13 -07001// SPDX-License-Identifier: GPL-2.0-only
Aditya Srivastava10cb8232021-03-21 18:00:07 +05302/*
Kent Yodera00bd6e2012-05-14 11:05:23 +00003 * AES CCM routines supporting the Power 7+ Nest Accelerators driver
4 *
5 * Copyright (C) 2012 International Business Machines Inc.
6 *
Kent Yodera00bd6e2012-05-14 11:05:23 +00007 * Author: Kent Yoder <yoder1@us.ibm.com>
8 */
9
10#include <crypto/internal/aead.h>
11#include <crypto/aes.h>
12#include <crypto/algapi.h>
13#include <crypto/scatterwalk.h>
14#include <linux/module.h>
15#include <linux/types.h>
16#include <linux/crypto.h>
17#include <asm/vio.h>
18
19#include "nx_csbcpb.h"
20#include "nx.h"
21
22
23static int ccm_aes_nx_set_key(struct crypto_aead *tfm,
24 const u8 *in_key,
25 unsigned int key_len)
26{
27 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
28 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
29 struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
30
31 nx_ctx_init(nx_ctx, HCOP_FC_AES);
32
33 switch (key_len) {
34 case AES_KEYSIZE_128:
35 NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
36 NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128);
37 nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
38 break;
39 default:
40 return -EINVAL;
41 }
42
43 csbcpb->cpb.hdr.mode = NX_MODE_AES_CCM;
44 memcpy(csbcpb->cpb.aes_ccm.key, in_key, key_len);
45
46 csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_CCA;
47 memcpy(csbcpb_aead->cpb.aes_cca.key, in_key, key_len);
48
49 return 0;
50
51}
52
53static int ccm4309_aes_nx_set_key(struct crypto_aead *tfm,
54 const u8 *in_key,
55 unsigned int key_len)
56{
57 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
58
59 if (key_len < 3)
60 return -EINVAL;
61
62 key_len -= 3;
63
64 memcpy(nx_ctx->priv.ccm.nonce, in_key + key_len, 3);
65
66 return ccm_aes_nx_set_key(tfm, in_key, key_len);
67}
68
69static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm,
70 unsigned int authsize)
71{
72 switch (authsize) {
73 case 4:
74 case 6:
75 case 8:
76 case 10:
77 case 12:
78 case 14:
79 case 16:
80 break;
81 default:
82 return -EINVAL;
83 }
84
Kent Yodera00bd6e2012-05-14 11:05:23 +000085 return 0;
86}
87
88static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm,
89 unsigned int authsize)
90{
91 switch (authsize) {
92 case 8:
93 case 12:
94 case 16:
95 break;
96 default:
97 return -EINVAL;
98 }
99
Kent Yodera00bd6e2012-05-14 11:05:23 +0000100 return 0;
101}
102
103/* taken from crypto/ccm.c */
104static int set_msg_len(u8 *block, unsigned int msglen, int csize)
105{
106 __be32 data;
107
108 memset(block, 0, csize);
109 block += csize;
110
111 if (csize >= 4)
112 csize = 4;
113 else if (msglen > (unsigned int)(1 << (8 * csize)))
114 return -EOVERFLOW;
115
116 data = cpu_to_be32(msglen);
117 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
118
119 return 0;
120}
121
122/* taken from crypto/ccm.c */
123static inline int crypto_ccm_check_iv(const u8 *iv)
124{
125 /* 2 <= L <= 8, so 1 <= L' <= 7. */
126 if (1 > iv[0] || iv[0] > 7)
127 return -EINVAL;
128
129 return 0;
130}
131
132/* based on code from crypto/ccm.c */
133static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize,
134 unsigned int cryptlen, u8 *b0)
135{
136 unsigned int l, lp, m = authsize;
137 int rc;
138
139 memcpy(b0, iv, 16);
140
141 lp = b0[0];
142 l = lp + 1;
143
144 /* set m, bits 3-5 */
145 *b0 |= (8 * ((m - 2) / 2));
146
147 /* set adata, bit 6, if associated data is used */
148 if (assoclen)
149 *b0 |= 64;
150
151 rc = set_msg_len(b0 + 16 - l, cryptlen, l);
152
153 return rc;
154}
155
156static int generate_pat(u8 *iv,
157 struct aead_request *req,
158 struct nx_crypto_ctx *nx_ctx,
159 unsigned int authsize,
160 unsigned int nbytes,
Herbert Xucc815652015-07-14 16:53:21 +0800161 unsigned int assoclen,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000162 u8 *out)
163{
164 struct nx_sg *nx_insg = nx_ctx->in_sg;
165 struct nx_sg *nx_outsg = nx_ctx->out_sg;
166 unsigned int iauth_len = 0;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000167 u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL;
168 int rc;
Leonidas S. Barbosa9247f0b2014-10-28 15:44:30 -0200169 unsigned int max_sg_len;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000170
171 /* zero the ctr value */
172 memset(iv + 15 - iv[0], 0, iv[0] + 1);
173
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300174 /* page 78 of nx_wb.pdf has,
175 * Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes
176 * in length. If a full message is used, the AES CCA implementation
177 * restricts the maximum AAD length to 2^32 -1 bytes.
178 * If partial messages are used, the implementation supports
179 * 2^64 -1 bytes maximum AAD length.
180 *
181 * However, in the cryptoapi's aead_request structure,
182 * assoclen is an unsigned int, thus it cannot hold a length
183 * value greater than 2^32 - 1.
184 * Thus the AAD is further constrained by this and is never
185 * greater than 2^32.
186 */
187
Herbert Xucc815652015-07-14 16:53:21 +0800188 if (!assoclen) {
Kent Yodera00bd6e2012-05-14 11:05:23 +0000189 b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
Herbert Xucc815652015-07-14 16:53:21 +0800190 } else if (assoclen <= 14) {
Kent Yodera00bd6e2012-05-14 11:05:23 +0000191 /* if associated data is 14 bytes or less, we do 1 GCM
192 * operation on 2 AES blocks, B0 (stored in the csbcpb) and B1,
193 * which is fed in through the source buffers here */
194 b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
195 b1 = nx_ctx->priv.ccm.iauth_tag;
Herbert Xucc815652015-07-14 16:53:21 +0800196 iauth_len = assoclen;
197 } else if (assoclen <= 65280) {
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300198 /* if associated data is less than (2^16 - 2^8), we construct
199 * B1 differently and feed in the associated data to a CCA
200 * operation */
201 b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
202 b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
203 iauth_len = 14;
204 } else {
205 b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
206 b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
207 iauth_len = 10;
208 }
Kent Yodera00bd6e2012-05-14 11:05:23 +0000209
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300210 /* generate B0 */
Herbert Xucc815652015-07-14 16:53:21 +0800211 rc = generate_b0(iv, assoclen, authsize, nbytes, b0);
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300212 if (rc)
213 return rc;
214
215 /* generate B1:
216 * add control info for associated data
217 * RFC 3610 and NIST Special Publication 800-38C
218 */
219 if (b1) {
220 memset(b1, 0, 16);
Herbert Xucc815652015-07-14 16:53:21 +0800221 if (assoclen <= 65280) {
222 *(u16 *)b1 = assoclen;
223 scatterwalk_map_and_copy(b1 + 2, req->src, 0,
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300224 iauth_len, SCATTERWALK_FROM_SG);
225 } else {
226 *(u16 *)b1 = (u16)(0xfffe);
Herbert Xucc815652015-07-14 16:53:21 +0800227 *(u32 *)&b1[2] = assoclen;
228 scatterwalk_map_and_copy(b1 + 6, req->src, 0,
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300229 iauth_len, SCATTERWALK_FROM_SG);
230 }
231 }
232
233 /* now copy any remaining AAD to scatterlist and call nx... */
Herbert Xucc815652015-07-14 16:53:21 +0800234 if (!assoclen) {
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300235 return rc;
Herbert Xucc815652015-07-14 16:53:21 +0800236 } else if (assoclen <= 14) {
Leonidas S. Barbosa9247f0b2014-10-28 15:44:30 -0200237 unsigned int len = 16;
238
239 nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen);
240
241 if (len != 16)
242 return -EINVAL;
243
244 nx_outsg = nx_build_sg_list(nx_outsg, tmp, &len,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000245 nx_ctx->ap->sglen);
246
Leonidas S. Barbosa9247f0b2014-10-28 15:44:30 -0200247 if (len != 16)
248 return -EINVAL;
249
Kent Yodera00bd6e2012-05-14 11:05:23 +0000250 /* inlen should be negative, indicating to phyp that its a
251 * pointer to an sg list */
252 nx_ctx->op.inlen = (nx_ctx->in_sg - nx_insg) *
253 sizeof(struct nx_sg);
254 nx_ctx->op.outlen = (nx_ctx->out_sg - nx_outsg) *
255 sizeof(struct nx_sg);
256
257 NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT;
258 NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE;
259
Kent Yodera00bd6e2012-05-14 11:05:23 +0000260 result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000261
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300262 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000263 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
264 if (rc)
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300265 return rc;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000266
267 atomic_inc(&(nx_ctx->stats->aes_ops));
Herbert Xucc815652015-07-14 16:53:21 +0800268 atomic64_add(assoclen, &nx_ctx->stats->aes_bytes);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000269
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300270 } else {
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300271 unsigned int processed = 0, to_process;
272
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300273 processed += iauth_len;
274
Leonidas S. Barbosa9247f0b2014-10-28 15:44:30 -0200275 /* page_limit: number of sg entries that fit on one page */
276 max_sg_len = min_t(u64, nx_ctx->ap->sglen,
277 nx_driver.of.max_sg_len/sizeof(struct nx_sg));
278 max_sg_len = min_t(u64, max_sg_len,
279 nx_ctx->ap->databytelen/NX_PAGE_SIZE);
280
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300281 do {
Herbert Xucc815652015-07-14 16:53:21 +0800282 to_process = min_t(u32, assoclen - processed,
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300283 nx_ctx->ap->databytelen);
Leonidas S. Barbosa9247f0b2014-10-28 15:44:30 -0200284
285 nx_insg = nx_walk_and_build(nx_ctx->in_sg,
286 nx_ctx->ap->sglen,
Herbert Xucc815652015-07-14 16:53:21 +0800287 req->src, processed,
Leonidas S. Barbosa9247f0b2014-10-28 15:44:30 -0200288 &to_process);
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300289
Herbert Xucc815652015-07-14 16:53:21 +0800290 if ((to_process + processed) < assoclen) {
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300291 NX_CPB_FDM(nx_ctx->csbcpb_aead) |=
292 NX_FDM_INTERMEDIATE;
293 } else {
294 NX_CPB_FDM(nx_ctx->csbcpb_aead) &=
295 ~NX_FDM_INTERMEDIATE;
296 }
297
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300298
299 nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) *
300 sizeof(struct nx_sg);
301
302 result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
303
304 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
305 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
306 if (rc)
307 return rc;
308
309 memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0,
310 nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0,
311 AES_BLOCK_SIZE);
312
313 NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION;
314
315 atomic_inc(&(nx_ctx->stats->aes_ops));
Herbert Xucc815652015-07-14 16:53:21 +0800316 atomic64_add(assoclen, &nx_ctx->stats->aes_bytes);
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300317
318 processed += to_process;
Herbert Xucc815652015-07-14 16:53:21 +0800319 } while (processed < assoclen);
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300320
321 result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000322 }
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300323
324 memcpy(out, result, AES_BLOCK_SIZE);
325
Kent Yodera00bd6e2012-05-14 11:05:23 +0000326 return rc;
327}
328
329static int ccm_nx_decrypt(struct aead_request *req,
Eric Biggers7740bd52019-10-12 21:39:15 -0700330 u8 *iv,
Herbert Xucc815652015-07-14 16:53:21 +0800331 unsigned int assoclen)
Kent Yodera00bd6e2012-05-14 11:05:23 +0000332{
333 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
334 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
335 unsigned int nbytes = req->cryptlen;
336 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
337 struct nx_ccm_priv *priv = &nx_ctx->priv.ccm;
Marcelo Cerric8491632013-08-12 18:49:37 -0300338 unsigned long irq_flags;
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300339 unsigned int processed = 0, to_process;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000340 int rc = -1;
341
Marcelo Cerric8491632013-08-12 18:49:37 -0300342 spin_lock_irqsave(&nx_ctx->lock, irq_flags);
343
Kent Yodera00bd6e2012-05-14 11:05:23 +0000344 nbytes -= authsize;
345
346 /* copy out the auth tag to compare with later */
347 scatterwalk_map_and_copy(priv->oauth_tag,
Herbert Xucc815652015-07-14 16:53:21 +0800348 req->src, nbytes + req->assoclen, authsize,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000349 SCATTERWALK_FROM_SG);
350
Eric Biggers7740bd52019-10-12 21:39:15 -0700351 rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000352 csbcpb->cpb.aes_ccm.in_pat_or_b0);
353 if (rc)
354 goto out;
355
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300356 do {
Kent Yodera00bd6e2012-05-14 11:05:23 +0000357
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300358 /* to_process: the AES_BLOCK_SIZE data chunk to process in this
359 * update. This value is bound by sg list limits.
360 */
Leonidas S. Barbosa9247f0b2014-10-28 15:44:30 -0200361 to_process = nbytes - processed;
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300362
363 if ((to_process + processed) < nbytes)
364 NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
365 else
366 NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
367
368 NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
369
Eric Biggers7740bd52019-10-12 21:39:15 -0700370 rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src,
Herbert Xucc815652015-07-14 16:53:21 +0800371 &to_process, processed + req->assoclen,
372 csbcpb->cpb.aes_ccm.iv_or_ctr);
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300373 if (rc)
374 goto out;
375
376 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000377 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300378 if (rc)
379 goto out;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000380
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300381 /* for partial completion, copy following for next
382 * entry into loop...
383 */
Eric Biggers7740bd52019-10-12 21:39:15 -0700384 memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300385 memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
386 csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
387 memcpy(csbcpb->cpb.aes_ccm.in_s0,
388 csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
389
390 NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
391
392 /* update stats */
393 atomic_inc(&(nx_ctx->stats->aes_ops));
Herbert Xub20d9a72021-06-17 16:00:12 +0800394 atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300395 &(nx_ctx->stats->aes_bytes));
396
397 processed += to_process;
398 } while (processed < nbytes);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000399
David Gstircb8affb2015-11-15 17:14:41 +0100400 rc = crypto_memneq(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000401 authsize) ? -EBADMSG : 0;
402out:
Marcelo Cerric8491632013-08-12 18:49:37 -0300403 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000404 return rc;
405}
406
407static int ccm_nx_encrypt(struct aead_request *req,
Eric Biggers7740bd52019-10-12 21:39:15 -0700408 u8 *iv,
Herbert Xucc815652015-07-14 16:53:21 +0800409 unsigned int assoclen)
Kent Yodera00bd6e2012-05-14 11:05:23 +0000410{
411 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
412 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
413 unsigned int nbytes = req->cryptlen;
414 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
Marcelo Cerric8491632013-08-12 18:49:37 -0300415 unsigned long irq_flags;
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300416 unsigned int processed = 0, to_process;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000417 int rc = -1;
418
Marcelo Cerric8491632013-08-12 18:49:37 -0300419 spin_lock_irqsave(&nx_ctx->lock, irq_flags);
420
Eric Biggers7740bd52019-10-12 21:39:15 -0700421 rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000422 csbcpb->cpb.aes_ccm.in_pat_or_b0);
423 if (rc)
424 goto out;
425
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300426 do {
427 /* to process: the AES_BLOCK_SIZE data chunk to process in this
428 * update. This value is bound by sg list limits.
429 */
Leonidas S. Barbosa9247f0b2014-10-28 15:44:30 -0200430 to_process = nbytes - processed;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000431
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300432 if ((to_process + processed) < nbytes)
433 NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
434 else
435 NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000436
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300437 NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
438
Eric Biggers7740bd52019-10-12 21:39:15 -0700439 rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src,
Herbert Xucc815652015-07-14 16:53:21 +0800440 &to_process, processed + req->assoclen,
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300441 csbcpb->cpb.aes_ccm.iv_or_ctr);
442 if (rc)
443 goto out;
444
445 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
446 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
447 if (rc)
448 goto out;
449
450 /* for partial completion, copy following for next
451 * entry into loop...
452 */
Eric Biggers7740bd52019-10-12 21:39:15 -0700453 memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300454 memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
455 csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
456 memcpy(csbcpb->cpb.aes_ccm.in_s0,
457 csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
458
459 NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
460
461 /* update stats */
462 atomic_inc(&(nx_ctx->stats->aes_ops));
Herbert Xub20d9a72021-06-17 16:00:12 +0800463 atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300464 &(nx_ctx->stats->aes_bytes));
465
466 processed += to_process;
467
468 } while (processed < nbytes);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000469
470 /* copy out the auth tag */
471 scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac,
Herbert Xucc815652015-07-14 16:53:21 +0800472 req->dst, nbytes + req->assoclen, authsize,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000473 SCATTERWALK_TO_SG);
Fionnuala Gunter2b188b3b82013-08-29 11:36:37 -0300474
Kent Yodera00bd6e2012-05-14 11:05:23 +0000475out:
Marcelo Cerric8491632013-08-12 18:49:37 -0300476 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000477 return rc;
478}
479
480static int ccm4309_aes_nx_encrypt(struct aead_request *req)
481{
482 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
Herbert Xu030f4e92015-07-07 17:30:25 +0800483 struct nx_gcm_rctx *rctx = aead_request_ctx(req);
Herbert Xu030f4e92015-07-07 17:30:25 +0800484 u8 *iv = rctx->iv;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000485
486 iv[0] = 3;
487 memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);
488 memcpy(iv + 4, req->iv, 8);
489
Eric Biggers7740bd52019-10-12 21:39:15 -0700490 return ccm_nx_encrypt(req, iv, req->assoclen - 8);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000491}
492
493static int ccm_aes_nx_encrypt(struct aead_request *req)
494{
Kent Yodera00bd6e2012-05-14 11:05:23 +0000495 int rc;
496
Eric Biggers7740bd52019-10-12 21:39:15 -0700497 rc = crypto_ccm_check_iv(req->iv);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000498 if (rc)
499 return rc;
500
Eric Biggers7740bd52019-10-12 21:39:15 -0700501 return ccm_nx_encrypt(req, req->iv, req->assoclen);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000502}
503
504static int ccm4309_aes_nx_decrypt(struct aead_request *req)
505{
506 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
Herbert Xu030f4e92015-07-07 17:30:25 +0800507 struct nx_gcm_rctx *rctx = aead_request_ctx(req);
Herbert Xu030f4e92015-07-07 17:30:25 +0800508 u8 *iv = rctx->iv;
Kent Yodera00bd6e2012-05-14 11:05:23 +0000509
510 iv[0] = 3;
511 memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);
512 memcpy(iv + 4, req->iv, 8);
513
Eric Biggers7740bd52019-10-12 21:39:15 -0700514 return ccm_nx_decrypt(req, iv, req->assoclen - 8);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000515}
516
517static int ccm_aes_nx_decrypt(struct aead_request *req)
518{
Kent Yodera00bd6e2012-05-14 11:05:23 +0000519 int rc;
520
Eric Biggers7740bd52019-10-12 21:39:15 -0700521 rc = crypto_ccm_check_iv(req->iv);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000522 if (rc)
523 return rc;
524
Eric Biggers7740bd52019-10-12 21:39:15 -0700525 return ccm_nx_decrypt(req, req->iv, req->assoclen);
Kent Yodera00bd6e2012-05-14 11:05:23 +0000526}
527
Herbert Xucc815652015-07-14 16:53:21 +0800528struct aead_alg nx_ccm_aes_alg = {
529 .base = {
530 .cra_name = "ccm(aes)",
531 .cra_driver_name = "ccm-aes-nx",
532 .cra_priority = 300,
Herbert Xu5e4b8c12015-08-13 17:29:06 +0800533 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
Herbert Xucc815652015-07-14 16:53:21 +0800534 .cra_blocksize = 1,
535 .cra_ctxsize = sizeof(struct nx_crypto_ctx),
536 .cra_module = THIS_MODULE,
537 },
538 .init = nx_crypto_ctx_aes_ccm_init,
539 .exit = nx_crypto_ctx_aead_exit,
540 .ivsize = AES_BLOCK_SIZE,
541 .maxauthsize = AES_BLOCK_SIZE,
542 .setkey = ccm_aes_nx_set_key,
543 .setauthsize = ccm_aes_nx_setauthsize,
544 .encrypt = ccm_aes_nx_encrypt,
545 .decrypt = ccm_aes_nx_decrypt,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000546};
547
Herbert Xucc815652015-07-14 16:53:21 +0800548struct aead_alg nx_ccm4309_aes_alg = {
549 .base = {
550 .cra_name = "rfc4309(ccm(aes))",
551 .cra_driver_name = "rfc4309-ccm-aes-nx",
552 .cra_priority = 300,
Herbert Xu5e4b8c12015-08-13 17:29:06 +0800553 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
Herbert Xucc815652015-07-14 16:53:21 +0800554 .cra_blocksize = 1,
555 .cra_ctxsize = sizeof(struct nx_crypto_ctx),
556 .cra_module = THIS_MODULE,
557 },
558 .init = nx_crypto_ctx_aes_ccm_init,
559 .exit = nx_crypto_ctx_aead_exit,
560 .ivsize = 8,
561 .maxauthsize = AES_BLOCK_SIZE,
562 .setkey = ccm4309_aes_nx_set_key,
563 .setauthsize = ccm4309_aes_nx_setauthsize,
564 .encrypt = ccm4309_aes_nx_encrypt,
565 .decrypt = ccm4309_aes_nx_decrypt,
Kent Yodera00bd6e2012-05-14 11:05:23 +0000566};