blob: 76c13dab217d933249cc605129fa824befabfd09 [file] [log] [blame]
Martin Willi71ebc4d2015-06-01 13:44:00 +02001/*
2 * ChaCha20-Poly1305 AEAD, RFC7539
3 *
4 * Copyright (C) 2015 Martin Willi
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 */
11
12#include <crypto/internal/aead.h>
13#include <crypto/internal/hash.h>
14#include <crypto/internal/skcipher.h>
15#include <crypto/scatterwalk.h>
Eric Biggers1ca1b912018-11-16 17:26:21 -080016#include <crypto/chacha.h>
Martin Willi2546f812015-07-16 19:14:05 +020017#include <crypto/poly1305.h>
Martin Willi71ebc4d2015-06-01 13:44:00 +020018#include <linux/err.h>
19#include <linux/init.h>
20#include <linux/kernel.h>
21#include <linux/module.h>
22
23#include "internal.h"
24
Martin Willi71ebc4d2015-06-01 13:44:00 +020025struct chachapoly_instance_ctx {
26 struct crypto_skcipher_spawn chacha;
27 struct crypto_ahash_spawn poly;
28 unsigned int saltlen;
29};
30
31struct chachapoly_ctx {
Herbert Xu1e1f0062016-07-12 13:17:40 +080032 struct crypto_skcipher *chacha;
Martin Willi71ebc4d2015-06-01 13:44:00 +020033 struct crypto_ahash *poly;
34 /* key bytes we use for the ChaCha20 IV */
35 unsigned int saltlen;
36 u8 salt[];
37};
38
39struct poly_req {
40 /* zero byte padding for AD/ciphertext, as needed */
41 u8 pad[POLY1305_BLOCK_SIZE];
42 /* tail data with AD/ciphertext lengths */
43 struct {
44 __le64 assoclen;
45 __le64 cryptlen;
46 } tail;
47 struct scatterlist src[1];
48 struct ahash_request req; /* must be last member */
49};
50
51struct chacha_req {
Eric Biggers1ca1b912018-11-16 17:26:21 -080052 u8 iv[CHACHA_IV_SIZE];
Martin Willi71ebc4d2015-06-01 13:44:00 +020053 struct scatterlist src[1];
Herbert Xu1e1f0062016-07-12 13:17:40 +080054 struct skcipher_request req; /* must be last member */
Martin Willi71ebc4d2015-06-01 13:44:00 +020055};
56
57struct chachapoly_req_ctx {
Herbert Xu74790922015-07-16 12:35:08 +080058 struct scatterlist src[2];
59 struct scatterlist dst[2];
Martin Willic2b7b20a2015-06-16 11:34:16 +020060 /* the key we generate for Poly1305 using Chacha20 */
61 u8 key[POLY1305_KEY_SIZE];
Martin Willi71ebc4d2015-06-01 13:44:00 +020062 /* calculated Poly1305 tag */
63 u8 tag[POLY1305_DIGEST_SIZE];
64 /* length of data to en/decrypt, without ICV */
65 unsigned int cryptlen;
Herbert Xu74790922015-07-16 12:35:08 +080066 /* Actual AD, excluding IV */
67 unsigned int assoclen;
Eric Biggers7545b6c2019-05-31 11:12:30 -070068 /* request flags, with MAY_SLEEP cleared if needed */
69 u32 flags;
Martin Willi71ebc4d2015-06-01 13:44:00 +020070 union {
71 struct poly_req poly;
72 struct chacha_req chacha;
73 } u;
74};
75
76static inline void async_done_continue(struct aead_request *req, int err,
77 int (*cont)(struct aead_request *))
78{
Eric Biggers7545b6c2019-05-31 11:12:30 -070079 if (!err) {
80 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
81
82 rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
Martin Willi71ebc4d2015-06-01 13:44:00 +020083 err = cont(req);
Eric Biggers7545b6c2019-05-31 11:12:30 -070084 }
Martin Willi71ebc4d2015-06-01 13:44:00 +020085
86 if (err != -EINPROGRESS && err != -EBUSY)
87 aead_request_complete(req, err);
88}
89
90static void chacha_iv(u8 *iv, struct aead_request *req, u32 icb)
91{
92 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
93 __le32 leicb = cpu_to_le32(icb);
94
95 memcpy(iv, &leicb, sizeof(leicb));
96 memcpy(iv + sizeof(leicb), ctx->salt, ctx->saltlen);
97 memcpy(iv + sizeof(leicb) + ctx->saltlen, req->iv,
Eric Biggers1ca1b912018-11-16 17:26:21 -080098 CHACHA_IV_SIZE - sizeof(leicb) - ctx->saltlen);
Martin Willi71ebc4d2015-06-01 13:44:00 +020099}
100
101static int poly_verify_tag(struct aead_request *req)
102{
103 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
104 u8 tag[sizeof(rctx->tag)];
105
Herbert Xu74790922015-07-16 12:35:08 +0800106 scatterwalk_map_and_copy(tag, req->src,
107 req->assoclen + rctx->cryptlen,
108 sizeof(tag), 0);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200109 if (crypto_memneq(tag, rctx->tag, sizeof(tag)))
110 return -EBADMSG;
111 return 0;
112}
113
114static int poly_copy_tag(struct aead_request *req)
115{
116 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
117
Herbert Xu74790922015-07-16 12:35:08 +0800118 scatterwalk_map_and_copy(rctx->tag, req->dst,
119 req->assoclen + rctx->cryptlen,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200120 sizeof(rctx->tag), 1);
121 return 0;
122}
123
124static void chacha_decrypt_done(struct crypto_async_request *areq, int err)
125{
126 async_done_continue(areq->data, err, poly_verify_tag);
127}
128
129static int chacha_decrypt(struct aead_request *req)
130{
131 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
132 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
133 struct chacha_req *creq = &rctx->u.chacha;
Herbert Xu74790922015-07-16 12:35:08 +0800134 struct scatterlist *src, *dst;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200135 int err;
136
Jason A. Donenfeld161151d2015-12-06 02:51:38 +0100137 if (rctx->cryptlen == 0)
138 goto skip;
139
Martin Willi71ebc4d2015-06-01 13:44:00 +0200140 chacha_iv(creq->iv, req, 1);
141
Herbert Xu74790922015-07-16 12:35:08 +0800142 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
143 dst = src;
Eric Biggers76cadf22019-06-02 22:46:34 -0700144 if (req->src != req->dst)
Herbert Xu74790922015-07-16 12:35:08 +0800145 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
Herbert Xu74790922015-07-16 12:35:08 +0800146
Eric Biggers7545b6c2019-05-31 11:12:30 -0700147 skcipher_request_set_callback(&creq->req, rctx->flags,
Herbert Xu1e1f0062016-07-12 13:17:40 +0800148 chacha_decrypt_done, req);
149 skcipher_request_set_tfm(&creq->req, ctx->chacha);
150 skcipher_request_set_crypt(&creq->req, src, dst,
151 rctx->cryptlen, creq->iv);
152 err = crypto_skcipher_decrypt(&creq->req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200153 if (err)
154 return err;
155
Jason A. Donenfeld161151d2015-12-06 02:51:38 +0100156skip:
Martin Willi71ebc4d2015-06-01 13:44:00 +0200157 return poly_verify_tag(req);
158}
159
160static int poly_tail_continue(struct aead_request *req)
161{
162 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
163
164 if (rctx->cryptlen == req->cryptlen) /* encrypting */
165 return poly_copy_tag(req);
166
167 return chacha_decrypt(req);
168}
169
170static void poly_tail_done(struct crypto_async_request *areq, int err)
171{
172 async_done_continue(areq->data, err, poly_tail_continue);
173}
174
175static int poly_tail(struct aead_request *req)
176{
Herbert Xu74790922015-07-16 12:35:08 +0800177 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
178 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200179 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
180 struct poly_req *preq = &rctx->u.poly;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200181 int err;
182
Eric Biggers76cadf22019-06-02 22:46:34 -0700183 preq->tail.assoclen = cpu_to_le64(rctx->assoclen);
184 preq->tail.cryptlen = cpu_to_le64(rctx->cryptlen);
185 sg_init_one(preq->src, &preq->tail, sizeof(preq->tail));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200186
Eric Biggers7545b6c2019-05-31 11:12:30 -0700187 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200188 poly_tail_done, req);
189 ahash_request_set_tfm(&preq->req, ctx->poly);
190 ahash_request_set_crypt(&preq->req, preq->src,
191 rctx->tag, sizeof(preq->tail));
192
193 err = crypto_ahash_finup(&preq->req);
194 if (err)
195 return err;
196
197 return poly_tail_continue(req);
198}
199
200static void poly_cipherpad_done(struct crypto_async_request *areq, int err)
201{
202 async_done_continue(areq->data, err, poly_tail);
203}
204
205static int poly_cipherpad(struct aead_request *req)
206{
207 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
208 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
209 struct poly_req *preq = &rctx->u.poly;
Eric Biggers76cadf22019-06-02 22:46:34 -0700210 unsigned int padlen;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200211 int err;
212
Eric Biggers76cadf22019-06-02 22:46:34 -0700213 padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200214 memset(preq->pad, 0, sizeof(preq->pad));
Eric Biggers76cadf22019-06-02 22:46:34 -0700215 sg_init_one(preq->src, preq->pad, padlen);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200216
Eric Biggers7545b6c2019-05-31 11:12:30 -0700217 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200218 poly_cipherpad_done, req);
219 ahash_request_set_tfm(&preq->req, ctx->poly);
220 ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
221
222 err = crypto_ahash_update(&preq->req);
223 if (err)
224 return err;
225
226 return poly_tail(req);
227}
228
229static void poly_cipher_done(struct crypto_async_request *areq, int err)
230{
231 async_done_continue(areq->data, err, poly_cipherpad);
232}
233
234static int poly_cipher(struct aead_request *req)
235{
236 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
237 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
238 struct poly_req *preq = &rctx->u.poly;
239 struct scatterlist *crypt = req->src;
240 int err;
241
242 if (rctx->cryptlen == req->cryptlen) /* encrypting */
243 crypt = req->dst;
244
Herbert Xu74790922015-07-16 12:35:08 +0800245 crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen);
246
Eric Biggers7545b6c2019-05-31 11:12:30 -0700247 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200248 poly_cipher_done, req);
249 ahash_request_set_tfm(&preq->req, ctx->poly);
250 ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen);
251
252 err = crypto_ahash_update(&preq->req);
253 if (err)
254 return err;
255
256 return poly_cipherpad(req);
257}
258
259static void poly_adpad_done(struct crypto_async_request *areq, int err)
260{
261 async_done_continue(areq->data, err, poly_cipher);
262}
263
264static int poly_adpad(struct aead_request *req)
265{
266 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
267 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
268 struct poly_req *preq = &rctx->u.poly;
Eric Biggers76cadf22019-06-02 22:46:34 -0700269 unsigned int padlen;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200270 int err;
271
Eric Biggers76cadf22019-06-02 22:46:34 -0700272 padlen = -rctx->assoclen % POLY1305_BLOCK_SIZE;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200273 memset(preq->pad, 0, sizeof(preq->pad));
Eric Biggers76cadf22019-06-02 22:46:34 -0700274 sg_init_one(preq->src, preq->pad, padlen);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200275
Eric Biggers7545b6c2019-05-31 11:12:30 -0700276 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200277 poly_adpad_done, req);
278 ahash_request_set_tfm(&preq->req, ctx->poly);
279 ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
280
281 err = crypto_ahash_update(&preq->req);
282 if (err)
283 return err;
284
285 return poly_cipher(req);
286}
287
288static void poly_ad_done(struct crypto_async_request *areq, int err)
289{
290 async_done_continue(areq->data, err, poly_adpad);
291}
292
293static int poly_ad(struct aead_request *req)
294{
295 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
296 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
297 struct poly_req *preq = &rctx->u.poly;
298 int err;
299
Eric Biggers7545b6c2019-05-31 11:12:30 -0700300 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200301 poly_ad_done, req);
302 ahash_request_set_tfm(&preq->req, ctx->poly);
Herbert Xu74790922015-07-16 12:35:08 +0800303 ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200304
305 err = crypto_ahash_update(&preq->req);
306 if (err)
307 return err;
308
309 return poly_adpad(req);
310}
311
Martin Willic2b7b20a2015-06-16 11:34:16 +0200312static void poly_setkey_done(struct crypto_async_request *areq, int err)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200313{
314 async_done_continue(areq->data, err, poly_ad);
315}
316
Martin Willic2b7b20a2015-06-16 11:34:16 +0200317static int poly_setkey(struct aead_request *req)
318{
319 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
320 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
321 struct poly_req *preq = &rctx->u.poly;
322 int err;
323
Eric Biggers76cadf22019-06-02 22:46:34 -0700324 sg_init_one(preq->src, rctx->key, sizeof(rctx->key));
Martin Willic2b7b20a2015-06-16 11:34:16 +0200325
Eric Biggers7545b6c2019-05-31 11:12:30 -0700326 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willic2b7b20a2015-06-16 11:34:16 +0200327 poly_setkey_done, req);
328 ahash_request_set_tfm(&preq->req, ctx->poly);
329 ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key));
330
331 err = crypto_ahash_update(&preq->req);
332 if (err)
333 return err;
334
335 return poly_ad(req);
336}
337
338static void poly_init_done(struct crypto_async_request *areq, int err)
339{
340 async_done_continue(areq->data, err, poly_setkey);
341}
342
Martin Willi71ebc4d2015-06-01 13:44:00 +0200343static int poly_init(struct aead_request *req)
344{
345 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
346 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
347 struct poly_req *preq = &rctx->u.poly;
348 int err;
349
Eric Biggers7545b6c2019-05-31 11:12:30 -0700350 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200351 poly_init_done, req);
352 ahash_request_set_tfm(&preq->req, ctx->poly);
353
354 err = crypto_ahash_init(&preq->req);
355 if (err)
356 return err;
357
Martin Willic2b7b20a2015-06-16 11:34:16 +0200358 return poly_setkey(req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200359}
360
361static void poly_genkey_done(struct crypto_async_request *areq, int err)
362{
Martin Willic2b7b20a2015-06-16 11:34:16 +0200363 async_done_continue(areq->data, err, poly_init);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200364}
365
366static int poly_genkey(struct aead_request *req)
367{
Herbert Xu74790922015-07-16 12:35:08 +0800368 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
369 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200370 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
371 struct chacha_req *creq = &rctx->u.chacha;
372 int err;
373
Herbert Xu74790922015-07-16 12:35:08 +0800374 rctx->assoclen = req->assoclen;
375
376 if (crypto_aead_ivsize(tfm) == 8) {
377 if (rctx->assoclen < 8)
378 return -EINVAL;
379 rctx->assoclen -= 8;
380 }
381
Martin Willic2b7b20a2015-06-16 11:34:16 +0200382 memset(rctx->key, 0, sizeof(rctx->key));
Eric Biggers76cadf22019-06-02 22:46:34 -0700383 sg_init_one(creq->src, rctx->key, sizeof(rctx->key));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200384
385 chacha_iv(creq->iv, req, 0);
386
Eric Biggers7545b6c2019-05-31 11:12:30 -0700387 skcipher_request_set_callback(&creq->req, rctx->flags,
Herbert Xu1e1f0062016-07-12 13:17:40 +0800388 poly_genkey_done, req);
389 skcipher_request_set_tfm(&creq->req, ctx->chacha);
390 skcipher_request_set_crypt(&creq->req, creq->src, creq->src,
391 POLY1305_KEY_SIZE, creq->iv);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200392
Herbert Xu1e1f0062016-07-12 13:17:40 +0800393 err = crypto_skcipher_decrypt(&creq->req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200394 if (err)
395 return err;
396
Martin Willic2b7b20a2015-06-16 11:34:16 +0200397 return poly_init(req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200398}
399
400static void chacha_encrypt_done(struct crypto_async_request *areq, int err)
401{
402 async_done_continue(areq->data, err, poly_genkey);
403}
404
405static int chacha_encrypt(struct aead_request *req)
406{
407 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
408 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
409 struct chacha_req *creq = &rctx->u.chacha;
Herbert Xu74790922015-07-16 12:35:08 +0800410 struct scatterlist *src, *dst;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200411 int err;
412
Jason A. Donenfeld161151d2015-12-06 02:51:38 +0100413 if (req->cryptlen == 0)
414 goto skip;
415
Martin Willi71ebc4d2015-06-01 13:44:00 +0200416 chacha_iv(creq->iv, req, 1);
417
Herbert Xu74790922015-07-16 12:35:08 +0800418 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
419 dst = src;
Eric Biggers76cadf22019-06-02 22:46:34 -0700420 if (req->src != req->dst)
Herbert Xu74790922015-07-16 12:35:08 +0800421 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
Herbert Xu74790922015-07-16 12:35:08 +0800422
Eric Biggers7545b6c2019-05-31 11:12:30 -0700423 skcipher_request_set_callback(&creq->req, rctx->flags,
Herbert Xu1e1f0062016-07-12 13:17:40 +0800424 chacha_encrypt_done, req);
425 skcipher_request_set_tfm(&creq->req, ctx->chacha);
426 skcipher_request_set_crypt(&creq->req, src, dst,
427 req->cryptlen, creq->iv);
428 err = crypto_skcipher_encrypt(&creq->req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200429 if (err)
430 return err;
431
Jason A. Donenfeld161151d2015-12-06 02:51:38 +0100432skip:
Martin Willi71ebc4d2015-06-01 13:44:00 +0200433 return poly_genkey(req);
434}
435
436static int chachapoly_encrypt(struct aead_request *req)
437{
438 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
439
440 rctx->cryptlen = req->cryptlen;
Eric Biggers7545b6c2019-05-31 11:12:30 -0700441 rctx->flags = aead_request_flags(req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200442
443 /* encrypt call chain:
444 * - chacha_encrypt/done()
Martin Willic2b7b20a2015-06-16 11:34:16 +0200445 * - poly_genkey/done()
Martin Willi71ebc4d2015-06-01 13:44:00 +0200446 * - poly_init/done()
Martin Willic2b7b20a2015-06-16 11:34:16 +0200447 * - poly_setkey/done()
Martin Willi71ebc4d2015-06-01 13:44:00 +0200448 * - poly_ad/done()
449 * - poly_adpad/done()
450 * - poly_cipher/done()
451 * - poly_cipherpad/done()
452 * - poly_tail/done/continue()
453 * - poly_copy_tag()
454 */
455 return chacha_encrypt(req);
456}
457
458static int chachapoly_decrypt(struct aead_request *req)
459{
460 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
461
Martin Willi71ebc4d2015-06-01 13:44:00 +0200462 rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE;
Eric Biggers7545b6c2019-05-31 11:12:30 -0700463 rctx->flags = aead_request_flags(req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200464
465 /* decrypt call chain:
Martin Willic2b7b20a2015-06-16 11:34:16 +0200466 * - poly_genkey/done()
Martin Willi71ebc4d2015-06-01 13:44:00 +0200467 * - poly_init/done()
Martin Willic2b7b20a2015-06-16 11:34:16 +0200468 * - poly_setkey/done()
Martin Willi71ebc4d2015-06-01 13:44:00 +0200469 * - poly_ad/done()
470 * - poly_adpad/done()
471 * - poly_cipher/done()
472 * - poly_cipherpad/done()
473 * - poly_tail/done/continue()
474 * - chacha_decrypt/done()
475 * - poly_verify_tag()
476 */
477 return poly_genkey(req);
478}
479
480static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
481 unsigned int keylen)
482{
483 struct chachapoly_ctx *ctx = crypto_aead_ctx(aead);
484 int err;
485
Eric Biggers1ca1b912018-11-16 17:26:21 -0800486 if (keylen != ctx->saltlen + CHACHA_KEY_SIZE)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200487 return -EINVAL;
488
489 keylen -= ctx->saltlen;
490 memcpy(ctx->salt, key + keylen, ctx->saltlen);
491
Herbert Xu1e1f0062016-07-12 13:17:40 +0800492 crypto_skcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK);
493 crypto_skcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) &
494 CRYPTO_TFM_REQ_MASK);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200495
Herbert Xu1e1f0062016-07-12 13:17:40 +0800496 err = crypto_skcipher_setkey(ctx->chacha, key, keylen);
497 crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctx->chacha) &
498 CRYPTO_TFM_RES_MASK);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200499 return err;
500}
501
502static int chachapoly_setauthsize(struct crypto_aead *tfm,
503 unsigned int authsize)
504{
505 if (authsize != POLY1305_DIGEST_SIZE)
506 return -EINVAL;
507
508 return 0;
509}
510
Herbert Xu74790922015-07-16 12:35:08 +0800511static int chachapoly_init(struct crypto_aead *tfm)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200512{
Herbert Xu74790922015-07-16 12:35:08 +0800513 struct aead_instance *inst = aead_alg_instance(tfm);
514 struct chachapoly_instance_ctx *ictx = aead_instance_ctx(inst);
515 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
Herbert Xu1e1f0062016-07-12 13:17:40 +0800516 struct crypto_skcipher *chacha;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200517 struct crypto_ahash *poly;
518 unsigned long align;
519
520 poly = crypto_spawn_ahash(&ictx->poly);
521 if (IS_ERR(poly))
522 return PTR_ERR(poly);
523
Eric Biggers60425a82016-10-28 09:52:19 -0700524 chacha = crypto_spawn_skcipher(&ictx->chacha);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200525 if (IS_ERR(chacha)) {
526 crypto_free_ahash(poly);
527 return PTR_ERR(chacha);
528 }
529
530 ctx->chacha = chacha;
531 ctx->poly = poly;
532 ctx->saltlen = ictx->saltlen;
533
Herbert Xu74790922015-07-16 12:35:08 +0800534 align = crypto_aead_alignmask(tfm);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200535 align &= ~(crypto_tfm_ctx_alignment() - 1);
Herbert Xu74790922015-07-16 12:35:08 +0800536 crypto_aead_set_reqsize(
537 tfm,
538 align + offsetof(struct chachapoly_req_ctx, u) +
539 max(offsetof(struct chacha_req, req) +
Herbert Xu1e1f0062016-07-12 13:17:40 +0800540 sizeof(struct skcipher_request) +
541 crypto_skcipher_reqsize(chacha),
Herbert Xu74790922015-07-16 12:35:08 +0800542 offsetof(struct poly_req, req) +
543 sizeof(struct ahash_request) +
544 crypto_ahash_reqsize(poly)));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200545
546 return 0;
547}
548
Herbert Xu74790922015-07-16 12:35:08 +0800549static void chachapoly_exit(struct crypto_aead *tfm)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200550{
Herbert Xu74790922015-07-16 12:35:08 +0800551 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200552
553 crypto_free_ahash(ctx->poly);
Herbert Xu1e1f0062016-07-12 13:17:40 +0800554 crypto_free_skcipher(ctx->chacha);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200555}
556
Herbert Xu74790922015-07-16 12:35:08 +0800557static void chachapoly_free(struct aead_instance *inst)
558{
559 struct chachapoly_instance_ctx *ctx = aead_instance_ctx(inst);
560
561 crypto_drop_skcipher(&ctx->chacha);
562 crypto_drop_ahash(&ctx->poly);
563 kfree(inst);
564}
565
566static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb,
567 const char *name, unsigned int ivsize)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200568{
569 struct crypto_attr_type *algt;
Herbert Xu74790922015-07-16 12:35:08 +0800570 struct aead_instance *inst;
Herbert Xu1e1f0062016-07-12 13:17:40 +0800571 struct skcipher_alg *chacha;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200572 struct crypto_alg *poly;
Herbert Xu74790922015-07-16 12:35:08 +0800573 struct hash_alg_common *poly_hash;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200574 struct chachapoly_instance_ctx *ctx;
575 const char *chacha_name, *poly_name;
576 int err;
577
578 if (ivsize > CHACHAPOLY_IV_SIZE)
Herbert Xu74790922015-07-16 12:35:08 +0800579 return -EINVAL;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200580
581 algt = crypto_get_attr_type(tb);
582 if (IS_ERR(algt))
Herbert Xu74790922015-07-16 12:35:08 +0800583 return PTR_ERR(algt);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200584
Herbert Xu5e4b8c12015-08-13 17:29:06 +0800585 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
Herbert Xu74790922015-07-16 12:35:08 +0800586 return -EINVAL;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200587
588 chacha_name = crypto_attr_alg_name(tb[1]);
589 if (IS_ERR(chacha_name))
Herbert Xu74790922015-07-16 12:35:08 +0800590 return PTR_ERR(chacha_name);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200591 poly_name = crypto_attr_alg_name(tb[2]);
592 if (IS_ERR(poly_name))
Herbert Xu74790922015-07-16 12:35:08 +0800593 return PTR_ERR(poly_name);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200594
595 poly = crypto_find_alg(poly_name, &crypto_ahash_type,
596 CRYPTO_ALG_TYPE_HASH,
Herbert Xu1e1f0062016-07-12 13:17:40 +0800597 CRYPTO_ALG_TYPE_AHASH_MASK |
598 crypto_requires_sync(algt->type,
599 algt->mask));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200600 if (IS_ERR(poly))
Herbert Xu74790922015-07-16 12:35:08 +0800601 return PTR_ERR(poly);
Eric Biggerse57121d2017-12-11 12:15:17 -0800602 poly_hash = __crypto_hash_alg_common(poly);
603
604 err = -EINVAL;
605 if (poly_hash->digestsize != POLY1305_DIGEST_SIZE)
606 goto out_put_poly;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200607
608 err = -ENOMEM;
609 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
610 if (!inst)
611 goto out_put_poly;
612
Herbert Xu74790922015-07-16 12:35:08 +0800613 ctx = aead_instance_ctx(inst);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200614 ctx->saltlen = CHACHAPOLY_IV_SIZE - ivsize;
Herbert Xu74790922015-07-16 12:35:08 +0800615 err = crypto_init_ahash_spawn(&ctx->poly, poly_hash,
616 aead_crypto_instance(inst));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200617 if (err)
618 goto err_free_inst;
619
Herbert Xu74790922015-07-16 12:35:08 +0800620 crypto_set_skcipher_spawn(&ctx->chacha, aead_crypto_instance(inst));
Eric Biggersa35528e2016-10-28 09:51:13 -0700621 err = crypto_grab_skcipher(&ctx->chacha, chacha_name, 0,
622 crypto_requires_sync(algt->type,
623 algt->mask));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200624 if (err)
625 goto err_drop_poly;
626
Herbert Xu1e1f0062016-07-12 13:17:40 +0800627 chacha = crypto_spawn_skcipher_alg(&ctx->chacha);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200628
629 err = -EINVAL;
630 /* Need 16-byte IV size, including Initial Block Counter value */
Eric Biggers1ca1b912018-11-16 17:26:21 -0800631 if (crypto_skcipher_alg_ivsize(chacha) != CHACHA_IV_SIZE)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200632 goto out_drop_chacha;
633 /* Not a stream cipher? */
Herbert Xu1e1f0062016-07-12 13:17:40 +0800634 if (chacha->base.cra_blocksize != 1)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200635 goto out_drop_chacha;
636
637 err = -ENAMETOOLONG;
Herbert Xu74790922015-07-16 12:35:08 +0800638 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
Eric Biggers5e27f382019-03-31 13:04:16 -0700639 "%s(%s,%s)", name, chacha->base.cra_name,
640 poly->cra_name) >= CRYPTO_MAX_ALG_NAME)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200641 goto out_drop_chacha;
Herbert Xu74790922015-07-16 12:35:08 +0800642 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
Herbert Xu1e1f0062016-07-12 13:17:40 +0800643 "%s(%s,%s)", name, chacha->base.cra_driver_name,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200644 poly->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
645 goto out_drop_chacha;
646
Herbert Xu1e1f0062016-07-12 13:17:40 +0800647 inst->alg.base.cra_flags = (chacha->base.cra_flags | poly->cra_flags) &
Herbert Xu74790922015-07-16 12:35:08 +0800648 CRYPTO_ALG_ASYNC;
Herbert Xu1e1f0062016-07-12 13:17:40 +0800649 inst->alg.base.cra_priority = (chacha->base.cra_priority +
Herbert Xu74790922015-07-16 12:35:08 +0800650 poly->cra_priority) / 2;
651 inst->alg.base.cra_blocksize = 1;
Herbert Xu1e1f0062016-07-12 13:17:40 +0800652 inst->alg.base.cra_alignmask = chacha->base.cra_alignmask |
Herbert Xu74790922015-07-16 12:35:08 +0800653 poly->cra_alignmask;
654 inst->alg.base.cra_ctxsize = sizeof(struct chachapoly_ctx) +
655 ctx->saltlen;
656 inst->alg.ivsize = ivsize;
Herbert Xu1e1f0062016-07-12 13:17:40 +0800657 inst->alg.chunksize = crypto_skcipher_alg_chunksize(chacha);
Herbert Xu74790922015-07-16 12:35:08 +0800658 inst->alg.maxauthsize = POLY1305_DIGEST_SIZE;
659 inst->alg.init = chachapoly_init;
660 inst->alg.exit = chachapoly_exit;
661 inst->alg.encrypt = chachapoly_encrypt;
662 inst->alg.decrypt = chachapoly_decrypt;
663 inst->alg.setkey = chachapoly_setkey;
664 inst->alg.setauthsize = chachapoly_setauthsize;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200665
Herbert Xu74790922015-07-16 12:35:08 +0800666 inst->free = chachapoly_free;
667
668 err = aead_register_instance(tmpl, inst);
669 if (err)
670 goto out_drop_chacha;
671
672out_put_poly:
Martin Willi71ebc4d2015-06-01 13:44:00 +0200673 crypto_mod_put(poly);
Herbert Xu74790922015-07-16 12:35:08 +0800674 return err;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200675
676out_drop_chacha:
677 crypto_drop_skcipher(&ctx->chacha);
678err_drop_poly:
679 crypto_drop_ahash(&ctx->poly);
680err_free_inst:
681 kfree(inst);
Herbert Xu74790922015-07-16 12:35:08 +0800682 goto out_put_poly;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200683}
684
Herbert Xu74790922015-07-16 12:35:08 +0800685static int rfc7539_create(struct crypto_template *tmpl, struct rtattr **tb)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200686{
Herbert Xu74790922015-07-16 12:35:08 +0800687 return chachapoly_create(tmpl, tb, "rfc7539", 12);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200688}
689
Herbert Xu74790922015-07-16 12:35:08 +0800690static int rfc7539esp_create(struct crypto_template *tmpl, struct rtattr **tb)
Martin Willi4db4ad22015-06-01 13:44:02 +0200691{
Herbert Xu74790922015-07-16 12:35:08 +0800692 return chachapoly_create(tmpl, tb, "rfc7539esp", 8);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200693}
694
Xiongfeng Wang1a5e02b2019-01-18 13:58:15 +0800695static struct crypto_template rfc7539_tmpls[] = {
696 {
697 .name = "rfc7539",
698 .create = rfc7539_create,
699 .module = THIS_MODULE,
700 }, {
701 .name = "rfc7539esp",
702 .create = rfc7539esp_create,
703 .module = THIS_MODULE,
704 },
Martin Willi4db4ad22015-06-01 13:44:02 +0200705};
706
Martin Willi71ebc4d2015-06-01 13:44:00 +0200707static int __init chacha20poly1305_module_init(void)
708{
Xiongfeng Wang1a5e02b2019-01-18 13:58:15 +0800709 return crypto_register_templates(rfc7539_tmpls,
710 ARRAY_SIZE(rfc7539_tmpls));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200711}
712
713static void __exit chacha20poly1305_module_exit(void)
714{
Xiongfeng Wang1a5e02b2019-01-18 13:58:15 +0800715 crypto_unregister_templates(rfc7539_tmpls,
716 ARRAY_SIZE(rfc7539_tmpls));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200717}
718
Eric Biggersc4741b22019-04-11 21:57:42 -0700719subsys_initcall(chacha20poly1305_module_init);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200720module_exit(chacha20poly1305_module_exit);
721
722MODULE_LICENSE("GPL");
723MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
724MODULE_DESCRIPTION("ChaCha20-Poly1305 AEAD");
Martin Willi71ebc4d2015-06-01 13:44:00 +0200725MODULE_ALIAS_CRYPTO("rfc7539");
Martin Willi4db4ad22015-06-01 13:44:02 +0200726MODULE_ALIAS_CRYPTO("rfc7539esp");