Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This contains encryption functions for per-file encryption. |
| 3 | * |
| 4 | * Copyright (C) 2015, Google, Inc. |
| 5 | * Copyright (C) 2015, Motorola Mobility |
| 6 | * |
| 7 | * Written by Michael Halcrow, 2014. |
| 8 | * |
| 9 | * Filename encryption additions |
| 10 | * Uday Savagaonkar, 2014 |
| 11 | * Encryption policy handling additions |
| 12 | * Ildar Muslukhov, 2014 |
| 13 | * Add fscrypt_pullback_bio_page() |
| 14 | * Jaegeuk Kim, 2015. |
| 15 | * |
| 16 | * This has not yet undergone a rigorous security audit. |
| 17 | * |
| 18 | * The usage of AES-XTS should conform to recommendations in NIST |
| 19 | * Special Publication 800-38E and IEEE P1619/D16. |
| 20 | */ |
| 21 | |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 22 | #include <linux/pagemap.h> |
| 23 | #include <linux/mempool.h> |
| 24 | #include <linux/module.h> |
| 25 | #include <linux/scatterlist.h> |
| 26 | #include <linux/ratelimit.h> |
| 27 | #include <linux/bio.h> |
| 28 | #include <linux/dcache.h> |
Jaegeuk Kim | 03a8bb0 | 2016-04-12 16:05:36 -0700 | [diff] [blame] | 29 | #include <linux/namei.h> |
Theodore Ts'o | cc4e0df | 2016-11-26 22:05:18 -0500 | [diff] [blame] | 30 | #include "fscrypt_private.h" |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 31 | |
| 32 | static unsigned int num_prealloc_crypto_pages = 32; |
| 33 | static unsigned int num_prealloc_crypto_ctxs = 128; |
| 34 | |
| 35 | module_param(num_prealloc_crypto_pages, uint, 0444); |
| 36 | MODULE_PARM_DESC(num_prealloc_crypto_pages, |
| 37 | "Number of crypto pages to preallocate"); |
| 38 | module_param(num_prealloc_crypto_ctxs, uint, 0444); |
| 39 | MODULE_PARM_DESC(num_prealloc_crypto_ctxs, |
| 40 | "Number of crypto contexts to preallocate"); |
| 41 | |
| 42 | static mempool_t *fscrypt_bounce_page_pool = NULL; |
| 43 | |
| 44 | static LIST_HEAD(fscrypt_free_ctxs); |
| 45 | static DEFINE_SPINLOCK(fscrypt_ctx_lock); |
| 46 | |
| 47 | static struct workqueue_struct *fscrypt_read_workqueue; |
| 48 | static DEFINE_MUTEX(fscrypt_init_mutex); |
| 49 | |
| 50 | static struct kmem_cache *fscrypt_ctx_cachep; |
| 51 | struct kmem_cache *fscrypt_info_cachep; |
| 52 | |
| 53 | /** |
| 54 | * fscrypt_release_ctx() - Releases an encryption context |
| 55 | * @ctx: The encryption context to release. |
| 56 | * |
| 57 | * If the encryption context was allocated from the pre-allocated pool, returns |
| 58 | * it to that pool. Else, frees it. |
| 59 | * |
| 60 | * If there's a bounce page in the context, this frees that. |
| 61 | */ |
| 62 | void fscrypt_release_ctx(struct fscrypt_ctx *ctx) |
| 63 | { |
| 64 | unsigned long flags; |
| 65 | |
David Gstir | 6a34e4d | 2016-12-06 23:53:58 +0100 | [diff] [blame] | 66 | if (ctx->flags & FS_CTX_HAS_BOUNCE_BUFFER_FL && ctx->w.bounce_page) { |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 67 | mempool_free(ctx->w.bounce_page, fscrypt_bounce_page_pool); |
| 68 | ctx->w.bounce_page = NULL; |
| 69 | } |
| 70 | ctx->w.control_page = NULL; |
| 71 | if (ctx->flags & FS_CTX_REQUIRES_FREE_ENCRYPT_FL) { |
| 72 | kmem_cache_free(fscrypt_ctx_cachep, ctx); |
| 73 | } else { |
| 74 | spin_lock_irqsave(&fscrypt_ctx_lock, flags); |
| 75 | list_add(&ctx->free_list, &fscrypt_free_ctxs); |
| 76 | spin_unlock_irqrestore(&fscrypt_ctx_lock, flags); |
| 77 | } |
| 78 | } |
| 79 | EXPORT_SYMBOL(fscrypt_release_ctx); |
| 80 | |
| 81 | /** |
| 82 | * fscrypt_get_ctx() - Gets an encryption context |
| 83 | * @inode: The inode for which we are doing the crypto |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 84 | * @gfp_flags: The gfp flag for memory allocation |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 85 | * |
| 86 | * Allocates and initializes an encryption context. |
| 87 | * |
| 88 | * Return: An allocated and initialized encryption context on success; error |
| 89 | * value or NULL otherwise. |
| 90 | */ |
David Gstir | 0b93e1b | 2016-11-13 22:20:47 +0100 | [diff] [blame] | 91 | struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *inode, gfp_t gfp_flags) |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 92 | { |
| 93 | struct fscrypt_ctx *ctx = NULL; |
| 94 | struct fscrypt_info *ci = inode->i_crypt_info; |
| 95 | unsigned long flags; |
| 96 | |
| 97 | if (ci == NULL) |
| 98 | return ERR_PTR(-ENOKEY); |
| 99 | |
| 100 | /* |
| 101 | * We first try getting the ctx from a free list because in |
| 102 | * the common case the ctx will have an allocated and |
| 103 | * initialized crypto tfm, so it's probably a worthwhile |
| 104 | * optimization. For the bounce page, we first try getting it |
| 105 | * from the kernel allocator because that's just about as fast |
| 106 | * as getting it from a list and because a cache of free pages |
| 107 | * should generally be a "last resort" option for a filesystem |
| 108 | * to be able to do its job. |
| 109 | */ |
| 110 | spin_lock_irqsave(&fscrypt_ctx_lock, flags); |
| 111 | ctx = list_first_entry_or_null(&fscrypt_free_ctxs, |
| 112 | struct fscrypt_ctx, free_list); |
| 113 | if (ctx) |
| 114 | list_del(&ctx->free_list); |
| 115 | spin_unlock_irqrestore(&fscrypt_ctx_lock, flags); |
| 116 | if (!ctx) { |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 117 | ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, gfp_flags); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 118 | if (!ctx) |
| 119 | return ERR_PTR(-ENOMEM); |
| 120 | ctx->flags |= FS_CTX_REQUIRES_FREE_ENCRYPT_FL; |
| 121 | } else { |
| 122 | ctx->flags &= ~FS_CTX_REQUIRES_FREE_ENCRYPT_FL; |
| 123 | } |
David Gstir | 6a34e4d | 2016-12-06 23:53:58 +0100 | [diff] [blame] | 124 | ctx->flags &= ~FS_CTX_HAS_BOUNCE_BUFFER_FL; |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 125 | return ctx; |
| 126 | } |
| 127 | EXPORT_SYMBOL(fscrypt_get_ctx); |
| 128 | |
| 129 | /** |
Eric Biggers | 53fd755 | 2016-09-15 16:51:01 -0400 | [diff] [blame] | 130 | * page_crypt_complete() - completion callback for page crypto |
| 131 | * @req: The asynchronous cipher request context |
| 132 | * @res: The result of the cipher operation |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 133 | */ |
Eric Biggers | 53fd755 | 2016-09-15 16:51:01 -0400 | [diff] [blame] | 134 | static void page_crypt_complete(struct crypto_async_request *req, int res) |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 135 | { |
| 136 | struct fscrypt_completion_result *ecr = req->data; |
| 137 | |
| 138 | if (res == -EINPROGRESS) |
| 139 | return; |
| 140 | ecr->res = res; |
| 141 | complete(&ecr->completion); |
| 142 | } |
| 143 | |
| 144 | typedef enum { |
| 145 | FS_DECRYPT = 0, |
| 146 | FS_ENCRYPT, |
| 147 | } fscrypt_direction_t; |
| 148 | |
David Gstir | 0b93e1b | 2016-11-13 22:20:47 +0100 | [diff] [blame] | 149 | static int do_page_crypto(const struct inode *inode, |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 150 | fscrypt_direction_t rw, u64 lblk_num, |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 151 | struct page *src_page, struct page *dest_page, |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 152 | unsigned int len, unsigned int offs, |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 153 | gfp_t gfp_flags) |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 154 | { |
Eric Biggers | fb44543 | 2016-10-12 23:30:16 -0400 | [diff] [blame] | 155 | struct { |
| 156 | __le64 index; |
| 157 | u8 padding[FS_XTS_TWEAK_SIZE - sizeof(__le64)]; |
| 158 | } xts_tweak; |
Linus Torvalds | d407574 | 2016-03-21 11:03:02 -0700 | [diff] [blame] | 159 | struct skcipher_request *req = NULL; |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 160 | DECLARE_FS_COMPLETION_RESULT(ecr); |
| 161 | struct scatterlist dst, src; |
| 162 | struct fscrypt_info *ci = inode->i_crypt_info; |
Linus Torvalds | d407574 | 2016-03-21 11:03:02 -0700 | [diff] [blame] | 163 | struct crypto_skcipher *tfm = ci->ci_ctfm; |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 164 | int res = 0; |
| 165 | |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 166 | BUG_ON(len == 0); |
| 167 | |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 168 | req = skcipher_request_alloc(tfm, gfp_flags); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 169 | if (!req) { |
| 170 | printk_ratelimited(KERN_ERR |
| 171 | "%s: crypto_request_alloc() failed\n", |
| 172 | __func__); |
| 173 | return -ENOMEM; |
| 174 | } |
| 175 | |
Linus Torvalds | d407574 | 2016-03-21 11:03:02 -0700 | [diff] [blame] | 176 | skcipher_request_set_callback( |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 177 | req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, |
Eric Biggers | 53fd755 | 2016-09-15 16:51:01 -0400 | [diff] [blame] | 178 | page_crypt_complete, &ecr); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 179 | |
Eric Biggers | fb44543 | 2016-10-12 23:30:16 -0400 | [diff] [blame] | 180 | BUILD_BUG_ON(sizeof(xts_tweak) != FS_XTS_TWEAK_SIZE); |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 181 | xts_tweak.index = cpu_to_le64(lblk_num); |
Eric Biggers | fb44543 | 2016-10-12 23:30:16 -0400 | [diff] [blame] | 182 | memset(xts_tweak.padding, 0, sizeof(xts_tweak.padding)); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 183 | |
| 184 | sg_init_table(&dst, 1); |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 185 | sg_set_page(&dst, dest_page, len, offs); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 186 | sg_init_table(&src, 1); |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 187 | sg_set_page(&src, src_page, len, offs); |
| 188 | skcipher_request_set_crypt(req, &src, &dst, len, &xts_tweak); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 189 | if (rw == FS_DECRYPT) |
Linus Torvalds | d407574 | 2016-03-21 11:03:02 -0700 | [diff] [blame] | 190 | res = crypto_skcipher_decrypt(req); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 191 | else |
Linus Torvalds | d407574 | 2016-03-21 11:03:02 -0700 | [diff] [blame] | 192 | res = crypto_skcipher_encrypt(req); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 193 | if (res == -EINPROGRESS || res == -EBUSY) { |
| 194 | BUG_ON(req->base.data != &ecr); |
| 195 | wait_for_completion(&ecr.completion); |
| 196 | res = ecr.res; |
| 197 | } |
Linus Torvalds | d407574 | 2016-03-21 11:03:02 -0700 | [diff] [blame] | 198 | skcipher_request_free(req); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 199 | if (res) { |
| 200 | printk_ratelimited(KERN_ERR |
Linus Torvalds | d407574 | 2016-03-21 11:03:02 -0700 | [diff] [blame] | 201 | "%s: crypto_skcipher_encrypt() returned %d\n", |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 202 | __func__, res); |
| 203 | return res; |
| 204 | } |
| 205 | return 0; |
| 206 | } |
| 207 | |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 208 | static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags) |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 209 | { |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 210 | ctx->w.bounce_page = mempool_alloc(fscrypt_bounce_page_pool, gfp_flags); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 211 | if (ctx->w.bounce_page == NULL) |
| 212 | return ERR_PTR(-ENOMEM); |
David Gstir | 6a34e4d | 2016-12-06 23:53:58 +0100 | [diff] [blame] | 213 | ctx->flags |= FS_CTX_HAS_BOUNCE_BUFFER_FL; |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 214 | return ctx->w.bounce_page; |
| 215 | } |
| 216 | |
| 217 | /** |
| 218 | * fscypt_encrypt_page() - Encrypts a page |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 219 | * @inode: The inode for which the encryption should take place |
| 220 | * @page: The page to encrypt. Must be locked for bounce-page |
| 221 | * encryption. |
| 222 | * @len: Length of data to encrypt in @page and encrypted |
| 223 | * data in returned page. |
| 224 | * @offs: Offset of data within @page and returned |
| 225 | * page holding encrypted data. |
| 226 | * @lblk_num: Logical block number. This must be unique for multiple |
| 227 | * calls with same inode, except when overwriting |
| 228 | * previously written data. |
| 229 | * @gfp_flags: The gfp flag for memory allocation |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 230 | * |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 231 | * Encrypts @page using the ctx encryption context. Performs encryption |
| 232 | * either in-place or into a newly allocated bounce page. |
| 233 | * Called on the page write path. |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 234 | * |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 235 | * Bounce page allocation is the default. |
| 236 | * In this case, the contents of @page are encrypted and stored in an |
| 237 | * allocated bounce page. @page has to be locked and the caller must call |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 238 | * fscrypt_restore_control_page() on the returned ciphertext page to |
| 239 | * release the bounce buffer and the encryption context. |
| 240 | * |
David Gstir | bd7b829 | 2016-12-06 23:53:56 +0100 | [diff] [blame] | 241 | * In-place encryption is used by setting the FS_CFLG_OWN_PAGES flag in |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 242 | * fscrypt_operations. Here, the input-page is returned with its content |
| 243 | * encrypted. |
| 244 | * |
| 245 | * Return: A page with the encrypted content on success. Else, an |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 246 | * error value or NULL. |
| 247 | */ |
David Gstir | 0b93e1b | 2016-11-13 22:20:47 +0100 | [diff] [blame] | 248 | struct page *fscrypt_encrypt_page(const struct inode *inode, |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 249 | struct page *page, |
| 250 | unsigned int len, |
| 251 | unsigned int offs, |
| 252 | u64 lblk_num, gfp_t gfp_flags) |
David Gstir | 7821d4d | 2016-11-13 22:20:46 +0100 | [diff] [blame] | 253 | |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 254 | { |
| 255 | struct fscrypt_ctx *ctx; |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 256 | struct page *ciphertext_page = page; |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 257 | int err; |
| 258 | |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 259 | BUG_ON(len % FS_CRYPTO_BLOCK_SIZE != 0); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 260 | |
David Gstir | bd7b829 | 2016-12-06 23:53:56 +0100 | [diff] [blame] | 261 | if (inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES) { |
David Gstir | 9e53277 | 2016-12-06 23:53:54 +0100 | [diff] [blame] | 262 | /* with inplace-encryption we just encrypt the page */ |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 263 | err = do_page_crypto(inode, FS_ENCRYPT, lblk_num, |
| 264 | page, ciphertext_page, |
| 265 | len, offs, gfp_flags); |
David Gstir | 9e53277 | 2016-12-06 23:53:54 +0100 | [diff] [blame] | 266 | if (err) |
| 267 | return ERR_PTR(err); |
| 268 | |
| 269 | return ciphertext_page; |
| 270 | } |
| 271 | |
David Gstir | bd7b829 | 2016-12-06 23:53:56 +0100 | [diff] [blame] | 272 | BUG_ON(!PageLocked(page)); |
| 273 | |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 274 | ctx = fscrypt_get_ctx(inode, gfp_flags); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 275 | if (IS_ERR(ctx)) |
| 276 | return (struct page *)ctx; |
| 277 | |
David Gstir | 9e53277 | 2016-12-06 23:53:54 +0100 | [diff] [blame] | 278 | /* The encryption operation will require a bounce page. */ |
| 279 | ciphertext_page = alloc_bounce_page(ctx, gfp_flags); |
| 280 | if (IS_ERR(ciphertext_page)) |
| 281 | goto errout; |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 282 | |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 283 | ctx->w.control_page = page; |
| 284 | err = do_page_crypto(inode, FS_ENCRYPT, lblk_num, |
| 285 | page, ciphertext_page, |
| 286 | len, offs, gfp_flags); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 287 | if (err) { |
| 288 | ciphertext_page = ERR_PTR(err); |
| 289 | goto errout; |
| 290 | } |
David Gstir | 9e53277 | 2016-12-06 23:53:54 +0100 | [diff] [blame] | 291 | SetPagePrivate(ciphertext_page); |
| 292 | set_page_private(ciphertext_page, (unsigned long)ctx); |
| 293 | lock_page(ciphertext_page); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 294 | return ciphertext_page; |
| 295 | |
| 296 | errout: |
| 297 | fscrypt_release_ctx(ctx); |
| 298 | return ciphertext_page; |
| 299 | } |
| 300 | EXPORT_SYMBOL(fscrypt_encrypt_page); |
| 301 | |
| 302 | /** |
David Gstir | 7821d4d | 2016-11-13 22:20:46 +0100 | [diff] [blame] | 303 | * fscrypt_decrypt_page() - Decrypts a page in-place |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 304 | * @inode: The corresponding inode for the page to decrypt. |
| 305 | * @page: The page to decrypt. Must be locked in case |
David Gstir | bd7b829 | 2016-12-06 23:53:56 +0100 | [diff] [blame] | 306 | * it is a writeback page (FS_CFLG_OWN_PAGES unset). |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 307 | * @len: Number of bytes in @page to be decrypted. |
| 308 | * @offs: Start of data in @page. |
| 309 | * @lblk_num: Logical block number. |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 310 | * |
| 311 | * Decrypts page in-place using the ctx encryption context. |
| 312 | * |
| 313 | * Called from the read completion callback. |
| 314 | * |
| 315 | * Return: Zero on success, non-zero otherwise. |
| 316 | */ |
David Gstir | 0b93e1b | 2016-11-13 22:20:47 +0100 | [diff] [blame] | 317 | int fscrypt_decrypt_page(const struct inode *inode, struct page *page, |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 318 | unsigned int len, unsigned int offs, u64 lblk_num) |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 319 | { |
David Gstir | bd7b829 | 2016-12-06 23:53:56 +0100 | [diff] [blame] | 320 | if (!(inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES)) |
| 321 | BUG_ON(!PageLocked(page)); |
| 322 | |
David Gstir | 1400451 | 2016-12-06 23:53:55 +0100 | [diff] [blame] | 323 | return do_page_crypto(inode, FS_DECRYPT, lblk_num, page, page, len, |
| 324 | offs, GFP_NOFS); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 325 | } |
| 326 | EXPORT_SYMBOL(fscrypt_decrypt_page); |
| 327 | |
David Gstir | 0b93e1b | 2016-11-13 22:20:47 +0100 | [diff] [blame] | 328 | int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 329 | sector_t pblk, unsigned int len) |
| 330 | { |
| 331 | struct fscrypt_ctx *ctx; |
| 332 | struct page *ciphertext_page = NULL; |
| 333 | struct bio *bio; |
| 334 | int ret, err = 0; |
| 335 | |
Kirill A. Shutemov | 09cbfea | 2016-04-01 15:29:47 +0300 | [diff] [blame] | 336 | BUG_ON(inode->i_sb->s_blocksize != PAGE_SIZE); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 337 | |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 338 | ctx = fscrypt_get_ctx(inode, GFP_NOFS); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 339 | if (IS_ERR(ctx)) |
| 340 | return PTR_ERR(ctx); |
| 341 | |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 342 | ciphertext_page = alloc_bounce_page(ctx, GFP_NOWAIT); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 343 | if (IS_ERR(ciphertext_page)) { |
| 344 | err = PTR_ERR(ciphertext_page); |
| 345 | goto errout; |
| 346 | } |
| 347 | |
| 348 | while (len--) { |
| 349 | err = do_page_crypto(inode, FS_ENCRYPT, lblk, |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 350 | ZERO_PAGE(0), ciphertext_page, |
David Gstir | 7821d4d | 2016-11-13 22:20:46 +0100 | [diff] [blame] | 351 | PAGE_SIZE, 0, GFP_NOFS); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 352 | if (err) |
| 353 | goto errout; |
| 354 | |
Jaegeuk Kim | b32e4482 | 2016-04-11 15:51:57 -0700 | [diff] [blame] | 355 | bio = bio_alloc(GFP_NOWAIT, 1); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 356 | if (!bio) { |
| 357 | err = -ENOMEM; |
| 358 | goto errout; |
| 359 | } |
| 360 | bio->bi_bdev = inode->i_sb->s_bdev; |
| 361 | bio->bi_iter.bi_sector = |
| 362 | pblk << (inode->i_sb->s_blocksize_bits - 9); |
Mike Christie | 95fe6c1 | 2016-06-05 14:31:48 -0500 | [diff] [blame] | 363 | bio_set_op_attrs(bio, REQ_OP_WRITE, 0); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 364 | ret = bio_add_page(bio, ciphertext_page, |
| 365 | inode->i_sb->s_blocksize, 0); |
| 366 | if (ret != inode->i_sb->s_blocksize) { |
| 367 | /* should never happen! */ |
| 368 | WARN_ON(1); |
| 369 | bio_put(bio); |
| 370 | err = -EIO; |
| 371 | goto errout; |
| 372 | } |
Mike Christie | 4e49ea4 | 2016-06-05 14:31:41 -0500 | [diff] [blame] | 373 | err = submit_bio_wait(bio); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 374 | if ((err == 0) && bio->bi_error) |
| 375 | err = -EIO; |
| 376 | bio_put(bio); |
| 377 | if (err) |
| 378 | goto errout; |
| 379 | lblk++; |
| 380 | pblk++; |
| 381 | } |
| 382 | err = 0; |
| 383 | errout: |
| 384 | fscrypt_release_ctx(ctx); |
| 385 | return err; |
| 386 | } |
| 387 | EXPORT_SYMBOL(fscrypt_zeroout_range); |
| 388 | |
| 389 | /* |
| 390 | * Validate dentries for encrypted directories to make sure we aren't |
| 391 | * potentially caching stale data after a key has been added or |
| 392 | * removed. |
| 393 | */ |
| 394 | static int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags) |
| 395 | { |
Jaegeuk Kim | d7d7535 | 2016-04-11 15:10:11 -0700 | [diff] [blame] | 396 | struct dentry *dir; |
| 397 | struct fscrypt_info *ci; |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 398 | int dir_has_key, cached_with_key; |
| 399 | |
Jaegeuk Kim | 03a8bb0 | 2016-04-12 16:05:36 -0700 | [diff] [blame] | 400 | if (flags & LOOKUP_RCU) |
| 401 | return -ECHILD; |
| 402 | |
Jaegeuk Kim | d7d7535 | 2016-04-11 15:10:11 -0700 | [diff] [blame] | 403 | dir = dget_parent(dentry); |
| 404 | if (!d_inode(dir)->i_sb->s_cop->is_encrypted(d_inode(dir))) { |
| 405 | dput(dir); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 406 | return 0; |
Jaegeuk Kim | d7d7535 | 2016-04-11 15:10:11 -0700 | [diff] [blame] | 407 | } |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 408 | |
Jaegeuk Kim | d7d7535 | 2016-04-11 15:10:11 -0700 | [diff] [blame] | 409 | ci = d_inode(dir)->i_crypt_info; |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 410 | if (ci && ci->ci_keyring_key && |
| 411 | (ci->ci_keyring_key->flags & ((1 << KEY_FLAG_INVALIDATED) | |
| 412 | (1 << KEY_FLAG_REVOKED) | |
| 413 | (1 << KEY_FLAG_DEAD)))) |
| 414 | ci = NULL; |
| 415 | |
| 416 | /* this should eventually be an flag in d_flags */ |
| 417 | spin_lock(&dentry->d_lock); |
| 418 | cached_with_key = dentry->d_flags & DCACHE_ENCRYPTED_WITH_KEY; |
| 419 | spin_unlock(&dentry->d_lock); |
| 420 | dir_has_key = (ci != NULL); |
Jaegeuk Kim | d7d7535 | 2016-04-11 15:10:11 -0700 | [diff] [blame] | 421 | dput(dir); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 422 | |
| 423 | /* |
| 424 | * If the dentry was cached without the key, and it is a |
| 425 | * negative dentry, it might be a valid name. We can't check |
| 426 | * if the key has since been made available due to locking |
| 427 | * reasons, so we fail the validation so ext4_lookup() can do |
| 428 | * this check. |
| 429 | * |
| 430 | * We also fail the validation if the dentry was created with |
| 431 | * the key present, but we no longer have the key, or vice versa. |
| 432 | */ |
| 433 | if ((!cached_with_key && d_is_negative(dentry)) || |
| 434 | (!cached_with_key && dir_has_key) || |
| 435 | (cached_with_key && !dir_has_key)) |
| 436 | return 0; |
| 437 | return 1; |
| 438 | } |
| 439 | |
| 440 | const struct dentry_operations fscrypt_d_ops = { |
| 441 | .d_revalidate = fscrypt_d_revalidate, |
| 442 | }; |
| 443 | EXPORT_SYMBOL(fscrypt_d_ops); |
| 444 | |
| 445 | /* |
| 446 | * Call fscrypt_decrypt_page on every single page, reusing the encryption |
| 447 | * context. |
| 448 | */ |
| 449 | static void completion_pages(struct work_struct *work) |
| 450 | { |
| 451 | struct fscrypt_ctx *ctx = |
| 452 | container_of(work, struct fscrypt_ctx, r.work); |
| 453 | struct bio *bio = ctx->r.bio; |
| 454 | struct bio_vec *bv; |
| 455 | int i; |
| 456 | |
| 457 | bio_for_each_segment_all(bv, bio, i) { |
| 458 | struct page *page = bv->bv_page; |
David Gstir | 7821d4d | 2016-11-13 22:20:46 +0100 | [diff] [blame] | 459 | int ret = fscrypt_decrypt_page(page->mapping->host, page, |
David Gstir | 9c4bb8a | 2016-11-13 22:20:48 +0100 | [diff] [blame] | 460 | PAGE_SIZE, 0, page->index); |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 461 | |
| 462 | if (ret) { |
| 463 | WARN_ON_ONCE(1); |
| 464 | SetPageError(page); |
| 465 | } else { |
| 466 | SetPageUptodate(page); |
| 467 | } |
| 468 | unlock_page(page); |
| 469 | } |
| 470 | fscrypt_release_ctx(ctx); |
| 471 | bio_put(bio); |
| 472 | } |
| 473 | |
| 474 | void fscrypt_decrypt_bio_pages(struct fscrypt_ctx *ctx, struct bio *bio) |
| 475 | { |
| 476 | INIT_WORK(&ctx->r.work, completion_pages); |
| 477 | ctx->r.bio = bio; |
| 478 | queue_work(fscrypt_read_workqueue, &ctx->r.work); |
| 479 | } |
| 480 | EXPORT_SYMBOL(fscrypt_decrypt_bio_pages); |
| 481 | |
| 482 | void fscrypt_pullback_bio_page(struct page **page, bool restore) |
| 483 | { |
| 484 | struct fscrypt_ctx *ctx; |
| 485 | struct page *bounce_page; |
| 486 | |
| 487 | /* The bounce data pages are unmapped. */ |
| 488 | if ((*page)->mapping) |
| 489 | return; |
| 490 | |
| 491 | /* The bounce data page is unmapped. */ |
| 492 | bounce_page = *page; |
| 493 | ctx = (struct fscrypt_ctx *)page_private(bounce_page); |
| 494 | |
| 495 | /* restore control page */ |
| 496 | *page = ctx->w.control_page; |
| 497 | |
| 498 | if (restore) |
| 499 | fscrypt_restore_control_page(bounce_page); |
| 500 | } |
| 501 | EXPORT_SYMBOL(fscrypt_pullback_bio_page); |
| 502 | |
| 503 | void fscrypt_restore_control_page(struct page *page) |
| 504 | { |
| 505 | struct fscrypt_ctx *ctx; |
| 506 | |
| 507 | ctx = (struct fscrypt_ctx *)page_private(page); |
| 508 | set_page_private(page, (unsigned long)NULL); |
| 509 | ClearPagePrivate(page); |
| 510 | unlock_page(page); |
| 511 | fscrypt_release_ctx(ctx); |
| 512 | } |
| 513 | EXPORT_SYMBOL(fscrypt_restore_control_page); |
| 514 | |
| 515 | static void fscrypt_destroy(void) |
| 516 | { |
| 517 | struct fscrypt_ctx *pos, *n; |
| 518 | |
| 519 | list_for_each_entry_safe(pos, n, &fscrypt_free_ctxs, free_list) |
| 520 | kmem_cache_free(fscrypt_ctx_cachep, pos); |
| 521 | INIT_LIST_HEAD(&fscrypt_free_ctxs); |
| 522 | mempool_destroy(fscrypt_bounce_page_pool); |
| 523 | fscrypt_bounce_page_pool = NULL; |
| 524 | } |
| 525 | |
| 526 | /** |
| 527 | * fscrypt_initialize() - allocate major buffers for fs encryption. |
David Gstir | f32d7ac | 2016-12-06 23:53:57 +0100 | [diff] [blame] | 528 | * @cop_flags: fscrypt operations flags |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 529 | * |
| 530 | * We only call this when we start accessing encrypted files, since it |
| 531 | * results in memory getting allocated that wouldn't otherwise be used. |
| 532 | * |
| 533 | * Return: Zero on success, non-zero otherwise. |
| 534 | */ |
David Gstir | f32d7ac | 2016-12-06 23:53:57 +0100 | [diff] [blame] | 535 | int fscrypt_initialize(unsigned int cop_flags) |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 536 | { |
| 537 | int i, res = -ENOMEM; |
| 538 | |
David Gstir | f32d7ac | 2016-12-06 23:53:57 +0100 | [diff] [blame] | 539 | /* |
| 540 | * No need to allocate a bounce page pool if there already is one or |
| 541 | * this FS won't use it. |
| 542 | */ |
| 543 | if (cop_flags & FS_CFLG_OWN_PAGES || fscrypt_bounce_page_pool) |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 544 | return 0; |
| 545 | |
| 546 | mutex_lock(&fscrypt_init_mutex); |
| 547 | if (fscrypt_bounce_page_pool) |
| 548 | goto already_initialized; |
| 549 | |
| 550 | for (i = 0; i < num_prealloc_crypto_ctxs; i++) { |
| 551 | struct fscrypt_ctx *ctx; |
| 552 | |
| 553 | ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, GFP_NOFS); |
| 554 | if (!ctx) |
| 555 | goto fail; |
| 556 | list_add(&ctx->free_list, &fscrypt_free_ctxs); |
| 557 | } |
| 558 | |
| 559 | fscrypt_bounce_page_pool = |
| 560 | mempool_create_page_pool(num_prealloc_crypto_pages, 0); |
| 561 | if (!fscrypt_bounce_page_pool) |
| 562 | goto fail; |
| 563 | |
| 564 | already_initialized: |
| 565 | mutex_unlock(&fscrypt_init_mutex); |
| 566 | return 0; |
| 567 | fail: |
| 568 | fscrypt_destroy(); |
| 569 | mutex_unlock(&fscrypt_init_mutex); |
| 570 | return res; |
| 571 | } |
Jaegeuk Kim | 0b81d07 | 2015-05-15 16:26:10 -0700 | [diff] [blame] | 572 | |
| 573 | /** |
| 574 | * fscrypt_init() - Set up for fs encryption. |
| 575 | */ |
| 576 | static int __init fscrypt_init(void) |
| 577 | { |
| 578 | fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue", |
| 579 | WQ_HIGHPRI, 0); |
| 580 | if (!fscrypt_read_workqueue) |
| 581 | goto fail; |
| 582 | |
| 583 | fscrypt_ctx_cachep = KMEM_CACHE(fscrypt_ctx, SLAB_RECLAIM_ACCOUNT); |
| 584 | if (!fscrypt_ctx_cachep) |
| 585 | goto fail_free_queue; |
| 586 | |
| 587 | fscrypt_info_cachep = KMEM_CACHE(fscrypt_info, SLAB_RECLAIM_ACCOUNT); |
| 588 | if (!fscrypt_info_cachep) |
| 589 | goto fail_free_ctx; |
| 590 | |
| 591 | return 0; |
| 592 | |
| 593 | fail_free_ctx: |
| 594 | kmem_cache_destroy(fscrypt_ctx_cachep); |
| 595 | fail_free_queue: |
| 596 | destroy_workqueue(fscrypt_read_workqueue); |
| 597 | fail: |
| 598 | return -ENOMEM; |
| 599 | } |
| 600 | module_init(fscrypt_init) |
| 601 | |
| 602 | /** |
| 603 | * fscrypt_exit() - Shutdown the fs encryption system |
| 604 | */ |
| 605 | static void __exit fscrypt_exit(void) |
| 606 | { |
| 607 | fscrypt_destroy(); |
| 608 | |
| 609 | if (fscrypt_read_workqueue) |
| 610 | destroy_workqueue(fscrypt_read_workqueue); |
| 611 | kmem_cache_destroy(fscrypt_ctx_cachep); |
| 612 | kmem_cache_destroy(fscrypt_info_cachep); |
| 613 | } |
| 614 | module_exit(fscrypt_exit); |
| 615 | |
| 616 | MODULE_LICENSE("GPL"); |