dm crypt: move cipher data out of per_cpu struct
Move static dm-crypt cipher data out of per-cpu structure.
Cipher information is static, so it does not have to be in a per-cpu
structure.
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Alasdair G Kergon <agk@redhat.com>
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index d7f0e01..711e0ac5 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -109,9 +109,6 @@
*/
struct crypt_cpu {
struct ablkcipher_request *req;
- /* ESSIV: struct crypto_cipher *essiv_tfm */
- void *iv_private;
- struct crypto_ablkcipher *tfms[0];
};
/*
@@ -151,6 +148,10 @@
* per_cpu_ptr() only.
*/
struct crypt_cpu __percpu *cpu;
+
+ /* ESSIV: struct crypto_cipher *essiv_tfm */
+ void *iv_private;
+ struct crypto_ablkcipher **tfms;
unsigned tfms_count;
/*
@@ -193,7 +194,7 @@
*/
static struct crypto_ablkcipher *any_tfm(struct crypt_config *cc)
{
- return __this_cpu_ptr(cc->cpu)->tfms[0];
+ return cc->tfms[0];
}
/*
@@ -258,7 +259,7 @@
struct hash_desc desc;
struct scatterlist sg;
struct crypto_cipher *essiv_tfm;
- int err, cpu;
+ int err;
sg_init_one(&sg, cc->key, cc->key_size);
desc.tfm = essiv->hash_tfm;
@@ -268,14 +269,12 @@
if (err)
return err;
- for_each_possible_cpu(cpu) {
- essiv_tfm = per_cpu_ptr(cc->cpu, cpu)->iv_private,
+ essiv_tfm = cc->iv_private;
- err = crypto_cipher_setkey(essiv_tfm, essiv->salt,
- crypto_hash_digestsize(essiv->hash_tfm));
- if (err)
- return err;
- }
+ err = crypto_cipher_setkey(essiv_tfm, essiv->salt,
+ crypto_hash_digestsize(essiv->hash_tfm));
+ if (err)
+ return err;
return 0;
}
@@ -286,16 +285,14 @@
struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
unsigned salt_size = crypto_hash_digestsize(essiv->hash_tfm);
struct crypto_cipher *essiv_tfm;
- int cpu, r, err = 0;
+ int r, err = 0;
memset(essiv->salt, 0, salt_size);
- for_each_possible_cpu(cpu) {
- essiv_tfm = per_cpu_ptr(cc->cpu, cpu)->iv_private;
- r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size);
- if (r)
- err = r;
- }
+ essiv_tfm = cc->iv_private;
+ r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size);
+ if (r)
+ err = r;
return err;
}
@@ -335,8 +332,6 @@
static void crypt_iv_essiv_dtr(struct crypt_config *cc)
{
- int cpu;
- struct crypt_cpu *cpu_cc;
struct crypto_cipher *essiv_tfm;
struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
@@ -346,15 +341,12 @@
kzfree(essiv->salt);
essiv->salt = NULL;
- for_each_possible_cpu(cpu) {
- cpu_cc = per_cpu_ptr(cc->cpu, cpu);
- essiv_tfm = cpu_cc->iv_private;
+ essiv_tfm = cc->iv_private;
- if (essiv_tfm)
- crypto_free_cipher(essiv_tfm);
+ if (essiv_tfm)
+ crypto_free_cipher(essiv_tfm);
- cpu_cc->iv_private = NULL;
- }
+ cc->iv_private = NULL;
}
static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
@@ -363,7 +355,7 @@
struct crypto_cipher *essiv_tfm = NULL;
struct crypto_hash *hash_tfm = NULL;
u8 *salt = NULL;
- int err, cpu;
+ int err;
if (!opts) {
ti->error = "Digest algorithm missing for ESSIV mode";
@@ -388,15 +380,13 @@
cc->iv_gen_private.essiv.salt = salt;
cc->iv_gen_private.essiv.hash_tfm = hash_tfm;
- for_each_possible_cpu(cpu) {
- essiv_tfm = setup_essiv_cpu(cc, ti, salt,
- crypto_hash_digestsize(hash_tfm));
- if (IS_ERR(essiv_tfm)) {
- crypt_iv_essiv_dtr(cc);
- return PTR_ERR(essiv_tfm);
- }
- per_cpu_ptr(cc->cpu, cpu)->iv_private = essiv_tfm;
+ essiv_tfm = setup_essiv_cpu(cc, ti, salt,
+ crypto_hash_digestsize(hash_tfm));
+ if (IS_ERR(essiv_tfm)) {
+ crypt_iv_essiv_dtr(cc);
+ return PTR_ERR(essiv_tfm);
}
+ cc->iv_private = essiv_tfm;
return 0;
@@ -410,7 +400,7 @@
static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
struct dm_crypt_request *dmreq)
{
- struct crypto_cipher *essiv_tfm = this_crypt_config(cc)->iv_private;
+ struct crypto_cipher *essiv_tfm = cc->iv_private;
memset(iv, 0, cc->iv_size);
*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
@@ -754,7 +744,7 @@
if (!this_cc->req)
this_cc->req = mempool_alloc(cc->req_pool, GFP_NOIO);
- ablkcipher_request_set_tfm(this_cc->req, this_cc->tfms[key_index]);
+ ablkcipher_request_set_tfm(this_cc->req, cc->tfms[key_index]);
ablkcipher_request_set_callback(this_cc->req,
CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
kcryptd_async_done, dmreq_of_req(cc, this_cc->req));
@@ -1273,29 +1263,38 @@
}
}
-static void crypt_free_tfms(struct crypt_config *cc, int cpu)
+static void crypt_free_tfms(struct crypt_config *cc)
{
- struct crypt_cpu *cpu_cc = per_cpu_ptr(cc->cpu, cpu);
unsigned i;
+ if (!cc->tfms)
+ return;
+
for (i = 0; i < cc->tfms_count; i++)
- if (cpu_cc->tfms[i] && !IS_ERR(cpu_cc->tfms[i])) {
- crypto_free_ablkcipher(cpu_cc->tfms[i]);
- cpu_cc->tfms[i] = NULL;
+ if (cc->tfms[i] && !IS_ERR(cc->tfms[i])) {
+ crypto_free_ablkcipher(cc->tfms[i]);
+ cc->tfms[i] = NULL;
}
+
+ kfree(cc->tfms);
+ cc->tfms = NULL;
}
-static int crypt_alloc_tfms(struct crypt_config *cc, int cpu, char *ciphermode)
+static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
{
- struct crypt_cpu *cpu_cc = per_cpu_ptr(cc->cpu, cpu);
unsigned i;
int err;
+ cc->tfms = kmalloc(cc->tfms_count * sizeof(struct crypto_ablkcipher *),
+ GFP_KERNEL);
+ if (!cc->tfms)
+ return -ENOMEM;
+
for (i = 0; i < cc->tfms_count; i++) {
- cpu_cc->tfms[i] = crypto_alloc_ablkcipher(ciphermode, 0, 0);
- if (IS_ERR(cpu_cc->tfms[i])) {
- err = PTR_ERR(cpu_cc->tfms[i]);
- crypt_free_tfms(cc, cpu);
+ cc->tfms[i] = crypto_alloc_ablkcipher(ciphermode, 0, 0);
+ if (IS_ERR(cc->tfms[i])) {
+ err = PTR_ERR(cc->tfms[i]);
+ crypt_free_tfms(cc);
return err;
}
}
@@ -1306,15 +1305,14 @@
static int crypt_setkey_allcpus(struct crypt_config *cc)
{
unsigned subkey_size = cc->key_size >> ilog2(cc->tfms_count);
- int cpu, err = 0, i, r;
+ int err = 0, i, r;
- for_each_possible_cpu(cpu) {
- for (i = 0; i < cc->tfms_count; i++) {
- r = crypto_ablkcipher_setkey(per_cpu_ptr(cc->cpu, cpu)->tfms[i],
- cc->key + (i * subkey_size), subkey_size);
- if (r)
- err = r;
- }
+ for (i = 0; i < cc->tfms_count; i++) {
+ r = crypto_ablkcipher_setkey(cc->tfms[i],
+ cc->key + (i * subkey_size),
+ subkey_size);
+ if (r)
+ err = r;
}
return err;
@@ -1376,9 +1374,10 @@
cpu_cc = per_cpu_ptr(cc->cpu, cpu);
if (cpu_cc->req)
mempool_free(cpu_cc->req, cc->req_pool);
- crypt_free_tfms(cc, cpu);
}
+ crypt_free_tfms(cc);
+
if (cc->bs)
bioset_free(cc->bs);
@@ -1411,7 +1410,7 @@
struct crypt_config *cc = ti->private;
char *tmp, *cipher, *chainmode, *ivmode, *ivopts, *keycount;
char *cipher_api = NULL;
- int cpu, ret = -EINVAL;
+ int ret = -EINVAL;
char dummy;
/* Convert to crypto api definition? */
@@ -1452,8 +1451,7 @@
if (tmp)
DMWARN("Ignoring unexpected additional cipher options");
- cc->cpu = __alloc_percpu(sizeof(*(cc->cpu)) +
- cc->tfms_count * sizeof(*(cc->cpu->tfms)),
+ cc->cpu = __alloc_percpu(sizeof(*(cc->cpu)),
__alignof__(struct crypt_cpu));
if (!cc->cpu) {
ti->error = "Cannot allocate per cpu state";
@@ -1486,12 +1484,10 @@
}
/* Allocate cipher */
- for_each_possible_cpu(cpu) {
- ret = crypt_alloc_tfms(cc, cpu, cipher_api);
- if (ret < 0) {
- ti->error = "Error allocating crypto tfm";
- goto bad;
- }
+ ret = crypt_alloc_tfms(cc, cipher_api);
+ if (ret < 0) {
+ ti->error = "Error allocating crypto tfm";
+ goto bad;
}
/* Initialize and set key */