diff --git a/crypto/algapi.c b/crypto/algapi.c index acea250677c..36c4f1bdb52 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -10,6 +10,7 @@ * */ +#include #include #include #include @@ -73,27 +74,96 @@ static int crypto_check_alg(struct crypto_alg *alg) return crypto_set_driver_name(alg); } -static int __crypto_register_alg(struct crypto_alg *alg) +static void crypto_destroy_instance(struct crypto_alg *alg) +{ + struct crypto_instance *inst = (void *)alg; + struct crypto_template *tmpl = inst->tmpl; + + tmpl->free(inst); + crypto_tmpl_put(tmpl); +} + +static void crypto_remove_spawns(struct list_head *spawns, + struct list_head *list) +{ + struct crypto_spawn *spawn, *n; + + list_for_each_entry_safe(spawn, n, spawns, list) { + struct crypto_instance *inst = spawn->inst; + struct crypto_template *tmpl = inst->tmpl; + + list_del_init(&spawn->list); + spawn->alg = NULL; + + if (crypto_is_dead(&inst->alg)) + continue; + + inst->alg.cra_flags |= CRYPTO_ALG_DEAD; + if (!tmpl || !crypto_tmpl_get(tmpl)) + continue; + + crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, &inst->alg); + list_move(&inst->alg.cra_list, list); + hlist_del(&inst->list); + inst->alg.cra_destroy = crypto_destroy_instance; + + if (!list_empty(&inst->alg.cra_users)) { + if (&n->list == spawns) + n = list_entry(inst->alg.cra_users.next, + typeof(*n), list); + __list_splice(&inst->alg.cra_users, spawns->prev); + } + } +} + +static int __crypto_register_alg(struct crypto_alg *alg, + struct list_head *list) { struct crypto_alg *q; - int ret = -EEXIST; + int ret = -EAGAIN; + + if (crypto_is_dead(alg)) + goto out; + + INIT_LIST_HEAD(&alg->cra_users); + + ret = -EEXIST; atomic_set(&alg->cra_refcnt, 1); list_for_each_entry(q, &crypto_alg_list, cra_list) { if (q == alg) goto out; - if (crypto_is_larval(q) && - (!strcmp(alg->cra_name, q->cra_name) || - !strcmp(alg->cra_driver_name, q->cra_name))) { + + if (crypto_is_moribund(q)) + continue; + + if (crypto_is_larval(q)) { struct crypto_larval *larval = (void *)q; + if (strcmp(alg->cra_name, q->cra_name) && + strcmp(alg->cra_driver_name, q->cra_name)) + continue; + + if (larval->adult) + continue; if ((q->cra_flags ^ alg->cra_flags) & larval->mask) continue; if (!crypto_mod_get(alg)) continue; + larval->adult = alg; complete(&larval->completion); + continue; } + + if (strcmp(alg->cra_name, q->cra_name)) + continue; + + if (strcmp(alg->cra_driver_name, q->cra_driver_name) && + q->cra_priority > alg->cra_priority) + continue; + + crypto_remove_spawns(&q->cra_users, list); } list_add(&alg->cra_list, &crypto_alg_list); @@ -105,8 +175,20 @@ out: return ret; } +static void crypto_remove_final(struct list_head *list) +{ + struct crypto_alg *alg; + struct crypto_alg *n; + + list_for_each_entry_safe(alg, n, list, cra_list) { + list_del_init(&alg->cra_list); + crypto_alg_put(alg); + } +} + int crypto_register_alg(struct crypto_alg *alg) { + LIST_HEAD(list); int err; err = crypto_check_alg(alg); @@ -114,23 +196,35 @@ int crypto_register_alg(struct crypto_alg *alg) return err; down_write(&crypto_alg_sem); - err = __crypto_register_alg(alg); + err = __crypto_register_alg(alg, &list); up_write(&crypto_alg_sem); + crypto_remove_final(&list); return err; } EXPORT_SYMBOL_GPL(crypto_register_alg); +static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list) +{ + if (unlikely(list_empty(&alg->cra_list))) + return -ENOENT; + + alg->cra_flags |= CRYPTO_ALG_DEAD; + + crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, alg); + list_del_init(&alg->cra_list); + crypto_remove_spawns(&alg->cra_users, list); + + return 0; +} + int crypto_unregister_alg(struct crypto_alg *alg) { - int ret = -ENOENT; + int ret; + LIST_HEAD(list); down_write(&crypto_alg_sem); - if (likely(!list_empty(&alg->cra_list))) { - list_del_init(&alg->cra_list); - ret = 0; - } - crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, alg); + ret = crypto_remove_alg(alg, &list); up_write(&crypto_alg_sem); if (ret) @@ -140,6 +234,7 @@ int crypto_unregister_alg(struct crypto_alg *alg) if (alg->cra_destroy) alg->cra_destroy(alg); + crypto_remove_final(&list); return 0; } EXPORT_SYMBOL_GPL(crypto_unregister_alg); @@ -170,6 +265,7 @@ void crypto_unregister_template(struct crypto_template *tmpl) struct crypto_instance *inst; struct hlist_node *p, *n; struct hlist_head *list; + LIST_HEAD(users); down_write(&crypto_alg_sem); @@ -178,9 +274,8 @@ void crypto_unregister_template(struct crypto_template *tmpl) list = &tmpl->instances; hlist_for_each_entry(inst, p, list, list) { - BUG_ON(list_empty(&inst->alg.cra_list)); - list_del_init(&inst->alg.cra_list); - crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, &inst->alg); + int err = crypto_remove_alg(&inst->alg, &users); + BUG_ON(err); } crypto_notify(CRYPTO_MSG_TMPL_UNREGISTER, tmpl); @@ -191,6 +286,7 @@ void crypto_unregister_template(struct crypto_template *tmpl) BUG_ON(atomic_read(&inst->alg.cra_refcnt) != 1); tmpl->free(inst); } + crypto_remove_final(&users); } EXPORT_SYMBOL_GPL(crypto_unregister_template); @@ -222,6 +318,7 @@ EXPORT_SYMBOL_GPL(crypto_lookup_template); int crypto_register_instance(struct crypto_template *tmpl, struct crypto_instance *inst) { + LIST_HEAD(list); int err = -EINVAL; if (inst->alg.cra_destroy) @@ -235,7 +332,7 @@ int crypto_register_instance(struct crypto_template *tmpl, down_write(&crypto_alg_sem); - err = __crypto_register_alg(&inst->alg); + err = __crypto_register_alg(&inst->alg, &list); if (err) goto unlock; @@ -245,11 +342,67 @@ int crypto_register_instance(struct crypto_template *tmpl, unlock: up_write(&crypto_alg_sem); + crypto_remove_final(&list); + err: return err; } EXPORT_SYMBOL_GPL(crypto_register_instance); +int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, + struct crypto_instance *inst) +{ + int err = -EAGAIN; + + spawn->inst = inst; + + down_write(&crypto_alg_sem); + if (!crypto_is_moribund(alg)) { + list_add(&spawn->list, &alg->cra_users); + spawn->alg = alg; + err = 0; + } + up_write(&crypto_alg_sem); + + return err; +} +EXPORT_SYMBOL_GPL(crypto_init_spawn); + +void crypto_drop_spawn(struct crypto_spawn *spawn) +{ + down_write(&crypto_alg_sem); + list_del(&spawn->list); + up_write(&crypto_alg_sem); +} +EXPORT_SYMBOL_GPL(crypto_drop_spawn); + +struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn) +{ + struct crypto_alg *alg; + struct crypto_alg *alg2; + struct crypto_tfm *tfm; + + down_read(&crypto_alg_sem); + alg = spawn->alg; + alg2 = alg; + if (alg2) + alg2 = crypto_mod_get(alg2); + up_read(&crypto_alg_sem); + + if (!alg2) { + if (alg) + crypto_shoot_alg(alg); + return ERR_PTR(-EAGAIN); + } + + tfm = __crypto_alloc_tfm(alg, 0); + if (IS_ERR(tfm)) + crypto_mod_put(alg); + + return tfm; +} +EXPORT_SYMBOL_GPL(crypto_spawn_tfm); + int crypto_register_notifier(struct notifier_block *nb) { return blocking_notifier_chain_register(&crypto_chain, nb); diff --git a/crypto/api.c b/crypto/api.c index ddf6a767acd..7e5522cf856 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -15,11 +15,13 @@ * */ +#include #include #include #include #include #include +#include #include #include #include "internal.h" @@ -38,12 +40,6 @@ static inline struct crypto_alg *crypto_alg_get(struct crypto_alg *alg) return alg; } -static inline void crypto_alg_put(struct crypto_alg *alg) -{ - if (atomic_dec_and_test(&alg->cra_refcnt) && alg->cra_destroy) - alg->cra_destroy(alg); -} - struct crypto_alg *crypto_mod_get(struct crypto_alg *alg) { return try_module_get(alg->cra_module) ? crypto_alg_get(alg) : NULL; @@ -65,6 +61,9 @@ struct crypto_alg *__crypto_alg_lookup(const char *name, u32 type, u32 mask) list_for_each_entry(q, &crypto_alg_list, cra_list) { int exact, fuzzy; + if (crypto_is_moribund(q)) + continue; + if ((q->cra_flags ^ type) & mask) continue; @@ -111,7 +110,7 @@ static struct crypto_alg *crypto_larval_alloc(const char *name, u32 type, larval = kzalloc(sizeof(*larval), GFP_KERNEL); if (!larval) - return NULL; + return ERR_PTR(-ENOMEM); larval->mask = mask; larval->alg.cra_flags = CRYPTO_ALG_LARVAL | type; @@ -153,8 +152,11 @@ static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg) wait_for_completion_interruptible_timeout(&larval->completion, 60 * HZ); alg = larval->adult; - if (alg && !crypto_mod_get(alg)) - alg = NULL; + if (alg) { + if (!crypto_mod_get(alg)) + alg = ERR_PTR(-EAGAIN); + } else + alg = ERR_PTR(-ENOENT); crypto_mod_put(&larval->alg); return alg; @@ -165,9 +167,6 @@ static struct crypto_alg *crypto_alg_lookup(const char *name, u32 type, { struct crypto_alg *alg; - if (!name) - return NULL; - down_read(&crypto_alg_sem); alg = __crypto_alg_lookup(name, type, mask); up_read(&crypto_alg_sem); @@ -181,7 +180,10 @@ struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask) struct crypto_alg *larval; int ok; - mask &= ~CRYPTO_ALG_LARVAL; + if (!name) + return ERR_PTR(-ENOENT); + + mask &= ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD); type &= mask; alg = try_then_request_module(crypto_alg_lookup(name, type, mask), @@ -190,7 +192,7 @@ struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask) return crypto_is_larval(alg) ? crypto_larval_wait(alg) : alg; larval = crypto_larval_alloc(name, type, mask); - if (!larval || !crypto_is_larval(larval)) + if (IS_ERR(larval) || !crypto_is_larval(larval)) return larval; ok = crypto_notify(CRYPTO_MSG_ALG_REQUEST, larval); @@ -203,7 +205,7 @@ struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask) alg = crypto_larval_wait(larval); else { crypto_mod_put(larval); - alg = NULL; + alg = ERR_PTR(-ENOENT); } crypto_larval_kill(larval); return alg; @@ -298,31 +300,40 @@ static unsigned int crypto_ctxsize(struct crypto_alg *alg, int flags) return len + (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1)); } -struct crypto_tfm *crypto_alloc_tfm(const char *name, u32 flags) +void crypto_shoot_alg(struct crypto_alg *alg) +{ + down_write(&crypto_alg_sem); + alg->cra_flags |= CRYPTO_ALG_DYING; + up_write(&crypto_alg_sem); +} +EXPORT_SYMBOL_GPL(crypto_shoot_alg); + +struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 flags) { struct crypto_tfm *tfm = NULL; - struct crypto_alg *alg; unsigned int tfm_size; - - alg = crypto_alg_mod_lookup(name, 0, 0); - if (alg == NULL) - goto out; + int err = -ENOMEM; tfm_size = sizeof(*tfm) + crypto_ctxsize(alg, flags); tfm = kzalloc(tfm_size, GFP_KERNEL); if (tfm == NULL) - goto out_put; + goto out; tfm->__crt_alg = alg; - - if (crypto_init_flags(tfm, flags)) + + err = crypto_init_flags(tfm, flags); + if (err) goto out_free_tfm; - if (crypto_init_ops(tfm)) + err = crypto_init_ops(tfm); + if (err) goto out_free_tfm; - if (alg->cra_init && alg->cra_init(tfm)) + if (alg->cra_init && (err = alg->cra_init(tfm))) { + if (err == -EAGAIN) + crypto_shoot_alg(alg); goto cra_init_failed; + } goto out; @@ -330,12 +341,36 @@ cra_init_failed: crypto_exit_ops(tfm); out_free_tfm: kfree(tfm); - tfm = NULL; -out_put: - crypto_mod_put(alg); + tfm = ERR_PTR(err); out: return tfm; } +EXPORT_SYMBOL_GPL(__crypto_alloc_tfm); + +struct crypto_tfm *crypto_alloc_tfm(const char *name, u32 flags) +{ + struct crypto_tfm *tfm = NULL; + int err; + + do { + struct crypto_alg *alg; + + alg = crypto_alg_mod_lookup(name, 0, 0); + err = PTR_ERR(alg); + if (IS_ERR(alg)) + continue; + + tfm = __crypto_alloc_tfm(alg, flags); + err = 0; + if (IS_ERR(tfm)) { + crypto_mod_put(alg); + err = PTR_ERR(tfm); + tfm = NULL; + } + } while (err == -EAGAIN && !signal_pending(current)); + + return tfm; +} void crypto_free_tfm(struct crypto_tfm *tfm) { @@ -361,7 +396,7 @@ int crypto_alg_available(const char *name, u32 flags) int ret = 0; struct crypto_alg *alg = crypto_alg_mod_lookup(name, 0, 0); - if (alg) { + if (!IS_ERR(alg)) { crypto_mod_put(alg); ret = 1; } diff --git a/crypto/cryptomgr.c b/crypto/cryptomgr.c index ae54942e3b3..9b5b1560106 100644 --- a/crypto/cryptomgr.c +++ b/crypto/cryptomgr.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -44,21 +45,25 @@ static void cryptomgr_probe(void *data) struct cryptomgr_param *param = data; struct crypto_template *tmpl; struct crypto_instance *inst; + int err; tmpl = crypto_lookup_template(param->template); if (!tmpl) goto err; - inst = tmpl->alloc(¶m->alg, sizeof(param->alg)); - if (IS_ERR(inst)) - goto err; - else if ((err = crypto_register_instance(tmpl, inst))) { - tmpl->free(inst); - goto err; - } + do { + inst = tmpl->alloc(¶m->alg, sizeof(param->alg)); + if (IS_ERR(inst)) + err = PTR_ERR(inst); + else if ((err = crypto_register_instance(tmpl, inst))) + tmpl->free(inst); + } while (err == -EAGAIN && !signal_pending(current)); crypto_tmpl_put(tmpl); + if (err) + goto err; + out: kfree(param); return; diff --git a/crypto/internal.h b/crypto/internal.h index c08d93bdadc..03c00b0e6b6 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -142,12 +142,21 @@ void crypto_exit_compress_ops(struct crypto_tfm *tfm); void crypto_larval_error(const char *name, u32 type, u32 mask); +void crypto_shoot_alg(struct crypto_alg *alg); +struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 flags); + int crypto_register_instance(struct crypto_template *tmpl, struct crypto_instance *inst); int crypto_register_notifier(struct notifier_block *nb); int crypto_unregister_notifier(struct notifier_block *nb); +static inline void crypto_alg_put(struct crypto_alg *alg) +{ + if (atomic_dec_and_test(&alg->cra_refcnt) && alg->cra_destroy) + alg->cra_destroy(alg); +} + static inline int crypto_tmpl_get(struct crypto_template *tmpl) { return try_module_get(tmpl->module); @@ -163,6 +172,16 @@ static inline int crypto_is_larval(struct crypto_alg *alg) return alg->cra_flags & CRYPTO_ALG_LARVAL; } +static inline int crypto_is_dead(struct crypto_alg *alg) +{ + return alg->cra_flags & CRYPTO_ALG_DEAD; +} + +static inline int crypto_is_moribund(struct crypto_alg *alg) +{ + return alg->cra_flags & (CRYPTO_ALG_DEAD | CRYPTO_ALG_DYING); +} + static inline int crypto_notify(unsigned long val, void *v) { return blocking_notifier_call_chain(&crypto_chain, val, v); diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index ffec530d52f..b20f4bdb23b 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -36,10 +36,21 @@ struct crypto_template { char name[CRYPTO_MAX_ALG_NAME]; }; +struct crypto_spawn { + struct list_head list; + struct crypto_alg *alg; + struct crypto_instance *inst; +}; + int crypto_register_template(struct crypto_template *tmpl); void crypto_unregister_template(struct crypto_template *tmpl); struct crypto_template *crypto_lookup_template(const char *name); +int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, + struct crypto_instance *inst); +void crypto_drop_spawn(struct crypto_spawn *spawn); +struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn); + static inline void *crypto_instance_ctx(struct crypto_instance *inst) { return inst->__ctx; diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 85f73c38191..40a6330abc8 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -35,6 +35,8 @@ #define CRYPTO_ALG_TYPE_COMPRESS 0x00000004 #define CRYPTO_ALG_LARVAL 0x00000010 +#define CRYPTO_ALG_DEAD 0x00000020 +#define CRYPTO_ALG_DYING 0x00000040 /* * Transform masks and values (for crt_flags). @@ -145,6 +147,8 @@ struct compress_alg { struct crypto_alg { struct list_head cra_list; + struct list_head cra_users; + u32 cra_flags; unsigned int cra_blocksize; unsigned int cra_ctxsize;