diff options
Diffstat (limited to 'drivers/crypto')
83 files changed, 3048 insertions, 962 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 76ea9273d1dc..8d3b5d2890f8 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -725,6 +725,19 @@ config CRYPTO_DEV_TEGRA Select this to enable Tegra Security Engine which accelerates various AES encryption/decryption and HASH algorithms. +config CRYPTO_DEV_XILINX_TRNG + tristate "Support for Xilinx True Random Generator" + depends on ZYNQMP_FIRMWARE || COMPILE_TEST + select CRYPTO_DF80090A + select CRYPTO_RNG + select HW_RANDOM + help + Xilinx Versal SoC driver provides kernel-side support for True Random Number + Generator and Pseudo random Number in CTR_DRBG mode as defined in NIST SP800-90A. + + To compile this driver as a module, choose M here: the module + will be called xilinx-trng. + config CRYPTO_DEV_ZYNQMP_AES tristate "Support for Xilinx ZynqMP AES hw accelerator" depends on ZYNQMP_FIRMWARE || COMPILE_TEST @@ -864,5 +877,6 @@ config CRYPTO_DEV_SA2UL source "drivers/crypto/aspeed/Kconfig" source "drivers/crypto/starfive/Kconfig" source "drivers/crypto/inside-secure/eip93/Kconfig" +source "drivers/crypto/ti/Kconfig" endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 125b99b24af1..322ae8854e3e 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -49,3 +49,4 @@ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ obj-y += intel/ obj-y += starfive/ obj-y += cavium/ +obj-y += ti/ diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index 5663df49dd81..021614b65e39 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -111,7 +111,7 @@ static int sun8i_ce_cipher_fallback(struct skcipher_request *areq) if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - struct sun8i_ce_alg_template *algt __maybe_unused; + struct sun8i_ce_alg_template *algt; algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base); @@ -131,21 +131,19 @@ static int sun8i_ce_cipher_fallback(struct skcipher_request *areq) return err; } -static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req) +static int sun8i_ce_cipher_prepare(struct skcipher_request *areq, + struct ce_task *cet) { - struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun8i_ce_dev *ce = op->ce; struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); struct skcipher_alg *alg = crypto_skcipher_alg(tfm); struct sun8i_ce_alg_template *algt; - struct sun8i_ce_flow *chan; - struct ce_task *cet; struct scatterlist *sg; unsigned int todo, len, offset, ivsize; u32 common, sym; - int flow, i; + int i; int nr_sgs = 0; int nr_sgd = 0; int err = 0; @@ -163,14 +161,9 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) algt->stat_req++; - flow = rctx->flow; - - chan = &ce->chanlist[flow]; - - cet = chan->tl; memset(cet, 0, sizeof(struct ce_task)); - cet->t_id = cpu_to_le32(flow); + cet->t_id = cpu_to_le32(rctx->flow); common = ce->variant->alg_cipher[algt->ce_algo_id]; common |= rctx->op_dir | CE_COMM_INT; cet->t_common_ctl = cpu_to_le32(common); @@ -209,11 +202,11 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req if (areq->iv && ivsize > 0) { if (rctx->op_dir & CE_DECRYPTION) { offset = areq->cryptlen - ivsize; - scatterwalk_map_and_copy(chan->backup_iv, areq->src, + scatterwalk_map_and_copy(rctx->backup_iv, areq->src, offset, ivsize, 0); } - memcpy(chan->bounce_iv, areq->iv, ivsize); - rctx->addr_iv = dma_map_single(ce->dev, chan->bounce_iv, ivsize, + memcpy(rctx->bounce_iv, areq->iv, ivsize); + rctx->addr_iv = dma_map_single(ce->dev, rctx->bounce_iv, ivsize, DMA_TO_DEVICE); if (dma_mapping_error(ce->dev, rctx->addr_iv)) { dev_err(ce->dev, "Cannot DMA MAP IV\n"); @@ -276,7 +269,6 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req goto theend_sgs; } - chan->timeout = areq->cryptlen; rctx->nr_sgs = ns; rctx->nr_sgd = nd; return 0; @@ -300,13 +292,13 @@ theend_iv: offset = areq->cryptlen - ivsize; if (rctx->op_dir & CE_DECRYPTION) { - memcpy(areq->iv, chan->backup_iv, ivsize); - memzero_explicit(chan->backup_iv, ivsize); + memcpy(areq->iv, rctx->backup_iv, ivsize); + memzero_explicit(rctx->backup_iv, ivsize); } else { scatterwalk_map_and_copy(areq->iv, areq->dst, offset, ivsize, 0); } - memzero_explicit(chan->bounce_iv, ivsize); + memzero_explicit(rctx->bounce_iv, ivsize); } dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); @@ -315,24 +307,17 @@ theend: return err; } -static void sun8i_ce_cipher_unprepare(struct crypto_engine *engine, - void *async_req) +static void sun8i_ce_cipher_unprepare(struct skcipher_request *areq, + struct ce_task *cet) { - struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun8i_ce_dev *ce = op->ce; struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); - struct sun8i_ce_flow *chan; - struct ce_task *cet; unsigned int ivsize, offset; int nr_sgs = rctx->nr_sgs; int nr_sgd = rctx->nr_sgd; - int flow; - flow = rctx->flow; - chan = &ce->chanlist[flow]; - cet = chan->tl; ivsize = crypto_skcipher_ivsize(tfm); if (areq->src == areq->dst) { @@ -349,43 +334,43 @@ static void sun8i_ce_cipher_unprepare(struct crypto_engine *engine, DMA_TO_DEVICE); offset = areq->cryptlen - ivsize; if (rctx->op_dir & CE_DECRYPTION) { - memcpy(areq->iv, chan->backup_iv, ivsize); - memzero_explicit(chan->backup_iv, ivsize); + memcpy(areq->iv, rctx->backup_iv, ivsize); + memzero_explicit(rctx->backup_iv, ivsize); } else { scatterwalk_map_and_copy(areq->iv, areq->dst, offset, ivsize, 0); } - memzero_explicit(chan->bounce_iv, ivsize); + memzero_explicit(rctx->bounce_iv, ivsize); } dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); } -static void sun8i_ce_cipher_run(struct crypto_engine *engine, void *areq) -{ - struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(breq); - struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); - struct sun8i_ce_dev *ce = op->ce; - struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(breq); - int flow, err; - - flow = rctx->flow; - err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(breq->base.tfm)); - sun8i_ce_cipher_unprepare(engine, areq); - local_bh_disable(); - crypto_finalize_skcipher_request(engine, breq, err); - local_bh_enable(); -} - int sun8i_ce_cipher_do_one(struct crypto_engine *engine, void *areq) { - int err = sun8i_ce_cipher_prepare(engine, areq); + struct skcipher_request *req = skcipher_request_cast(areq); + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct sun8i_cipher_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sun8i_ce_dev *ce = ctx->ce; + struct sun8i_ce_flow *chan; + int err; + + chan = &ce->chanlist[rctx->flow]; + err = sun8i_ce_cipher_prepare(req, chan->tl); if (err) return err; - sun8i_ce_cipher_run(engine, areq); + err = sun8i_ce_run_task(ce, rctx->flow, + crypto_tfm_alg_name(req->base.tfm)); + + sun8i_ce_cipher_unprepare(req, chan->tl); + + local_bh_disable(); + crypto_finalize_skcipher_request(engine, req, err); + local_bh_enable(); + return 0; } diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index 658f520cee0c..c16bb6ce6ee3 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -169,6 +169,12 @@ static const struct ce_variant ce_r40_variant = { .trng = CE_ID_NOTSUPP, }; +static void sun8i_ce_dump_task_descriptors(struct sun8i_ce_flow *chan) +{ + print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, + chan->tl, sizeof(struct ce_task), false); +} + /* * sun8i_ce_get_engine_number() get the next channel slot * This is a simple round-robin way of getting the next channel @@ -183,7 +189,6 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) { u32 v; int err = 0; - struct ce_task *cet = ce->chanlist[flow].tl; #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG ce->chanlist[flow].stat_req++; @@ -210,11 +215,10 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) mutex_unlock(&ce->mlock); wait_for_completion_interruptible_timeout(&ce->chanlist[flow].complete, - msecs_to_jiffies(ce->chanlist[flow].timeout)); + msecs_to_jiffies(CE_DMA_TIMEOUT_MS)); if (ce->chanlist[flow].status == 0) { - dev_err(ce->dev, "DMA timeout for %s (tm=%d) on flow %d\n", name, - ce->chanlist[flow].timeout, flow); + dev_err(ce->dev, "DMA timeout for %s on flow %d\n", name, flow); err = -EFAULT; } /* No need to lock for this read, the channel is locked so @@ -226,9 +230,8 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) /* Sadly, the error bit is not per flow */ if (v) { dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + sun8i_ce_dump_task_descriptors(&ce->chanlist[flow]); err = -EFAULT; - print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, - cet, sizeof(struct ce_task), false); } if (v & CE_ERR_ALGO_NOTSUP) dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); @@ -245,9 +248,8 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) v &= 0xF; if (v) { dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + sun8i_ce_dump_task_descriptors(&ce->chanlist[flow]); err = -EFAULT; - print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, - cet, sizeof(struct ce_task), false); } if (v & CE_ERR_ALGO_NOTSUP) dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); @@ -261,9 +263,8 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) v &= 0xFF; if (v) { dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + sun8i_ce_dump_task_descriptors(&ce->chanlist[flow]); err = -EFAULT; - print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, - cet, sizeof(struct ce_task), false); } if (v & CE_ERR_ALGO_NOTSUP) dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); @@ -758,18 +759,6 @@ static int sun8i_ce_allocate_chanlist(struct sun8i_ce_dev *ce) err = -ENOMEM; goto error_engine; } - ce->chanlist[i].bounce_iv = devm_kmalloc(ce->dev, AES_BLOCK_SIZE, - GFP_KERNEL | GFP_DMA); - if (!ce->chanlist[i].bounce_iv) { - err = -ENOMEM; - goto error_engine; - } - ce->chanlist[i].backup_iv = devm_kmalloc(ce->dev, AES_BLOCK_SIZE, - GFP_KERNEL); - if (!ce->chanlist[i].backup_iv) { - err = -ENOMEM; - goto error_engine; - } } return 0; error_engine: @@ -1063,7 +1052,7 @@ static int sun8i_ce_probe(struct platform_device *pdev) pm_runtime_put_sync(ce->dev); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct dentry *dbgfs_dir __maybe_unused; + struct dentry *dbgfs_dir; struct dentry *dbgfs_stats __maybe_unused; /* Ignore error of debugfs */ diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 13bdfb8a2c62..d01594353d9a 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -26,7 +26,7 @@ static void sun8i_ce_hash_stat_fb_inc(struct crypto_ahash *tfm) { if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct sun8i_ce_alg_template *algt __maybe_unused; + struct sun8i_ce_alg_template *algt; struct ahash_alg *alg = crypto_ahash_alg(tfm); algt = container_of(alg, struct sun8i_ce_alg_template, @@ -58,7 +58,8 @@ int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm) crypto_ahash_set_reqsize(tfm, sizeof(struct sun8i_ce_hash_reqctx) + - crypto_ahash_reqsize(op->fallback_tfm)); + crypto_ahash_reqsize(op->fallback_tfm) + + CRYPTO_DMA_PADDING); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) memcpy(algt->fbname, @@ -84,7 +85,7 @@ void sun8i_ce_hash_exit_tfm(struct crypto_ahash *tfm) int sun8i_ce_hash_init(struct ahash_request *areq) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); @@ -100,7 +101,7 @@ int sun8i_ce_hash_init(struct ahash_request *areq) int sun8i_ce_hash_export(struct ahash_request *areq, void *out) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); @@ -114,7 +115,7 @@ int sun8i_ce_hash_export(struct ahash_request *areq, void *out) int sun8i_ce_hash_import(struct ahash_request *areq, const void *in) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); @@ -128,7 +129,7 @@ int sun8i_ce_hash_import(struct ahash_request *areq, const void *in) int sun8i_ce_hash_final(struct ahash_request *areq) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); @@ -145,7 +146,7 @@ int sun8i_ce_hash_final(struct ahash_request *areq) int sun8i_ce_hash_update(struct ahash_request *areq) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); @@ -160,7 +161,7 @@ int sun8i_ce_hash_update(struct ahash_request *areq) int sun8i_ce_hash_finup(struct ahash_request *areq) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); @@ -178,7 +179,7 @@ int sun8i_ce_hash_finup(struct ahash_request *areq) static int sun8i_ce_hash_digest_fb(struct ahash_request *areq) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); @@ -238,19 +239,15 @@ static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq) int sun8i_ce_hash_digest(struct ahash_request *areq) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); - struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); - struct sun8i_ce_alg_template *algt; - struct sun8i_ce_dev *ce; + struct sun8i_ce_hash_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct sun8i_ce_dev *ce = ctx->ce; struct crypto_engine *engine; int e; if (sun8i_ce_hash_need_fallback(areq)) return sun8i_ce_hash_digest_fb(areq); - algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); - ce = algt->ce; - e = sun8i_ce_get_engine_number(ce); rctx->flow = e; engine = ce->chanlist[e].engine; @@ -316,28 +313,22 @@ static u64 hash_pad(__le32 *buf, unsigned int bufsize, u64 padi, u64 byte_count, return j; } -int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) +static int sun8i_ce_hash_prepare(struct ahash_request *areq, struct ce_task *cet) { - struct ahash_request *areq = container_of(breq, struct ahash_request, base); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct sun8i_ce_alg_template *algt; struct sun8i_ce_dev *ce; - struct sun8i_ce_flow *chan; - struct ce_task *cet; struct scatterlist *sg; - int nr_sgs, flow, err; + int nr_sgs, err; unsigned int len; u32 common; u64 byte_count; __le32 *bf; - void *buf, *result; int j, i, todo; u64 bs; int digestsize; - dma_addr_t addr_res, addr_pad; - int ns = sg_nents_for_len(areq->src, areq->nbytes); algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); ce = algt->ce; @@ -349,32 +340,16 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (digestsize == SHA384_DIGEST_SIZE) digestsize = SHA512_DIGEST_SIZE; - /* the padding could be up to two block. */ - buf = kcalloc(2, bs, GFP_KERNEL | GFP_DMA); - if (!buf) { - err = -ENOMEM; - goto err_out; - } - bf = (__le32 *)buf; - - result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); - if (!result) { - err = -ENOMEM; - goto err_free_buf; - } - - flow = rctx->flow; - chan = &ce->chanlist[flow]; + bf = (__le32 *)rctx->pad; if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) algt->stat_req++; dev_dbg(ce->dev, "%s %s len=%d\n", __func__, crypto_tfm_alg_name(areq->base.tfm), areq->nbytes); - cet = chan->tl; memset(cet, 0, sizeof(struct ce_task)); - cet->t_id = cpu_to_le32(flow); + cet->t_id = cpu_to_le32(rctx->flow); common = ce->variant->alg_hash[algt->ce_algo_id]; common |= CE_COMM_INT; cet->t_common_ctl = cpu_to_le32(common); @@ -382,11 +357,12 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) cet->t_sym_ctl = 0; cet->t_asym_ctl = 0; - nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); + rctx->nr_sgs = sg_nents_for_len(areq->src, areq->nbytes); + nr_sgs = dma_map_sg(ce->dev, areq->src, rctx->nr_sgs, DMA_TO_DEVICE); if (nr_sgs <= 0 || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; - goto err_free_result; + goto err_out; } len = areq->nbytes; @@ -401,10 +377,13 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) err = -EINVAL; goto err_unmap_src; } - addr_res = dma_map_single(ce->dev, result, digestsize, DMA_FROM_DEVICE); - cet->t_dst[0].addr = desc_addr_val_le32(ce, addr_res); - cet->t_dst[0].len = cpu_to_le32(digestsize / 4); - if (dma_mapping_error(ce->dev, addr_res)) { + + rctx->result_len = digestsize; + rctx->addr_res = dma_map_single(ce->dev, rctx->result, rctx->result_len, + DMA_FROM_DEVICE); + cet->t_dst[0].addr = desc_addr_val_le32(ce, rctx->addr_res); + cet->t_dst[0].len = cpu_to_le32(rctx->result_len / 4); + if (dma_mapping_error(ce->dev, rctx->addr_res)) { dev_err(ce->dev, "DMA map dest\n"); err = -EINVAL; goto err_unmap_src; @@ -432,10 +411,12 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) goto err_unmap_result; } - addr_pad = dma_map_single(ce->dev, buf, j * 4, DMA_TO_DEVICE); - cet->t_src[i].addr = desc_addr_val_le32(ce, addr_pad); + rctx->pad_len = j * 4; + rctx->addr_pad = dma_map_single(ce->dev, rctx->pad, rctx->pad_len, + DMA_TO_DEVICE); + cet->t_src[i].addr = desc_addr_val_le32(ce, rctx->addr_pad); cet->t_src[i].len = cpu_to_le32(j); - if (dma_mapping_error(ce->dev, addr_pad)) { + if (dma_mapping_error(ce->dev, rctx->addr_pad)) { dev_err(ce->dev, "DMA error on padding SG\n"); err = -EINVAL; goto err_unmap_result; @@ -446,29 +427,59 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) else cet->t_dlen = cpu_to_le32(areq->nbytes / 4 + j); - chan->timeout = areq->nbytes; - - err = sun8i_ce_run_task(ce, flow, crypto_ahash_alg_name(tfm)); - - dma_unmap_single(ce->dev, addr_pad, j * 4, DMA_TO_DEVICE); + return 0; err_unmap_result: - dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE); - if (!err) - memcpy(areq->result, result, crypto_ahash_digestsize(tfm)); + dma_unmap_single(ce->dev, rctx->addr_res, rctx->result_len, + DMA_FROM_DEVICE); err_unmap_src: - dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); + dma_unmap_sg(ce->dev, areq->src, rctx->nr_sgs, DMA_TO_DEVICE); -err_free_result: - kfree(result); +err_out: + return err; +} -err_free_buf: - kfree(buf); +static void sun8i_ce_hash_unprepare(struct ahash_request *areq, + struct ce_task *cet) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + struct sun8i_ce_dev *ce = ctx->ce; + + dma_unmap_single(ce->dev, rctx->addr_pad, rctx->pad_len, DMA_TO_DEVICE); + dma_unmap_single(ce->dev, rctx->addr_res, rctx->result_len, + DMA_FROM_DEVICE); + dma_unmap_sg(ce->dev, areq->src, rctx->nr_sgs, DMA_TO_DEVICE); +} + +int sun8i_ce_hash_run(struct crypto_engine *engine, void *async_req) +{ + struct ahash_request *areq = ahash_request_cast(async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct sun8i_ce_dev *ce = ctx->ce; + struct sun8i_ce_flow *chan; + int err; + + chan = &ce->chanlist[rctx->flow]; + + err = sun8i_ce_hash_prepare(areq, chan->tl); + if (err) + return err; + + err = sun8i_ce_run_task(ce, rctx->flow, crypto_ahash_alg_name(tfm)); + + sun8i_ce_hash_unprepare(areq, chan->tl); + + if (!err) + memcpy(areq->result, rctx->result, + crypto_ahash_digestsize(tfm)); -err_out: local_bh_disable(); - crypto_finalize_hash_request(engine, breq, err); + crypto_finalize_hash_request(engine, async_req, err); local_bh_enable(); return 0; diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c index 762459867b6c..d0a1ac66738b 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c @@ -137,7 +137,6 @@ int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, cet->t_dst[0].addr = desc_addr_val_le32(ce, dma_dst); cet->t_dst[0].len = cpu_to_le32(todo / 4); - ce->chanlist[flow].timeout = 2000; err = sun8i_ce_run_task(ce, 3, "PRNG"); mutex_unlock(&ce->rnglock); diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c index e1e8bc15202e..244529bf0616 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c @@ -79,7 +79,6 @@ static int sun8i_ce_trng_read(struct hwrng *rng, void *data, size_t max, bool wa cet->t_dst[0].addr = desc_addr_val_le32(ce, dma_dst); cet->t_dst[0].len = cpu_to_le32(todo / 4); - ce->chanlist[flow].timeout = todo; err = sun8i_ce_run_task(ce, 3, "TRNG"); mutex_unlock(&ce->rnglock); diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 0f9a89067016..71f5a0cd3d45 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -106,9 +106,13 @@ #define MAX_SG 8 #define CE_MAX_CLOCKS 4 +#define CE_DMA_TIMEOUT_MS 3000 #define MAXFLOW 4 +#define CE_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE +#define CE_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE + /* * struct ce_clock - Describe clocks used by sun8i-ce * @name: Name of clock needed by this variant @@ -187,8 +191,6 @@ struct ce_task { * @status: set to 1 by interrupt if task is done * @t_phy: Physical address of task * @tl: pointer to the current ce_task for this flow - * @backup_iv: buffer which contain the next IV to store - * @bounce_iv: buffer which contain the IV * @stat_req: number of request done by this flow */ struct sun8i_ce_flow { @@ -196,10 +198,7 @@ struct sun8i_ce_flow { struct completion complete; int status; dma_addr_t t_phy; - int timeout; struct ce_task *tl; - void *backup_iv; - void *bounce_iv; #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG unsigned long stat_req; #endif @@ -264,6 +263,8 @@ static inline __le32 desc_addr_val_le32(struct sun8i_ce_dev *dev, * @nr_sgd: The number of destination SG (as given by dma_map_sg()) * @addr_iv: The IV addr returned by dma_map_single, need to unmap later * @addr_key: The key addr returned by dma_map_single, need to unmap later + * @bounce_iv: Current IV buffer + * @backup_iv: Next IV buffer * @fallback_req: request struct for invoking the fallback skcipher TFM */ struct sun8i_cipher_req_ctx { @@ -273,6 +274,8 @@ struct sun8i_cipher_req_ctx { int nr_sgd; dma_addr_t addr_iv; dma_addr_t addr_key; + u8 bounce_iv[AES_BLOCK_SIZE] __aligned(sizeof(u32)); + u8 backup_iv[AES_BLOCK_SIZE]; struct skcipher_request fallback_req; // keep at the end }; @@ -304,9 +307,23 @@ struct sun8i_ce_hash_tfm_ctx { * struct sun8i_ce_hash_reqctx - context for an ahash request * @fallback_req: pre-allocated fallback request * @flow: the flow to use for this request + * @nr_sgs: number of entries in the source scatterlist + * @result_len: result length in bytes + * @pad_len: padding length in bytes + * @addr_res: DMA address of the result buffer, returned by dma_map_single() + * @addr_pad: DMA address of the padding buffer, returned by dma_map_single() + * @result: per-request result buffer + * @pad: per-request padding buffer (up to 2 blocks) */ struct sun8i_ce_hash_reqctx { int flow; + int nr_sgs; + size_t result_len; + size_t pad_len; + dma_addr_t addr_res; + dma_addr_t addr_pad; + u8 result[CE_MAX_HASH_DIGEST_SIZE] __aligned(CRYPTO_DMA_ALIGN); + u8 pad[2 * CE_MAX_HASH_BLOCK_SIZE]; struct ahash_request fallback_req; // keep at the end }; diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index 8bc08089f044..36a1ebca2e70 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -502,6 +502,7 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); ss = algt->ss; + j = 0; digestsize = crypto_ahash_digestsize(tfm); if (digestsize == SHA224_DIGEST_SIZE) @@ -536,7 +537,6 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) goto err_dma_result; } - j = 0; len = areq->nbytes; sg = areq->src; i = 0; diff --git a/drivers/crypto/aspeed/aspeed-acry.c b/drivers/crypto/aspeed/aspeed-acry.c index 8d1c79aaca07..5993bcba9716 100644 --- a/drivers/crypto/aspeed/aspeed-acry.c +++ b/drivers/crypto/aspeed/aspeed-acry.c @@ -787,7 +787,6 @@ static int aspeed_acry_probe(struct platform_device *pdev) err_engine_rsa_start: crypto_engine_exit(acry_dev->crypt_engine_rsa); clk_exit: - clk_disable_unprepare(acry_dev->clk); return rc; } @@ -799,7 +798,6 @@ static void aspeed_acry_remove(struct platform_device *pdev) aspeed_acry_unregister(acry_dev); crypto_engine_exit(acry_dev->crypt_engine_rsa); tasklet_kill(&acry_dev->done_task); - clk_disable_unprepare(acry_dev->clk); } MODULE_DEVICE_TABLE(of, aspeed_acry_of_matches); diff --git a/drivers/crypto/aspeed/aspeed-hace-crypto.c b/drivers/crypto/aspeed/aspeed-hace-crypto.c index a72dfebc53ff..fa201dae1f81 100644 --- a/drivers/crypto/aspeed/aspeed-hace-crypto.c +++ b/drivers/crypto/aspeed/aspeed-hace-crypto.c @@ -346,7 +346,7 @@ free_req: } else { dma_unmap_sg(hace_dev->dev, req->dst, rctx->dst_nents, - DMA_TO_DEVICE); + DMA_FROM_DEVICE); dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); } diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c index a895e4289efa..9688d116d07e 100644 --- a/drivers/crypto/atmel-i2c.c +++ b/drivers/crypto/atmel-i2c.c @@ -402,7 +402,7 @@ EXPORT_SYMBOL(atmel_i2c_probe); static int __init atmel_i2c_init(void) { - atmel_wq = alloc_workqueue("atmel_wq", 0, 0); + atmel_wq = alloc_workqueue("atmel_wq", WQ_PERCPU, 0); return atmel_wq ? 0 : -ENOMEM; } diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 098f5532f389..3b2a92029b16 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -512,7 +512,7 @@ static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd) if (err && (dd->flags & TDES_FLAGS_FAST)) { dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); - dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); } return err; diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 75ee065da1ec..b04d6379244a 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -252,7 +252,7 @@ struct artpec6_crypto_dma_descriptors { }; enum artpec6_crypto_variant { - ARTPEC6_CRYPTO, + ARTPEC6_CRYPTO = 1, ARTPEC7_CRYPTO, }; @@ -2842,7 +2842,6 @@ MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); static int artpec6_crypto_probe(struct platform_device *pdev) { - const struct of_device_id *match; enum artpec6_crypto_variant variant; struct artpec6_crypto *ac; struct device *dev = &pdev->dev; @@ -2853,12 +2852,10 @@ static int artpec6_crypto_probe(struct platform_device *pdev) if (artpec6_crypto_dev) return -ENODEV; - match = of_match_node(artpec6_crypto_of_match, dev->of_node); - if (!match) + variant = (enum artpec6_crypto_variant)of_device_get_match_data(dev); + if (!variant) return -EINVAL; - variant = (enum artpec6_crypto_variant)match->data; - base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); diff --git a/drivers/crypto/caam/blob_gen.c b/drivers/crypto/caam/blob_gen.c index 079a22cc9f02..c18dbac56493 100644 --- a/drivers/crypto/caam/blob_gen.c +++ b/drivers/crypto/caam/blob_gen.c @@ -2,13 +2,14 @@ /* * Copyright (C) 2015 Pengutronix, Steffen Trumtrar <kernel@pengutronix.de> * Copyright (C) 2021 Pengutronix, Ahmad Fatoum <kernel@pengutronix.de> - * Copyright 2024 NXP + * Copyright 2024-2025 NXP */ #define pr_fmt(fmt) "caam blob_gen: " fmt #include <linux/bitfield.h> #include <linux/device.h> +#include <keys/trusted-type.h> #include <soc/fsl/caam-blob.h> #include "compat.h" @@ -60,18 +61,27 @@ static void caam_blob_job_done(struct device *dev, u32 *desc, u32 err, void *con complete(&res->completion); } +static u32 check_caam_state(struct device *jrdev) +{ + const struct caam_drv_private *ctrlpriv; + + ctrlpriv = dev_get_drvdata(jrdev->parent); + return FIELD_GET(CSTA_MOO, rd_reg32(&ctrlpriv->jr[0]->perfmon.status)); +} + int caam_process_blob(struct caam_blob_priv *priv, struct caam_blob_info *info, bool encap) { - const struct caam_drv_private *ctrlpriv; struct caam_blob_job_result testres; struct device *jrdev = &priv->jrdev; dma_addr_t dma_in, dma_out; int op = OP_PCLID_BLOB; + int hwbk_caam_ovhd = 0; size_t output_len; u32 *desc; u32 moo; int ret; + int len; if (info->key_mod_len > CAAM_BLOB_KEYMOD_LENGTH) return -EINVAL; @@ -82,14 +92,29 @@ int caam_process_blob(struct caam_blob_priv *priv, } else { op |= OP_TYPE_DECAP_PROTOCOL; output_len = info->input_len - CAAM_BLOB_OVERHEAD; + info->output_len = output_len; + } + + if (encap && info->pkey_info.is_pkey) { + op |= OP_PCL_BLOB_BLACK; + if (info->pkey_info.key_enc_algo == CAAM_ENC_ALGO_CCM) { + op |= OP_PCL_BLOB_EKT; + hwbk_caam_ovhd = CAAM_CCM_OVERHEAD; + } + if ((info->input_len + hwbk_caam_ovhd) > MAX_KEY_SIZE) + return -EINVAL; + + len = info->input_len + hwbk_caam_ovhd; + } else { + len = info->input_len; } desc = kzalloc(CAAM_BLOB_DESC_BYTES_MAX, GFP_KERNEL); if (!desc) return -ENOMEM; - dma_in = dma_map_single(jrdev, info->input, info->input_len, - DMA_TO_DEVICE); + dma_in = dma_map_single(jrdev, info->input, len, + encap ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); if (dma_mapping_error(jrdev, dma_in)) { dev_err(jrdev, "unable to map input DMA buffer\n"); ret = -ENOMEM; @@ -104,8 +129,7 @@ int caam_process_blob(struct caam_blob_priv *priv, goto out_unmap_in; } - ctrlpriv = dev_get_drvdata(jrdev->parent); - moo = FIELD_GET(CSTA_MOO, rd_reg32(&ctrlpriv->jr[0]->perfmon.status)); + moo = check_caam_state(jrdev); if (moo != CSTA_MOO_SECURE && moo != CSTA_MOO_TRUSTED) dev_warn(jrdev, "using insecure test key, enable HAB to use unique device key!\n"); @@ -117,18 +141,48 @@ int caam_process_blob(struct caam_blob_priv *priv, * Class 1 Context DWords 0+1+2+3. The random BK is stored in the * Class 1 Key Register. Operation Mode is set to AES-CCM. */ - init_job_desc(desc, 0); + + if (encap && info->pkey_info.is_pkey) { + /*!1. key command used to load class 1 key register + * from input plain key. + */ + append_key(desc, dma_in, info->input_len, + CLASS_1 | KEY_DEST_CLASS_REG); + /*!2. Fifostore to store protected key from class 1 key register. */ + if (info->pkey_info.key_enc_algo == CAAM_ENC_ALGO_CCM) { + append_fifo_store(desc, dma_in, info->input_len, + LDST_CLASS_1_CCB | + FIFOST_TYPE_KEY_CCM_JKEK); + } else { + append_fifo_store(desc, dma_in, info->input_len, + LDST_CLASS_1_CCB | + FIFOST_TYPE_KEY_KEK); + } + /* + * JUMP_OFFSET specifies the offset of the JUMP target from + * the JUMP command's address in the descriptor buffer. + */ + append_jump(desc, JUMP_COND_NOP | BIT(0) << JUMP_OFFSET_SHIFT); + } + + /*!3. Load class 2 key with key modifier. */ append_key_as_imm(desc, info->key_mod, info->key_mod_len, - info->key_mod_len, CLASS_2 | KEY_DEST_CLASS_REG); - append_seq_in_ptr_intlen(desc, dma_in, info->input_len, 0); - append_seq_out_ptr_intlen(desc, dma_out, output_len, 0); + info->key_mod_len, CLASS_2 | KEY_DEST_CLASS_REG); + + /*!4. SEQ IN PTR Command. */ + append_seq_in_ptr(desc, dma_in, info->input_len, 0); + + /*!5. SEQ OUT PTR Command. */ + append_seq_out_ptr(desc, dma_out, output_len, 0); + + /*!6. Blob encapsulation/decapsulation PROTOCOL Command. */ append_operation(desc, op); - print_hex_dump_debug("data@"__stringify(__LINE__)": ", + print_hex_dump_debug("data@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 1, info->input, - info->input_len, false); - print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + len, false); + print_hex_dump_debug("jobdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 1, desc, desc_bytes(desc), false); @@ -139,7 +193,7 @@ int caam_process_blob(struct caam_blob_priv *priv, if (ret == -EINPROGRESS) { wait_for_completion(&testres.completion); ret = testres.err; - print_hex_dump_debug("output@"__stringify(__LINE__)": ", + print_hex_dump_debug("output@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 1, info->output, output_len, false); } @@ -149,10 +203,10 @@ int caam_process_blob(struct caam_blob_priv *priv, dma_unmap_single(jrdev, dma_out, output_len, DMA_FROM_DEVICE); out_unmap_in: - dma_unmap_single(jrdev, dma_in, info->input_len, DMA_TO_DEVICE); + dma_unmap_single(jrdev, dma_in, len, + encap ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); out_free: kfree(desc); - return ret; } EXPORT_SYMBOL(caam_process_blob); diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 2cfb1b8d8c7c..32a6e6e15ee2 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -3,7 +3,7 @@ * caam - Freescale FSL CAAM support for crypto API * * Copyright 2008-2011 Freescale Semiconductor, Inc. - * Copyright 2016-2019, 2023 NXP + * Copyright 2016-2019, 2023, 2025 NXP * * Based on talitos crypto API driver. * @@ -61,13 +61,16 @@ #include <crypto/internal/engine.h> #include <crypto/internal/skcipher.h> #include <crypto/xts.h> +#include <keys/trusted-type.h> #include <linux/dma-mapping.h> #include <linux/device.h> #include <linux/err.h> #include <linux/module.h> #include <linux/kernel.h> +#include <linux/key-type.h> #include <linux/slab.h> #include <linux/string.h> +#include <soc/fsl/caam-blob.h> /* * crypto alg @@ -119,12 +122,15 @@ struct caam_ctx { dma_addr_t sh_desc_enc_dma; dma_addr_t sh_desc_dec_dma; dma_addr_t key_dma; + u8 protected_key[CAAM_MAX_KEY_SIZE]; + dma_addr_t protected_key_dma; enum dma_data_direction dir; struct device *jrdev; struct alginfo adata; struct alginfo cdata; unsigned int authsize; bool xts_key_fallback; + bool is_blob; struct crypto_skcipher *fallback; }; @@ -751,9 +757,14 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, print_hex_dump_debug("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + /* Here keylen is actual key length */ ctx->cdata.keylen = keylen; ctx->cdata.key_virt = key; ctx->cdata.key_inline = true; + /* Here protected key len is plain key length */ + ctx->cdata.plain_keylen = keylen; + ctx->cdata.key_cmd_opt = 0; + /* skcipher_encrypt shared descriptor */ desc = ctx->sh_desc_enc; @@ -772,6 +783,62 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, return 0; } +static int paes_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, + unsigned int keylen) +{ + struct caam_pkey_info *pkey_info = (struct caam_pkey_info *)key; + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *jrdev = ctx->jrdev; + int err; + + ctx->cdata.key_inline = false; + + keylen = keylen - CAAM_PKEY_HEADER; + + /* Retrieve the length of key */ + ctx->cdata.plain_keylen = pkey_info->plain_key_sz; + + /* Retrieve the length of blob*/ + ctx->cdata.keylen = keylen; + + /* Retrieve the address of the blob */ + ctx->cdata.key_virt = pkey_info->key_buf; + + /* Validate key length for AES algorithms */ + err = aes_check_keylen(ctx->cdata.plain_keylen); + if (err) { + dev_err(jrdev, "bad key length\n"); + return err; + } + + /* set command option */ + ctx->cdata.key_cmd_opt |= KEY_ENC; + + /* check if the Protected-Key is CCM key */ + if (pkey_info->key_enc_algo == CAAM_ENC_ALGO_CCM) + ctx->cdata.key_cmd_opt |= KEY_EKT; + + memcpy(ctx->key, ctx->cdata.key_virt, keylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.key_dma = ctx->key_dma; + + if (pkey_info->key_enc_algo == CAAM_ENC_ALGO_CCM) + ctx->protected_key_dma = dma_map_single(jrdev, ctx->protected_key, + ctx->cdata.plain_keylen + + CAAM_CCM_OVERHEAD, + DMA_FROM_DEVICE); + else + ctx->protected_key_dma = dma_map_single(jrdev, ctx->protected_key, + ctx->cdata.plain_keylen, + DMA_FROM_DEVICE); + + ctx->cdata.protected_key_dma = ctx->protected_key_dma; + ctx->is_blob = true; + + return 0; +} + static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, unsigned int keylen) { @@ -1254,7 +1321,9 @@ static void init_skcipher_job(struct skcipher_request *req, struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); struct device *jrdev = ctx->jrdev; int ivsize = crypto_skcipher_ivsize(skcipher); - u32 *desc = edesc->hw_desc; + u32 *desc = !ctx->is_blob ? edesc->hw_desc : + (u32 *)((u8 *)edesc->hw_desc + CAAM_DESC_BYTES_MAX); + dma_addr_t desc_dma; u32 *sh_desc; u32 in_options = 0, out_options = 0; dma_addr_t src_dma, dst_dma, ptr; @@ -1269,11 +1338,6 @@ static void init_skcipher_job(struct skcipher_request *req, DUMP_PREFIX_ADDRESS, 16, 4, req->src, edesc->src_nents > 1 ? 100 : req->cryptlen, 1); - sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; - ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; - - len = desc_len(sh_desc); - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); if (ivsize || edesc->mapped_src_nents > 1) { src_dma = edesc->sec4_sg_dma; @@ -1283,8 +1347,6 @@ static void init_skcipher_job(struct skcipher_request *req, src_dma = sg_dma_address(req->src); } - append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); - if (likely(req->src == req->dst)) { dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); out_options = in_options; @@ -1296,7 +1358,25 @@ static void init_skcipher_job(struct skcipher_request *req, out_options = LDST_SGF; } - append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); + if (ctx->is_blob) { + cnstr_desc_skcipher_enc_dec(desc, &ctx->cdata, + src_dma, dst_dma, req->cryptlen + ivsize, + in_options, out_options, + ivsize, encrypt); + + desc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE); + + cnstr_desc_protected_blob_decap(edesc->hw_desc, &ctx->cdata, desc_dma); + } else { + sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; + ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; + + len = desc_len(sh_desc); + init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); + append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); + + append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); + } } /* @@ -1817,6 +1897,7 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; int ret = 0; + int len; /* * XTS is expected to return an error even for input length = 0 @@ -1842,8 +1923,12 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) crypto_skcipher_decrypt(&rctx->fallback_req); } + len = DESC_JOB_IO_LEN * CAAM_CMD_SZ; + if (ctx->is_blob) + len += CAAM_DESC_BYTES_MAX; + /* allocate extended descriptor */ - edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ); + edesc = skcipher_edesc_alloc(req, len); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1888,6 +1973,27 @@ static struct caam_skcipher_alg driver_algs[] = { { .skcipher.base = { .base = { + .cra_name = "cbc(paes)", + .cra_driver_name = "cbc-paes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = paes_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + CAAM_BLOB_OVERHEAD + + CAAM_PKEY_HEADER, + .max_keysize = AES_MAX_KEY_SIZE + CAAM_BLOB_OVERHEAD + + CAAM_PKEY_HEADER, + .ivsize = AES_BLOCK_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .skcipher.base = { + .base = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-caam", .cra_blocksize = AES_BLOCK_SIZE, diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index 7571e1ac913b..04c1105eb1f5 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -2,12 +2,13 @@ /* * Shared descriptors for aead, skcipher algorithms * - * Copyright 2016-2019 NXP + * Copyright 2016-2019, 2025 NXP */ #include "compat.h" #include "desc_constr.h" #include "caamalg_desc.h" +#include <soc/fsl/caam-blob.h> /* * For aead functions, read payload and write payload, @@ -1364,6 +1365,84 @@ static inline void skcipher_append_src_dst(u32 *desc) append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); } +void cnstr_desc_skcipher_enc_dec(u32 * const desc, struct alginfo *cdata, + dma_addr_t src, dma_addr_t dst, unsigned int data_sz, + unsigned int in_options, unsigned int out_options, + unsigned int ivsize, const bool encrypt) +{ + u32 options = cdata->algtype | OP_ALG_AS_INIT; + + if (encrypt) + options |= OP_ALG_ENCRYPT; + else + options |= OP_ALG_DECRYPT; + + init_job_desc(desc, 0); + + append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL | + JUMP_COND_NOP | JUMP_TEST_ALL | 1); + + append_key(desc, cdata->protected_key_dma, cdata->plain_keylen, + CLASS_1 | KEY_DEST_CLASS_REG | cdata->key_cmd_opt); + + append_seq_in_ptr(desc, src, data_sz, in_options); + + append_seq_out_ptr(desc, dst, data_sz, out_options); + + /* Load IV, if there is one */ + if (ivsize) + append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB); + + append_operation(desc, options); + + skcipher_append_src_dst(desc); + + /* Store IV */ + if (ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB); + + print_hex_dump_debug("skcipher_enc_dec job desc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_desc_skcipher_enc_dec); + +void cnstr_desc_protected_blob_decap(u32 * const desc, struct alginfo *cdata, + dma_addr_t next_desc_addr) +{ + u32 protected_store; + + init_job_desc(desc, 0); + + /* Load key modifier */ + append_load_as_imm(desc, KEYMOD, sizeof(KEYMOD) - 1, + LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_KEY); + + append_seq_in_ptr_intlen(desc, cdata->key_dma, + cdata->plain_keylen + CAAM_BLOB_OVERHEAD, 0); + + append_seq_out_ptr_intlen(desc, cdata->protected_key_dma, + cdata->plain_keylen, 0); + + protected_store = OP_PCLID_BLOB | OP_PCL_BLOB_BLACK; + if ((cdata->key_cmd_opt >> KEY_EKT_OFFSET) & 1) + protected_store |= OP_PCL_BLOB_EKT; + + append_operation(desc, OP_TYPE_DECAP_PROTOCOL | protected_store); + + if (next_desc_addr) { + append_jump(desc, JUMP_TYPE_NONLOCAL | JUMP_TEST_ALL); + append_ptr(desc, next_desc_addr); + } + + print_hex_dump_debug("protected blob decap job desc@" __stringify(__LINE__) ":", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +} +EXPORT_SYMBOL(cnstr_desc_protected_blob_decap); + /** * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor * @desc: pointer to buffer used for descriptor construction @@ -1391,7 +1470,8 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, /* Load class1 key only */ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, - cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + cdata->plain_keylen, CLASS_1 | KEY_DEST_CLASS_REG + | cdata->key_cmd_opt); /* Load nonce into CONTEXT1 reg */ if (is_rfc3686) { @@ -1466,7 +1546,8 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, /* Load class1 key only */ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, - cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + cdata->plain_keylen, CLASS_1 | KEY_DEST_CLASS_REG + | cdata->key_cmd_opt); /* Load nonce into CONTEXT1 reg */ if (is_rfc3686) { diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h index f2893393ba5e..323490a4a756 100644 --- a/drivers/crypto/caam/caamalg_desc.h +++ b/drivers/crypto/caam/caamalg_desc.h @@ -2,7 +2,7 @@ /* * Shared descriptors for aead, skcipher algorithms * - * Copyright 2016 NXP + * Copyright 2016, 2025 NXP */ #ifndef _CAAMALG_DESC_H_ @@ -48,6 +48,9 @@ #define DESC_SKCIPHER_DEC_LEN (DESC_SKCIPHER_BASE + \ 16 * CAAM_CMD_SZ) +/* Key modifier for CAAM Protected blobs */ +#define KEYMOD "SECURE_KEY" + void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, unsigned int icvsize, int era); @@ -113,4 +116,12 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata); void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata); +void cnstr_desc_protected_blob_decap(u32 * const desc, struct alginfo *cdata, + dma_addr_t next_desc); + +void cnstr_desc_skcipher_enc_dec(u32 * const desc, struct alginfo *cdata, + dma_addr_t src, dma_addr_t dst, unsigned int data_sz, + unsigned int in_options, unsigned int out_options, + unsigned int ivsize, const bool encrypt); + #endif /* _CAAMALG_DESC_H_ */ diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index b3d14a7f4dd1..0eb43c862516 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -181,7 +181,9 @@ static inline void test_len(struct hwrng *rng, size_t len, bool wait) struct device *dev = ctx->ctrldev; buf = kcalloc(CAAM_RNG_MAX_FIFO_STORE_SIZE, sizeof(u8), GFP_KERNEL); - + if (!buf) { + return; + } while (len > 0) { read_len = rng->read(rng, buf, len, wait); diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index a93be395c878..320be5d77737 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -592,9 +592,9 @@ static int init_clocks(struct device *dev, const struct caam_imx_data *data) int ret; ctrlpriv->num_clks = data->num_clks; - ctrlpriv->clks = devm_kmemdup(dev, data->clks, - data->num_clks * sizeof(data->clks[0]), - GFP_KERNEL); + ctrlpriv->clks = devm_kmemdup_array(dev, data->clks, + data->num_clks, sizeof(*data->clks), + GFP_KERNEL); if (!ctrlpriv->clks) return -ENOMEM; @@ -703,12 +703,12 @@ static int caam_ctrl_rng_init(struct device *dev) */ if (needs_entropy_delay_adjustment()) ent_delay = 12000; - if (!(ctrlpriv->rng4_sh_init || inst_handles)) { + if (!inst_handles) { dev_info(dev, "Entropy delay = %u\n", ent_delay); kick_trng(dev, ent_delay); - ent_delay += 400; + ent_delay = ent_delay * 2; } /* * if instantiate_rng(...) fails, the loop will rerun diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index e13470901586..c28e94fcb8c7 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -4,7 +4,7 @@ * Definitions to support CAAM descriptor instruction generation * * Copyright 2008-2011 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018, 2025 NXP */ #ifndef DESC_H @@ -162,6 +162,7 @@ * Enhanced Encryption of Key */ #define KEY_EKT 0x00100000 +#define KEY_EKT_OFFSET 20 /* * Encrypted with Trusted Key @@ -403,6 +404,7 @@ #define FIFOST_TYPE_PKHA_N (0x08 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_A (0x0c << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_B (0x0d << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_KEY_CCM_JKEK (0x14 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_AF_SBOX_JKEK (0x20 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_AF_SBOX_TKEK (0x21 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_E_JKEK (0x22 << FIFOST_TYPE_SHIFT) @@ -1001,6 +1003,11 @@ #define OP_PCL_TLS12_AES_256_CBC_SHA384 0xff63 #define OP_PCL_TLS12_AES_256_CBC_SHA512 0xff65 +/* Blob protocol protinfo bits */ + +#define OP_PCL_BLOB_BLACK 0x0004 +#define OP_PCL_BLOB_EKT 0x0100 + /* For DTLS - OP_PCLID_DTLS */ #define OP_PCL_DTLS_AES_128_CBC_SHA 0x002f diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 824c94d44f94..2a29dd2c9c8a 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -3,7 +3,7 @@ * caam descriptor construction helper functions * * Copyright 2008-2012 Freescale Semiconductor, Inc. - * Copyright 2019 NXP + * Copyright 2019, 2025 NXP */ #ifndef DESC_CONSTR_H @@ -498,17 +498,23 @@ do { \ * @keylen: length of the provided algorithm key, in bytes * @keylen_pad: padded length of the provided algorithm key, in bytes * @key_dma: dma (bus) address where algorithm key resides + * @protected_key_dma: dma (bus) address where protected key resides * @key_virt: virtual address where algorithm key resides * @key_inline: true - key can be inlined in the descriptor; false - key is * referenced by the descriptor + * @plain_keylen: size of the key to be loaded by the CAAM + * @key_cmd_opt: optional parameters for KEY command */ struct alginfo { u32 algtype; unsigned int keylen; unsigned int keylen_pad; dma_addr_t key_dma; + dma_addr_t protected_key_dma; const void *key_virt; bool key_inline; + u32 plain_keylen; + u32 key_cmd_opt; }; /** diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c index d4e06999af9b..a6a76e50ba84 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.c +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -192,7 +192,7 @@ int nitrox_mbox_init(struct nitrox_device *ndev) } /* allocate pf2vf response workqueue */ - ndev->iov.pf2vf_wq = alloc_workqueue("nitrox_pf2vf", 0, 0); + ndev->iov.pf2vf_wq = alloc_workqueue("nitrox_pf2vf", WQ_PERCPU, 0); if (!ndev->iov.pf2vf_wq) { kfree(ndev->iov.vfdev); ndev->iov.vfdev = NULL; diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index c531d13d971f..246801912e1a 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -507,7 +507,7 @@ int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) { struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng); u32 trng_value; - int len = min_t(int, sizeof(trng_value), max); + int len = min(sizeof(trng_value), max); /* Locking is provided by the caller so we can update device * hwrng-related fields safely diff --git a/drivers/crypto/ccp/hsti.c b/drivers/crypto/ccp/hsti.c index 1b39a4fb55c0..c29c6a9c0f3f 100644 --- a/drivers/crypto/ccp/hsti.c +++ b/drivers/crypto/ccp/hsti.c @@ -74,7 +74,7 @@ struct attribute_group psp_security_attr_group = { .is_visible = psp_security_is_visible, }; -static int psp_poulate_hsti(struct psp_device *psp) +static int psp_populate_hsti(struct psp_device *psp) { struct hsti_request *req; int ret; @@ -84,11 +84,11 @@ static int psp_poulate_hsti(struct psp_device *psp) return 0; /* Allocate command-response buffer */ - req = kzalloc(sizeof(*req), GFP_KERNEL | __GFP_ZERO); + req = kzalloc(sizeof(*req), GFP_KERNEL); if (!req) return -ENOMEM; - req->header.payload_size = sizeof(req); + req->header.payload_size = sizeof(*req); ret = psp_send_platform_access_msg(PSP_CMD_HSTI_QUERY, (struct psp_request *)req); if (ret) @@ -114,7 +114,7 @@ int psp_init_hsti(struct psp_device *psp) int ret; if (PSP_FEATURE(psp, HSTI)) { - ret = psp_poulate_hsti(psp); + ret = psp_populate_hsti(psp); if (ret) return ret; } diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 8dff5c2c40fd..b28a6f50daaa 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -249,6 +249,8 @@ static int sev_cmd_buffer_len(int cmd) case SEV_CMD_SNP_GUEST_REQUEST: return sizeof(struct sev_data_snp_guest_request); case SEV_CMD_SNP_CONFIG: return sizeof(struct sev_user_data_snp_config); case SEV_CMD_SNP_COMMIT: return sizeof(struct sev_data_snp_commit); + case SEV_CMD_SNP_FEATURE_INFO: return sizeof(struct sev_data_snp_feature_info); + case SEV_CMD_SNP_VLEK_LOAD: return sizeof(struct sev_user_data_snp_vlek_load); default: return 0; } @@ -257,27 +259,20 @@ static int sev_cmd_buffer_len(int cmd) static struct file *open_file_as_root(const char *filename, int flags, umode_t mode) { - struct file *fp; - struct path root; - struct cred *cred; - const struct cred *old_cred; + struct path root __free(path_put) = {}; task_lock(&init_task); get_fs_root(init_task.fs, &root); task_unlock(&init_task); - cred = prepare_creds(); + CLASS(prepare_creds, cred)(); if (!cred) return ERR_PTR(-ENOMEM); - cred->fsuid = GLOBAL_ROOT_UID; - old_cred = override_creds(cred); - fp = file_open_root(&root, filename, flags, mode); - path_put(&root); - - put_cred(revert_creds(old_cred)); + cred->fsuid = GLOBAL_ROOT_UID; - return fp; + scoped_with_creds(cred) + return file_open_root(&root, filename, flags, mode); } static int sev_read_init_ex_file(void) @@ -862,9 +857,10 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) struct sev_device *sev; unsigned int cmdbuff_hi, cmdbuff_lo; unsigned int phys_lsb, phys_msb; - unsigned int reg, ret = 0; + unsigned int reg; void *cmd_buf; int buf_len; + int ret = 0; if (!psp || !psp->sev_data) return -ENODEV; @@ -1248,6 +1244,88 @@ static void snp_leak_hv_fixed_pages(void) 1 << entry->order, false); } +bool sev_is_snp_ciphertext_hiding_supported(void) +{ + struct psp_device *psp = psp_master; + struct sev_device *sev; + + if (!psp || !psp->sev_data) + return false; + + sev = psp->sev_data; + + /* + * Feature information indicates if CipherTextHiding feature is + * supported by the SEV firmware and additionally platform status + * indicates if CipherTextHiding feature is enabled in the + * Platform BIOS. + */ + return ((sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED) && + sev->snp_plat_status.ciphertext_hiding_cap); +} +EXPORT_SYMBOL_GPL(sev_is_snp_ciphertext_hiding_supported); + +static int snp_get_platform_data(struct sev_device *sev, int *error) +{ + struct sev_data_snp_feature_info snp_feat_info; + struct snp_feature_info *feat_info; + struct sev_data_snp_addr buf; + struct page *page; + int rc; + + /* + * This function is expected to be called before SNP is + * initialized. + */ + if (sev->snp_initialized) + return -EINVAL; + + buf.address = __psp_pa(&sev->snp_plat_status); + rc = sev_do_cmd(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error); + if (rc) { + dev_err(sev->dev, "SNP PLATFORM_STATUS command failed, ret = %d, error = %#x\n", + rc, *error); + return rc; + } + + sev->api_major = sev->snp_plat_status.api_major; + sev->api_minor = sev->snp_plat_status.api_minor; + sev->build = sev->snp_plat_status.build_id; + + /* + * Do feature discovery of the currently loaded firmware, + * and cache feature information from CPUID 0x8000_0024, + * sub-function 0. + */ + if (!sev->snp_plat_status.feature_info) + return 0; + + /* + * Use dynamically allocated structure for the SNP_FEATURE_INFO + * command to ensure structure is 8-byte aligned, and does not + * cross a page boundary. + */ + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + + feat_info = page_address(page); + snp_feat_info.length = sizeof(snp_feat_info); + snp_feat_info.ecx_in = 0; + snp_feat_info.feature_info_paddr = __psp_pa(feat_info); + + rc = sev_do_cmd(SEV_CMD_SNP_FEATURE_INFO, &snp_feat_info, error); + if (!rc) + sev->snp_feat_info_0 = *feat_info; + else + dev_err(sev->dev, "SNP FEATURE_INFO command failed, ret = %d, error = %#x\n", + rc, *error); + + __free_page(page); + + return rc; +} + static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) { struct sev_data_range_list *range_list = arg; @@ -1278,7 +1356,7 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) return 0; } -static int __sev_snp_init_locked(int *error) +static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) { struct psp_device *psp = psp_master; struct sev_data_snp_init_ex data; @@ -1345,6 +1423,12 @@ static int __sev_snp_init_locked(int *error) snp_add_hv_fixed_pages(sev, snp_range_list); memset(&data, 0, sizeof(data)); + + if (max_snp_asid) { + data.ciphertext_hiding_en = 1; + data.max_snp_asid = max_snp_asid; + } + data.init_rmp = 1; data.list_paddr_en = 1; data.list_paddr = __psp_pa(snp_range_list); @@ -1468,7 +1552,7 @@ static int __sev_platform_init_locked(int *error) sev = psp_master->sev_data; - if (sev->state == SEV_STATE_INIT) + if (sev->sev_plat_status.state == SEV_STATE_INIT) return 0; __sev_platform_init_handle_tmr(sev); @@ -1500,7 +1584,7 @@ static int __sev_platform_init_locked(int *error) return rc; } - sev->state = SEV_STATE_INIT; + sev->sev_plat_status.state = SEV_STATE_INIT; /* Prepare for first SEV guest launch after INIT */ wbinvd_on_all_cpus(); @@ -1538,10 +1622,10 @@ static int _sev_platform_init_locked(struct sev_platform_init_args *args) sev = psp_master->sev_data; - if (sev->state == SEV_STATE_INIT) + if (sev->sev_plat_status.state == SEV_STATE_INIT) return 0; - rc = __sev_snp_init_locked(&args->error); + rc = __sev_snp_init_locked(&args->error, args->max_snp_asid); if (rc && rc != -ENODEV) return rc; @@ -1575,7 +1659,7 @@ static int __sev_platform_shutdown_locked(int *error) sev = psp->sev_data; - if (sev->state == SEV_STATE_UNINIT) + if (sev->sev_plat_status.state == SEV_STATE_UNINIT) return 0; ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error); @@ -1585,7 +1669,7 @@ static int __sev_platform_shutdown_locked(int *error) return ret; } - sev->state = SEV_STATE_UNINIT; + sev->sev_plat_status.state = SEV_STATE_UNINIT; dev_dbg(sev->dev, "SEV firmware shutdown\n"); return ret; @@ -1624,7 +1708,7 @@ static int snp_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_req { int error, rc; - rc = __sev_snp_init_locked(&error); + rc = __sev_snp_init_locked(&error, 0); if (rc) { argp->error = SEV_RET_INVALID_PLATFORM_STATE; return rc; @@ -1693,7 +1777,7 @@ static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool wr if (!writable) return -EPERM; - if (sev->state == SEV_STATE_UNINIT) { + if (sev->sev_plat_status.state == SEV_STATE_UNINIT) { rc = sev_move_to_init_state(argp, &shutdown_required); if (rc) return rc; @@ -1742,7 +1826,7 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) data.len = input.length; cmd: - if (sev->state == SEV_STATE_UNINIT) { + if (sev->sev_plat_status.state == SEV_STATE_UNINIT) { ret = sev_move_to_init_state(argp, &shutdown_required); if (ret) goto e_free_blob; @@ -1790,6 +1874,16 @@ static int sev_get_api_version(void) struct sev_user_data_status status; int error = 0, ret; + /* + * Cache SNP platform status and SNP feature information + * if SNP is available. + */ + if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) { + ret = snp_get_platform_data(sev, &error); + if (ret) + return 1; + } + ret = sev_platform_status(&status, &error); if (ret) { dev_err(sev->dev, @@ -1797,10 +1891,12 @@ static int sev_get_api_version(void) return 1; } + /* Cache SEV platform status */ + sev->sev_plat_status = status; + sev->api_major = status.api_major; sev->api_minor = status.api_minor; sev->build = status.build; - sev->state = status.state; return 0; } @@ -2029,7 +2125,7 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) data.oca_cert_len = input.oca_cert_len; /* If platform is not in INIT state then transition it to INIT */ - if (sev->state != SEV_STATE_INIT) { + if (sev->sev_plat_status.state != SEV_STATE_INIT) { ret = sev_move_to_init_state(argp, &shutdown_required); if (ret) goto e_free_oca; @@ -2200,7 +2296,7 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) cmd: /* If platform is not in INIT state then transition it to INIT. */ - if (sev->state != SEV_STATE_INIT) { + if (sev->sev_plat_status.state != SEV_STATE_INIT) { if (!writable) { ret = -EPERM; goto e_free_cert; diff --git a/drivers/crypto/ccp/sev-dev.h b/drivers/crypto/ccp/sev-dev.h index 28021abc85ad..ac03bd0848f7 100644 --- a/drivers/crypto/ccp/sev-dev.h +++ b/drivers/crypto/ccp/sev-dev.h @@ -42,7 +42,6 @@ struct sev_device { struct sev_vdata *vdata; - int state; unsigned int int_rcvd; wait_queue_head_t int_queue; struct sev_misc_dev *misc; @@ -57,6 +56,11 @@ struct sev_device { bool cmd_buf_backup_active; bool snp_initialized; + + struct sev_user_data_status sev_plat_status; + + struct sev_user_data_snp_status snp_plat_status; + struct snp_feature_info snp_feat_info_0; }; int sev_dev_init(struct psp_device *psp); diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h index 6f9d7063257d..1335a83fe052 100644 --- a/drivers/crypto/ccp/sp-dev.h +++ b/drivers/crypto/ccp/sp-dev.h @@ -95,7 +95,7 @@ struct sp_device { struct device *dev; - struct sp_dev_vdata *dev_vdata; + const struct sp_dev_vdata *dev_vdata; unsigned int ord; char name[SP_MAX_NAME_LEN]; diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index e7bb803912a6..8891ceee1d7d 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -459,6 +459,17 @@ static const struct psp_vdata pspv6 = { .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ }; +static const struct psp_vdata pspv7 = { + .tee = &teev2, + .cmdresp_reg = 0x10944, /* C2PMSG_17 */ + .cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */ + .cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ + .feature_reg = 0x109fc, /* C2PMSG_63 */ + .inten_reg = 0x10510, /* P2CMSG_INTEN */ + .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ +}; + #endif static const struct sp_dev_vdata dev_vdata[] = { @@ -525,6 +536,13 @@ static const struct sp_dev_vdata dev_vdata[] = { .psp_vdata = &pspv6, #endif }, + { /* 9 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv7, +#endif + }, + }; static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] }, @@ -539,6 +557,7 @@ static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x17E0), (kernel_ulong_t)&dev_vdata[7] }, { PCI_VDEVICE(AMD, 0x156E), (kernel_ulong_t)&dev_vdata[8] }, { PCI_VDEVICE(AMD, 0x17D8), (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x115A), (kernel_ulong_t)&dev_vdata[9] }, /* Last entry must be zero */ { 0, } }; diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c index 3933cac1694d..3f9843fa7782 100644 --- a/drivers/crypto/ccp/sp-platform.c +++ b/drivers/crypto/ccp/sp-platform.c @@ -52,24 +52,13 @@ static const struct of_device_id sp_of_match[] = { }; MODULE_DEVICE_TABLE(of, sp_of_match); -static struct sp_dev_vdata *sp_get_of_version(struct platform_device *pdev) -{ - const struct of_device_id *match; - - match = of_match_node(sp_of_match, pdev->dev.of_node); - if (match && match->data) - return (struct sp_dev_vdata *)match->data; - - return NULL; -} - -static struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev) +static const struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev) { const struct acpi_device_id *match; match = acpi_match_device(sp_acpi_match, &pdev->dev); if (match && match->driver_data) - return (struct sp_dev_vdata *)match->driver_data; + return (const struct sp_dev_vdata *)match->driver_data; return NULL; } @@ -123,7 +112,7 @@ static int sp_platform_probe(struct platform_device *pdev) goto e_err; sp->dev_specific = sp_platform; - sp->dev_vdata = pdev->dev.of_node ? sp_get_of_version(pdev) + sp->dev_vdata = pdev->dev.of_node ? of_device_get_match_data(&pdev->dev) : sp_get_acpi_version(pdev); if (!sp->dev_vdata) { ret = -ENODEV; diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c index 3963bb91321f..dc7e0cd51c25 100644 --- a/drivers/crypto/ccree/cc_buffer_mgr.c +++ b/drivers/crypto/ccree/cc_buffer_mgr.c @@ -1235,6 +1235,7 @@ int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, int rc = 0; u32 dummy = 0; u32 mapped_nents = 0; + int sg_nents; dev_dbg(dev, " update params : curr_buff=%p curr_buff_cnt=0x%X nbytes=0x%X src=%p curr_index=%u\n", curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index); @@ -1248,7 +1249,10 @@ int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, if (total_in_len < block_size) { dev_dbg(dev, " less than one block: curr_buff=%p *curr_buff_cnt=0x%X copy_to=%p\n", curr_buff, *curr_buff_cnt, &curr_buff[*curr_buff_cnt]); - areq_ctx->in_nents = sg_nents_for_len(src, nbytes); + sg_nents = sg_nents_for_len(src, nbytes); + if (sg_nents < 0) + return sg_nents; + areq_ctx->in_nents = sg_nents; sg_copy_to_buffer(src, areq_ctx->in_nents, &curr_buff[*curr_buff_cnt], nbytes); *curr_buff_cnt += nbytes; diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 5dd3f6a4781a..37294bb74003 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -4,9 +4,9 @@ config CRYPTO_DEV_CHELSIO depends on CHELSIO_T4 select CRYPTO_LIB_AES select CRYPTO_LIB_GF128MUL - select CRYPTO_SHA1 - select CRYPTO_SHA256 - select CRYPTO_SHA512 + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 select CRYPTO_AUTHENC help The Chelsio Crypto Co-processor driver for T6 adapters. diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index be21e4e2016c..22cbc343198a 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -51,7 +51,6 @@ #include <crypto/aes.h> #include <crypto/algapi.h> -#include <crypto/hash.h> #include <crypto/gcm.h> #include <crypto/sha1.h> #include <crypto/sha2.h> @@ -277,88 +276,60 @@ static void get_aes_decrypt_key(unsigned char *dec_key, } } -static struct crypto_shash *chcr_alloc_shash(unsigned int ds) +static int chcr_prepare_hmac_key(const u8 *raw_key, unsigned int raw_key_len, + int digestsize, void *istate, void *ostate) { - struct crypto_shash *base_hash = ERR_PTR(-EINVAL); - - switch (ds) { + __be32 *istate32 = istate, *ostate32 = ostate; + __be64 *istate64 = istate, *ostate64 = ostate; + union { + struct hmac_sha1_key sha1; + struct hmac_sha224_key sha224; + struct hmac_sha256_key sha256; + struct hmac_sha384_key sha384; + struct hmac_sha512_key sha512; + } k; + + switch (digestsize) { case SHA1_DIGEST_SIZE: - base_hash = crypto_alloc_shash("sha1", 0, 0); + hmac_sha1_preparekey(&k.sha1, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha1.istate.h); i++) { + istate32[i] = cpu_to_be32(k.sha1.istate.h[i]); + ostate32[i] = cpu_to_be32(k.sha1.ostate.h[i]); + } break; case SHA224_DIGEST_SIZE: - base_hash = crypto_alloc_shash("sha224", 0, 0); + hmac_sha224_preparekey(&k.sha224, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha224.key.istate.h); i++) { + istate32[i] = cpu_to_be32(k.sha224.key.istate.h[i]); + ostate32[i] = cpu_to_be32(k.sha224.key.ostate.h[i]); + } break; case SHA256_DIGEST_SIZE: - base_hash = crypto_alloc_shash("sha256", 0, 0); + hmac_sha256_preparekey(&k.sha256, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha256.key.istate.h); i++) { + istate32[i] = cpu_to_be32(k.sha256.key.istate.h[i]); + ostate32[i] = cpu_to_be32(k.sha256.key.ostate.h[i]); + } break; case SHA384_DIGEST_SIZE: - base_hash = crypto_alloc_shash("sha384", 0, 0); + hmac_sha384_preparekey(&k.sha384, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha384.key.istate.h); i++) { + istate64[i] = cpu_to_be64(k.sha384.key.istate.h[i]); + ostate64[i] = cpu_to_be64(k.sha384.key.ostate.h[i]); + } break; case SHA512_DIGEST_SIZE: - base_hash = crypto_alloc_shash("sha512", 0, 0); + hmac_sha512_preparekey(&k.sha512, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha512.key.istate.h); i++) { + istate64[i] = cpu_to_be64(k.sha512.key.istate.h[i]); + ostate64[i] = cpu_to_be64(k.sha512.key.ostate.h[i]); + } break; + default: + return -EINVAL; } - - return base_hash; -} - -static int chcr_compute_partial_hash(struct shash_desc *desc, - char *iopad, char *result_hash, - int digest_size) -{ - struct sha1_state sha1_st; - struct sha256_state sha256_st; - struct sha512_state sha512_st; - int error; - - if (digest_size == SHA1_DIGEST_SIZE) { - error = crypto_shash_init(desc) ?: - crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?: - crypto_shash_export_core(desc, &sha1_st); - memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE); - } else if (digest_size == SHA224_DIGEST_SIZE) { - error = crypto_shash_init(desc) ?: - crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: - crypto_shash_export_core(desc, &sha256_st); - memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); - - } else if (digest_size == SHA256_DIGEST_SIZE) { - error = crypto_shash_init(desc) ?: - crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: - crypto_shash_export_core(desc, &sha256_st); - memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); - - } else if (digest_size == SHA384_DIGEST_SIZE) { - error = crypto_shash_init(desc) ?: - crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: - crypto_shash_export_core(desc, &sha512_st); - memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); - - } else if (digest_size == SHA512_DIGEST_SIZE) { - error = crypto_shash_init(desc) ?: - crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: - crypto_shash_export_core(desc, &sha512_st); - memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); - } else { - error = -EINVAL; - pr_err("Unknown digest size %d\n", digest_size); - } - return error; -} - -static void chcr_change_order(char *buf, int ds) -{ - int i; - - if (ds == SHA512_DIGEST_SIZE) { - for (i = 0; i < (ds / sizeof(u64)); i++) - *((__be64 *)buf + i) = - cpu_to_be64(*((u64 *)buf + i)); - } else { - for (i = 0; i < (ds / sizeof(u32)); i++) - *((__be32 *)buf + i) = - cpu_to_be32(*((u32 *)buf + i)); - } + memzero_explicit(&k, sizeof(k)); + return 0; } static inline int is_hmac(struct crypto_tfm *tfm) @@ -1547,11 +1518,6 @@ static int get_alg_config(struct algo_param *params, return 0; } -static inline void chcr_free_shash(struct crypto_shash *base_hash) -{ - crypto_free_shash(base_hash); -} - /** * create_hash_wr - Create hash work request * @req: Cipher req base @@ -2202,53 +2168,13 @@ static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); - unsigned int digestsize = crypto_ahash_digestsize(tfm); - unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); - unsigned int i, err = 0, updated_digestsize; - - SHASH_DESC_ON_STACK(shash, hmacctx->base_hash); /* use the key to calculate the ipad and opad. ipad will sent with the * first request's data. opad will be sent with the final hash result * ipad in hmacctx->ipad and opad in hmacctx->opad location */ - shash->tfm = hmacctx->base_hash; - if (keylen > bs) { - err = crypto_shash_digest(shash, key, keylen, - hmacctx->ipad); - if (err) - goto out; - keylen = digestsize; - } else { - memcpy(hmacctx->ipad, key, keylen); - } - memset(hmacctx->ipad + keylen, 0, bs - keylen); - unsafe_memcpy(hmacctx->opad, hmacctx->ipad, bs, - "fortified memcpy causes -Wrestrict warning"); - - for (i = 0; i < bs / sizeof(int); i++) { - *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA; - *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA; - } - - updated_digestsize = digestsize; - if (digestsize == SHA224_DIGEST_SIZE) - updated_digestsize = SHA256_DIGEST_SIZE; - else if (digestsize == SHA384_DIGEST_SIZE) - updated_digestsize = SHA512_DIGEST_SIZE; - err = chcr_compute_partial_hash(shash, hmacctx->ipad, - hmacctx->ipad, digestsize); - if (err) - goto out; - chcr_change_order(hmacctx->ipad, updated_digestsize); - - err = chcr_compute_partial_hash(shash, hmacctx->opad, - hmacctx->opad, digestsize); - if (err) - goto out; - chcr_change_order(hmacctx->opad, updated_digestsize); -out: - return err; + return chcr_prepare_hmac_key(key, keylen, crypto_ahash_digestsize(tfm), + hmacctx->ipad, hmacctx->opad); } static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, @@ -2344,30 +2270,11 @@ static int chcr_hmac_init(struct ahash_request *areq) static int chcr_hmac_cra_init(struct crypto_tfm *tfm) { - struct chcr_context *ctx = crypto_tfm_ctx(tfm); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); - unsigned int digestsize = - crypto_ahash_digestsize(__crypto_ahash_cast(tfm)); - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct chcr_ahash_req_ctx)); - hmacctx->base_hash = chcr_alloc_shash(digestsize); - if (IS_ERR(hmacctx->base_hash)) - return PTR_ERR(hmacctx->base_hash); return chcr_device_init(crypto_tfm_ctx(tfm)); } -static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) -{ - struct chcr_context *ctx = crypto_tfm_ctx(tfm); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); - - if (hmacctx->base_hash) { - chcr_free_shash(hmacctx->base_hash); - hmacctx->base_hash = NULL; - } -} - inline void chcr_aead_common_exit(struct aead_request *req) { struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); @@ -3557,15 +3464,12 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); /* it contains auth and cipher key both*/ struct crypto_authenc_keys keys; - unsigned int bs, subtype; + unsigned int subtype; unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; - int err = 0, i, key_ctx_len = 0; + int err = 0, key_ctx_len = 0; unsigned char ck_size = 0; - unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; - struct crypto_shash *base_hash = ERR_PTR(-EINVAL); struct algo_param param; int align; - u8 *o_ptr = NULL; crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) @@ -3613,68 +3517,26 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, aeadctx->enckey_len << 3); } - base_hash = chcr_alloc_shash(max_authsize); - if (IS_ERR(base_hash)) { - pr_err("Base driver cannot be loaded\n"); + + align = KEYCTX_ALIGN_PAD(max_authsize); + err = chcr_prepare_hmac_key(keys.authkey, keys.authkeylen, max_authsize, + actx->h_iopad, + actx->h_iopad + param.result_size + align); + if (err) goto out; - } - { - SHASH_DESC_ON_STACK(shash, base_hash); - - shash->tfm = base_hash; - bs = crypto_shash_blocksize(base_hash); - align = KEYCTX_ALIGN_PAD(max_authsize); - o_ptr = actx->h_iopad + param.result_size + align; - - if (keys.authkeylen > bs) { - err = crypto_shash_digest(shash, keys.authkey, - keys.authkeylen, - o_ptr); - if (err) { - pr_err("Base driver cannot be loaded\n"); - goto out; - } - keys.authkeylen = max_authsize; - } else - memcpy(o_ptr, keys.authkey, keys.authkeylen); - - /* Compute the ipad-digest*/ - memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); - memcpy(pad, o_ptr, keys.authkeylen); - for (i = 0; i < bs >> 2; i++) - *((unsigned int *)pad + i) ^= IPAD_DATA; - - if (chcr_compute_partial_hash(shash, pad, actx->h_iopad, - max_authsize)) - goto out; - /* Compute the opad-digest */ - memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); - memcpy(pad, o_ptr, keys.authkeylen); - for (i = 0; i < bs >> 2; i++) - *((unsigned int *)pad + i) ^= OPAD_DATA; - if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize)) - goto out; + key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16) + + (param.result_size + align) * 2; + aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 0, 1, + key_ctx_len >> 4); + actx->auth_mode = param.auth_mode; + + memzero_explicit(&keys, sizeof(keys)); + return 0; - /* convert the ipad and opad digest to network order */ - chcr_change_order(actx->h_iopad, param.result_size); - chcr_change_order(o_ptr, param.result_size); - key_ctx_len = sizeof(struct _key_ctx) + - roundup(keys.enckeylen, 16) + - (param.result_size + align) * 2; - aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, - 0, 1, key_ctx_len >> 4); - actx->auth_mode = param.auth_mode; - chcr_free_shash(base_hash); - - memzero_explicit(&keys, sizeof(keys)); - return 0; - } out: aeadctx->enckey_len = 0; memzero_explicit(&keys, sizeof(keys)); - if (!IS_ERR(base_hash)) - chcr_free_shash(base_hash); return -EINVAL; } @@ -4490,7 +4352,6 @@ static int chcr_register_alg(void) if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { a_hash->halg.base.cra_init = chcr_hmac_cra_init; - a_hash->halg.base.cra_exit = chcr_hmac_cra_exit; a_hash->init = chcr_hmac_init; a_hash->setkey = chcr_ahash_setkey; a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 1d693b8436e6..e1e79e5f01e7 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -241,7 +241,6 @@ struct chcr_aead_ctx { }; struct hmac_ctx { - struct crypto_shash *base_hash; u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128]; u8 opad[CHCR_HASH_MAX_BLOCK_SIZE_128]; }; diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 925991526745..edf36f6add52 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -913,11 +913,10 @@ static void hifn_init_pll(struct hifn_device *dev) else pllcfg |= HIFN_PLL_REF_CLK_HBI; - if (hifn_pll_ref[3] != '\0') - freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); - else { + if (hifn_pll_ref[3] == '\0' || + kstrtouint(hifn_pll_ref + 3, 10, &freq)) { freq = 66; - dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", + dev_info(&dev->pdev->dev, "assuming %u MHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", freq, hifn_pll_ref); } diff --git a/drivers/crypto/hisilicon/debugfs.c b/drivers/crypto/hisilicon/debugfs.c index 45e130b901eb..17eb236e9ee4 100644 --- a/drivers/crypto/hisilicon/debugfs.c +++ b/drivers/crypto/hisilicon/debugfs.c @@ -888,6 +888,7 @@ static int qm_diff_regs_init(struct hisi_qm *qm, dfx_regs_uninit(qm, qm->debug.qm_diff_regs, ARRAY_SIZE(qm_diff_regs)); ret = PTR_ERR(qm->debug.acc_diff_regs); qm->debug.acc_diff_regs = NULL; + qm->debug.qm_diff_regs = NULL; return ret; } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index f5b47e5ff48a..b94fecd765ee 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -39,6 +39,7 @@ #define HPRE_HAC_RAS_NFE_ENB 0x301414 #define HPRE_HAC_RAS_FE_ENB 0x301418 #define HPRE_HAC_INT_SET 0x301500 +#define HPRE_AXI_ERROR_MASK GENMASK(21, 10) #define HPRE_RNG_TIMEOUT_NUM 0x301A34 #define HPRE_CORE_INT_ENABLE 0 #define HPRE_RDCHN_INI_ST 0x301a00 @@ -78,6 +79,11 @@ #define HPRE_PREFETCH_ENABLE (~(BIT(0) | BIT(30))) #define HPRE_PREFETCH_DISABLE BIT(30) #define HPRE_SVA_DISABLE_READY (BIT(4) | BIT(8)) +#define HPRE_SVA_PREFTCH_DFX4 0x301144 +#define HPRE_WAIT_SVA_READY 500000 +#define HPRE_READ_SVA_STATUS_TIMES 3 +#define HPRE_WAIT_US_MIN 10 +#define HPRE_WAIT_US_MAX 20 /* clock gate */ #define HPRE_CLKGATE_CTL 0x301a10 @@ -466,6 +472,33 @@ struct hisi_qp *hpre_create_qp(u8 type) return NULL; } +static int hpre_wait_sva_ready(struct hisi_qm *qm) +{ + u32 val, try_times = 0; + u8 count = 0; + + /* + * Read the register value every 10-20us. If the value is 0 for three + * consecutive times, the SVA module is ready. + */ + do { + val = readl(qm->io_base + HPRE_SVA_PREFTCH_DFX4); + if (val) + count = 0; + else if (++count == HPRE_READ_SVA_STATUS_TIMES) + break; + + usleep_range(HPRE_WAIT_US_MIN, HPRE_WAIT_US_MAX); + } while (++try_times < HPRE_WAIT_SVA_READY); + + if (try_times == HPRE_WAIT_SVA_READY) { + pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); + return -ETIMEDOUT; + } + + return 0; +} + static void hpre_config_pasid(struct hisi_qm *qm) { u32 val1, val2; @@ -563,7 +596,7 @@ static void disable_flr_of_bme(struct hisi_qm *qm) writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE); } -static void hpre_open_sva_prefetch(struct hisi_qm *qm) +static void hpre_close_sva_prefetch(struct hisi_qm *qm) { u32 val; int ret; @@ -571,20 +604,21 @@ static void hpre_open_sva_prefetch(struct hisi_qm *qm) if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; - /* Enable prefetch */ val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); - val &= HPRE_PREFETCH_ENABLE; + val |= HPRE_PREFETCH_DISABLE; writel(val, qm->io_base + HPRE_PREFETCH_CFG); - ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_PREFETCH_CFG, - val, !(val & HPRE_PREFETCH_DISABLE), + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_SVA_PREFTCH_DFX, + val, !(val & HPRE_SVA_DISABLE_READY), HPRE_REG_RD_INTVRL_US, HPRE_REG_RD_TMOUT_US); if (ret) - pci_err(qm->pdev, "failed to open sva prefetch\n"); + pci_err(qm->pdev, "failed to close sva prefetch\n"); + + (void)hpre_wait_sva_ready(qm); } -static void hpre_close_sva_prefetch(struct hisi_qm *qm) +static void hpre_open_sva_prefetch(struct hisi_qm *qm) { u32 val; int ret; @@ -592,16 +626,24 @@ static void hpre_close_sva_prefetch(struct hisi_qm *qm) if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; + /* Enable prefetch */ val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); - val |= HPRE_PREFETCH_DISABLE; + val &= HPRE_PREFETCH_ENABLE; writel(val, qm->io_base + HPRE_PREFETCH_CFG); - ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_SVA_PREFTCH_DFX, - val, !(val & HPRE_SVA_DISABLE_READY), + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_PREFETCH_CFG, + val, !(val & HPRE_PREFETCH_DISABLE), HPRE_REG_RD_INTVRL_US, HPRE_REG_RD_TMOUT_US); + if (ret) { + pci_err(qm->pdev, "failed to open sva prefetch\n"); + hpre_close_sva_prefetch(qm); + return; + } + + ret = hpre_wait_sva_ready(qm); if (ret) - pci_err(qm->pdev, "failed to close sva prefetch\n"); + hpre_close_sva_prefetch(qm); } static void hpre_enable_clock_gate(struct hisi_qm *qm) @@ -721,6 +763,7 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) /* Config data buffer pasid needed by Kunpeng 920 */ hpre_config_pasid(qm); + hpre_open_sva_prefetch(qm); hpre_enable_clock_gate(qm); @@ -756,8 +799,7 @@ static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); if (enable) { val1 |= HPRE_AM_OOO_SHUTDOWN_ENABLE; - val2 = hisi_qm_get_hw_info(qm, hpre_basic_info, - HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + val2 = qm->err_info.dev_err.shutdown_mask; } else { val1 &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE; val2 = 0x0; @@ -771,38 +813,33 @@ static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void hpre_hw_error_disable(struct hisi_qm *qm) { - u32 ce, nfe; - - ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); - nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; /* disable hpre hw error interrupts */ - writel(ce | nfe | HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_INT_MASK); + writel(err_mask, qm->io_base + HPRE_INT_MASK); /* disable HPRE block master OOO when nfe occurs on Kunpeng930 */ hpre_master_ooo_ctrl(qm, false); } static void hpre_hw_error_enable(struct hisi_qm *qm) { - u32 ce, nfe, err_en; - - ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); - nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; /* clear HPRE hw error source if having */ - writel(ce | nfe | HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_HAC_SOURCE_INT); + writel(err_mask, qm->io_base + HPRE_HAC_SOURCE_INT); /* configure error type */ - writel(ce, qm->io_base + HPRE_RAS_CE_ENB); - writel(nfe, qm->io_base + HPRE_RAS_NFE_ENB); - writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB); + writel(dev_err->ce, qm->io_base + HPRE_RAS_CE_ENB); + writel(dev_err->nfe, qm->io_base + HPRE_RAS_NFE_ENB); + writel(dev_err->fe, qm->io_base + HPRE_RAS_FE_ENB); /* enable HPRE block master OOO when nfe occurs on Kunpeng930 */ hpre_master_ooo_ctrl(qm, true); /* enable hpre hw error interrupts */ - err_en = ce | nfe | HPRE_HAC_RAS_FE_ENABLE; - writel(~err_en, qm->io_base + HPRE_INT_MASK); + writel(~err_mask, qm->io_base + HPRE_INT_MASK); } static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file) @@ -1171,7 +1208,7 @@ static int hpre_pre_store_cap_reg(struct hisi_qm *qm) size_t i, size; size = ARRAY_SIZE(hpre_cap_query_info); - hpre_cap = devm_kzalloc(dev, sizeof(*hpre_cap) * size, GFP_KERNEL); + hpre_cap = devm_kcalloc(dev, size, sizeof(*hpre_cap), GFP_KERNEL); if (!hpre_cap) return -ENOMEM; @@ -1357,12 +1394,20 @@ static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) static void hpre_disable_error_report(struct hisi_qm *qm, u32 err_type) { - u32 nfe_mask; + u32 nfe_mask = qm->err_info.dev_err.nfe; - nfe_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); writel(nfe_mask & (~err_type), qm->io_base + HPRE_RAS_NFE_ENB); } +static void hpre_enable_error_report(struct hisi_qm *qm) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + u32 ce_mask = qm->err_info.dev_err.ce; + + writel(nfe_mask, qm->io_base + HPRE_RAS_NFE_ENB); + writel(ce_mask, qm->io_base + HPRE_RAS_CE_ENB); +} + static void hpre_open_axi_master_ooo(struct hisi_qm *qm) { u32 value; @@ -1380,16 +1425,18 @@ static enum acc_err_result hpre_get_err_result(struct hisi_qm *qm) err_status = hpre_get_hw_err_status(qm); if (err_status) { - if (err_status & qm->err_info.ecc_2bits_mask) + if (err_status & qm->err_info.dev_err.ecc_2bits_mask) qm->err_status.is_dev_ecc_mbit = true; hpre_log_hw_error(qm, err_status); - if (err_status & qm->err_info.dev_reset_mask) { + if (err_status & qm->err_info.dev_err.reset_mask) { /* Disable the same error reporting until device is recovered. */ hpre_disable_error_report(qm, err_status); return ACC_ERR_NEED_RESET; } hpre_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + hpre_enable_error_report(qm); } return ACC_ERR_RECOVERED; @@ -1400,28 +1447,64 @@ static bool hpre_dev_is_abnormal(struct hisi_qm *qm) u32 err_status; err_status = hpre_get_hw_err_status(qm); - if (err_status & qm->err_info.dev_shutdown_mask) + if (err_status & qm->err_info.dev_err.shutdown_mask) return true; return false; } +static void hpre_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + u32 val; + + val = ~(err_mask & (~HPRE_AXI_ERROR_MASK)); + writel(val, qm->io_base + HPRE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask & (~HPRE_AXI_ERROR_MASK), + qm->io_base + HPRE_OOO_SHUTDOWN_SEL); +} + +static void hpre_enable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + /* clear axi error source */ + writel(HPRE_AXI_ERROR_MASK, qm->io_base + HPRE_HAC_SOURCE_INT); + + writel(~err_mask, qm->io_base + HPRE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask, qm->io_base + HPRE_OOO_SHUTDOWN_SEL); +} + static void hpre_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &err_info->qm_err; + struct hisi_qm_err_mask *dev_err = &err_info->dev_err; + + qm_err->fe = HPRE_HAC_RAS_FE_ENABLE; + qm_err->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_CE_MASK_CAP, qm->cap_ver); + qm_err->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_NFE_MASK_CAP, qm->cap_ver); + qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + qm_err->reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_QM_RESET_MASK_CAP, qm->cap_ver); + qm_err->ecc_2bits_mask = QM_ECC_MBIT; + + dev_err->fe = HPRE_HAC_RAS_FE_ENABLE; + dev_err->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); + dev_err->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + dev_err->reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_RESET_MASK_CAP, qm->cap_ver); + dev_err->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | HPRE_OOO_ECC_2BIT_ERR; - err_info->fe = HPRE_HAC_RAS_FE_ENABLE; - err_info->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_CE_MASK_CAP, qm->cap_ver); - err_info->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_NFE_MASK_CAP, qm->cap_ver); - err_info->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | HPRE_OOO_ECC_2BIT_ERR; - err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, - HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, - HPRE_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, - HPRE_QM_RESET_MASK_CAP, qm->cap_ver); - err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, - HPRE_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = HPRE_WR_MSI_PORT; err_info->acpi_rst = "HRST"; } @@ -1439,6 +1522,8 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .err_info_init = hpre_err_info_init, .get_err_result = hpre_get_err_result, .dev_is_abnormal = hpre_dev_is_abnormal, + .disable_axi_error = hpre_disable_axi_error, + .enable_axi_error = hpre_enable_axi_error, }; static int hpre_pf_probe_init(struct hpre *hpre) @@ -1450,8 +1535,6 @@ static int hpre_pf_probe_init(struct hpre *hpre) if (ret) return ret; - hpre_open_sva_prefetch(qm); - hisi_qm_dev_err_init(qm); ret = hpre_show_last_regs_init(qm); if (ret) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 2e4ee7ecfdfb..be25ecbdba69 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -45,6 +45,8 @@ #define QM_SQ_TYPE_MASK GENMASK(3, 0) #define QM_SQ_TAIL_IDX(sqc) ((le16_to_cpu((sqc).w11) >> 6) & 0x1) +#define QM_SQC_DISABLE_QP (1U << 6) +#define QM_XQC_RANDOM_DATA 0xaaaa /* cqc shift */ #define QM_CQ_HOP_NUM_SHIFT 0 @@ -62,10 +64,10 @@ #define QM_EQE_AEQE_SIZE (2UL << 12) #define QM_EQC_PHASE_SHIFT 16 -#define QM_EQE_PHASE(eqe) ((le32_to_cpu((eqe)->dw0) >> 16) & 0x1) +#define QM_EQE_PHASE(dw0) (((dw0) >> 16) & 0x1) #define QM_EQE_CQN_MASK GENMASK(15, 0) -#define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1) +#define QM_AEQE_PHASE(dw0) (((dw0) >> 16) & 0x1) #define QM_AEQE_TYPE_SHIFT 17 #define QM_AEQE_TYPE_MASK 0xf #define QM_AEQE_CQN_MASK GENMASK(15, 0) @@ -145,9 +147,9 @@ #define QM_RAS_CE_TIMES_PER_IRQ 1 #define QM_OOO_SHUTDOWN_SEL 0x1040f8 #define QM_AXI_RRESP_ERR BIT(0) -#define QM_ECC_MBIT BIT(2) #define QM_DB_TIMEOUT BIT(10) #define QM_OF_FIFO_OF BIT(11) +#define QM_RAS_AXI_ERROR (BIT(0) | BIT(1) | BIT(12)) #define QM_RESET_WAIT_TIMEOUT 400 #define QM_PEH_VENDOR_ID 0x1000d8 @@ -163,7 +165,6 @@ #define ACC_MASTER_TRANS_RETURN 0x300150 #define ACC_MASTER_GLOBAL_CTRL 0x300000 #define ACC_AM_CFG_PORT_WR_EN 0x30001c -#define QM_RAS_NFE_MBIT_DISABLE ~QM_ECC_MBIT #define ACC_AM_ROB_ECC_INT_STS 0x300104 #define ACC_ROB_ECC_ERR_MULTPL BIT(1) #define QM_MSI_CAP_ENABLE BIT(16) @@ -520,7 +521,7 @@ static bool qm_check_dev_error(struct hisi_qm *qm) return false; err_status = qm_get_hw_error_status(pf_qm); - if (err_status & pf_qm->err_info.qm_shutdown_mask) + if (err_status & pf_qm->err_info.qm_err.shutdown_mask) return true; if (pf_qm->err_ini->dev_is_abnormal) @@ -975,23 +976,23 @@ static void qm_get_complete_eqe_num(struct hisi_qm *qm) { struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; struct hisi_qm_poll_data *poll_data = NULL; + u32 dw0 = le32_to_cpu(eqe->dw0); u16 eq_depth = qm->eq_depth; u16 cqn, eqe_num = 0; - if (QM_EQE_PHASE(eqe) != qm->status.eqc_phase) { + if (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) { atomic64_inc(&qm->debug.dfx.err_irq_cnt); qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); return; } - cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; + cqn = dw0 & QM_EQE_CQN_MASK; if (unlikely(cqn >= qm->qp_num)) return; poll_data = &qm->poll_data[cqn]; - while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) { - cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; - poll_data->qp_finish_id[eqe_num] = cqn; + while (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) { + poll_data->qp_finish_id[eqe_num] = dw0 & QM_EQE_CQN_MASK; eqe_num++; if (qm->status.eq_head == eq_depth - 1) { @@ -1005,6 +1006,8 @@ static void qm_get_complete_eqe_num(struct hisi_qm *qm) if (eqe_num == (eq_depth >> 1) - 1) break; + + dw0 = le32_to_cpu(eqe->dw0); } poll_data->eqe_num = eqe_num; @@ -1097,15 +1100,15 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) { struct hisi_qm *qm = data; struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; + u32 dw0 = le32_to_cpu(aeqe->dw0); u16 aeq_depth = qm->aeq_depth; u32 type, qp_id; atomic64_inc(&qm->debug.dfx.aeq_irq_cnt); - while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) { - type = (le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT) & - QM_AEQE_TYPE_MASK; - qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK; + while (QM_AEQE_PHASE(dw0) == qm->status.aeqc_phase) { + type = (dw0 >> QM_AEQE_TYPE_SHIFT) & QM_AEQE_TYPE_MASK; + qp_id = dw0 & QM_AEQE_CQN_MASK; switch (type) { case QM_EQ_OVERFLOW: @@ -1133,6 +1136,7 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) aeqe++; qm->status.aeq_head++; } + dw0 = le32_to_cpu(aeqe->dw0); } qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); @@ -1282,6 +1286,13 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT); } break; + /* + * Note: The current logic only needs to handle the above three types + * If new types are added, they need to be supplemented here, + * otherwise undefined behavior may occur. + */ + default: + break; } } @@ -1395,17 +1406,17 @@ static void qm_hw_error_init_v1(struct hisi_qm *qm) static void qm_hw_error_cfg(struct hisi_qm *qm) { - struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; - qm->error_mask = err_info->nfe | err_info->ce | err_info->fe; + qm->error_mask = qm_err->nfe | qm_err->ce | qm_err->fe; /* clear QM hw residual error source */ writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); /* configure error type */ - writel(err_info->ce, qm->io_base + QM_RAS_CE_ENABLE); + writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); - writel(err_info->nfe, qm->io_base + QM_RAS_NFE_ENABLE); - writel(err_info->fe, qm->io_base + QM_RAS_FE_ENABLE); + writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); + writel(qm_err->fe, qm->io_base + QM_RAS_FE_ENABLE); } static void qm_hw_error_init_v2(struct hisi_qm *qm) @@ -1434,7 +1445,7 @@ static void qm_hw_error_init_v3(struct hisi_qm *qm) qm_hw_error_cfg(qm); /* enable close master ooo when hardware error happened */ - writel(qm->err_info.qm_shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); + writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); irq_unmask = ~qm->error_mask; irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); @@ -1496,6 +1507,7 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) { + struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; u32 error_status; error_status = qm_get_hw_error_status(qm); @@ -1504,17 +1516,16 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) qm->err_status.is_qm_ecc_mbit = true; qm_log_hw_error(qm, error_status); - if (error_status & qm->err_info.qm_reset_mask) { + if (error_status & qm_err->reset_mask) { /* Disable the same error reporting until device is recovered. */ - writel(qm->err_info.nfe & (~error_status), - qm->io_base + QM_RAS_NFE_ENABLE); + writel(qm_err->nfe & (~error_status), qm->io_base + QM_RAS_NFE_ENABLE); return ACC_ERR_NEED_RESET; } /* Clear error source if not need reset. */ writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); - writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE); - writel(qm->err_info.ce, qm->io_base + QM_RAS_CE_ENABLE); + writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); + writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); } return ACC_ERR_RECOVERED; @@ -2651,10 +2662,10 @@ static int qm_hw_err_isolate(struct hisi_qm *qm) } } list_add(&hw_err->list, &isolate->qm_hw_errs); - mutex_unlock(&isolate->isolate_lock); if (count >= isolate->err_threshold) isolate->is_isolate = true; + mutex_unlock(&isolate->isolate_lock); return 0; } @@ -2663,12 +2674,10 @@ static void qm_hw_err_destroy(struct hisi_qm *qm) { struct qm_hw_err *err, *tmp; - mutex_lock(&qm->isolate_data.isolate_lock); list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) { list_del(&err->list); kfree(err); } - mutex_unlock(&qm->isolate_data.isolate_lock); } static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce) @@ -2696,10 +2705,12 @@ static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num) if (qm->isolate_data.is_isolate) return -EPERM; + mutex_lock(&qm->isolate_data.isolate_lock); qm->isolate_data.err_threshold = num; /* After the policy is updated, need to reset the hardware err list */ qm_hw_err_destroy(qm); + mutex_unlock(&qm->isolate_data.isolate_lock); return 0; } @@ -2736,12 +2747,36 @@ static void qm_remove_uacce(struct hisi_qm *qm) struct uacce_device *uacce = qm->uacce; if (qm->use_sva) { + mutex_lock(&qm->isolate_data.isolate_lock); qm_hw_err_destroy(qm); + mutex_unlock(&qm->isolate_data.isolate_lock); + uacce_remove(uacce); qm->uacce = NULL; } } +static void qm_uacce_api_ver_init(struct hisi_qm *qm) +{ + struct uacce_device *uacce = qm->uacce; + + switch (qm->ver) { + case QM_HW_V1: + uacce->api_ver = HISI_QM_API_VER_BASE; + break; + case QM_HW_V2: + uacce->api_ver = HISI_QM_API_VER2_BASE; + break; + case QM_HW_V3: + case QM_HW_V4: + uacce->api_ver = HISI_QM_API_VER3_BASE; + break; + default: + uacce->api_ver = HISI_QM_API_VER5_BASE; + break; + } +} + static int qm_alloc_uacce(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -2776,13 +2811,6 @@ static int qm_alloc_uacce(struct hisi_qm *qm) uacce->priv = qm; if (qm->ver == QM_HW_V1) - uacce->api_ver = HISI_QM_API_VER_BASE; - else if (qm->ver == QM_HW_V2) - uacce->api_ver = HISI_QM_API_VER2_BASE; - else - uacce->api_ver = HISI_QM_API_VER3_BASE; - - if (qm->ver == QM_HW_V1) mmio_page_nr = QM_DOORBELL_PAGE_NR; else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) mmio_page_nr = QM_DOORBELL_PAGE_NR + @@ -2801,6 +2829,7 @@ static int qm_alloc_uacce(struct hisi_qm *qm) uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr; qm->uacce = uacce; + qm_uacce_api_ver_init(qm); INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs); mutex_init(&qm->isolate_data.isolate_lock); @@ -3179,6 +3208,9 @@ static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) qm_init_eq_aeq_status(qm); + /* Before starting the dev, clear the memory and then configure to device using. */ + memset(qm->qdma.va, 0, qm->qdma.size); + ret = qm_eq_ctx_cfg(qm); if (ret) { dev_err(dev, "Set eqc failed!\n"); @@ -3190,9 +3222,13 @@ static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) static int __hisi_qm_start(struct hisi_qm *qm) { + struct device *dev = &qm->pdev->dev; int ret; - WARN_ON(!qm->qdma.va); + if (!qm->qdma.va) { + dev_err(dev, "qm qdma is NULL!\n"); + return -EINVAL; + } if (qm->fun_type == QM_HW_PF) { ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num); @@ -3266,7 +3302,7 @@ static int qm_restart(struct hisi_qm *qm) for (i = 0; i < qm->qp_num; i++) { qp = &qm->qp_array[i]; if (atomic_read(&qp->qp_status.flags) == QP_STOP && - qp->is_resetting == true) { + qp->is_resetting == true && qp->is_in_kernel == true) { ret = qm_start_qp_nolock(qp, 0); if (ret < 0) { dev_err(dev, "Failed to start qp%d!\n", i); @@ -3298,24 +3334,44 @@ static void qm_stop_started_qp(struct hisi_qm *qm) } /** - * qm_clear_queues() - Clear all queues memory in a qm. - * @qm: The qm in which the queues will be cleared. + * qm_invalid_queues() - invalid all queues in use. + * @qm: The qm in which the queues will be invalidated. * - * This function clears all queues memory in a qm. Reset of accelerator can - * use this to clear queues. + * This function invalid all queues in use. If the doorbell command is sent + * to device in user space after the device is reset, the device discards + * the doorbell command. */ -static void qm_clear_queues(struct hisi_qm *qm) +static void qm_invalid_queues(struct hisi_qm *qm) { struct hisi_qp *qp; + struct qm_sqc *sqc; + struct qm_cqc *cqc; int i; + /* + * Normal stop queues is no longer used and does not need to be + * invalid queues. + */ + if (qm->status.stop_reason == QM_NORMAL) + return; + + if (qm->status.stop_reason == QM_DOWN) + hisi_qm_cache_wb(qm); + for (i = 0; i < qm->qp_num; i++) { qp = &qm->qp_array[i]; - if (qp->is_in_kernel && qp->is_resetting) + if (!qp->is_resetting) + continue; + + /* Modify random data and set sqc close bit to invalid queue. */ + sqc = qm->sqc + i; + cqc = qm->cqc + i; + sqc->w8 = cpu_to_le16(QM_XQC_RANDOM_DATA); + sqc->w13 = cpu_to_le16(QM_SQC_DISABLE_QP); + cqc->w8 = cpu_to_le16(QM_XQC_RANDOM_DATA); + if (qp->is_in_kernel) memset(qp->qdma.va, 0, qp->qdma.size); } - - memset(qm->qdma.va, 0, qm->qdma.size); } /** @@ -3372,7 +3428,7 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) } } - qm_clear_queues(qm); + qm_invalid_queues(qm); qm->status.stop_reason = QM_NORMAL; err_unlock: @@ -3617,24 +3673,25 @@ static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs) return 0; } -static int qm_clear_vft_config(struct hisi_qm *qm) +static void qm_clear_vft_config(struct hisi_qm *qm) { - int ret; u32 i; - for (i = 1; i <= qm->vfs_num; i++) { - ret = hisi_qm_set_vft(qm, i, 0, 0); - if (ret) - return ret; - } - qm->vfs_num = 0; + /* + * When disabling SR-IOV, clear the configuration of each VF in the hardware + * sequentially. Failure to clear a single VF should not affect the clearing + * operation of other VFs. + */ + for (i = 1; i <= qm->vfs_num; i++) + (void)hisi_qm_set_vft(qm, i, 0, 0); - return 0; + qm->vfs_num = 0; } static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) { struct device *dev = &qm->pdev->dev; + struct qm_shaper_factor t_factor; u32 ir = qos * QM_QOS_RATE; int ret, total_vfs, i; @@ -3642,6 +3699,7 @@ static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) if (fun_index > total_vfs) return -EINVAL; + memcpy(&t_factor, &qm->factor[fun_index], sizeof(t_factor)); qm->factor[fun_index].func_qos = qos; ret = qm_get_shaper_para(ir, &qm->factor[fun_index]); @@ -3655,11 +3713,21 @@ static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); if (ret) { dev_err(dev, "type: %d, failed to set shaper vft!\n", i); - return -EINVAL; + goto back_func_qos; } } return 0; + +back_func_qos: + memcpy(&qm->factor[fun_index], &t_factor, sizeof(t_factor)); + for (i--; i >= ALG_TYPE_0; i--) { + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); + if (ret) + dev_err(dev, "failed to restore shaper vft during rollback!\n"); + } + + return -EINVAL; } static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) @@ -3826,8 +3894,14 @@ static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, } pdev = container_of(dev, struct pci_dev, dev); + if (pci_physfn(pdev) != qm->pdev) { + pci_err(qm->pdev, "the pdev input does not match the pf!\n"); + put_device(dev); + return -EINVAL; + } *fun_index = pdev->devfn; + put_device(dev); return 0; } @@ -3960,13 +4034,13 @@ int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs) goto err_put_sync; } + qm->vfs_num = num_vfs; ret = pci_enable_sriov(pdev, num_vfs); if (ret) { pci_err(pdev, "Can't enable VF!\n"); qm_clear_vft_config(qm); goto err_put_sync; } - qm->vfs_num = num_vfs; pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs); @@ -4001,11 +4075,10 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) } pci_disable_sriov(pdev); - - qm->vfs_num = 0; + qm_clear_vft_config(qm); qm_pm_put_sync(qm); - return qm_clear_vft_config(qm); + return 0; } EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); @@ -4179,9 +4252,9 @@ static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm) !qm->err_status.is_qm_ecc_mbit && !qm->err_ini->close_axi_master_ooo) { nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE); - writel(nfe_enb & QM_RAS_NFE_MBIT_DISABLE, + writel(nfe_enb & ~qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_RAS_NFE_ENABLE); - writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET); + writel(qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_ABNORMAL_INT_SET); } } @@ -4447,9 +4520,6 @@ static void qm_restart_prepare(struct hisi_qm *qm) { u32 value; - if (qm->err_ini->open_sva_prefetch) - qm->err_ini->open_sva_prefetch(qm); - if (qm->ver >= QM_HW_V3) return; @@ -4463,12 +4533,12 @@ static void qm_restart_prepare(struct hisi_qm *qm) qm->io_base + ACC_AM_CFG_PORT_WR_EN); /* clear dev ecc 2bit error source if having */ - value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask; + value = qm_get_dev_err_status(qm) & qm->err_info.dev_err.ecc_2bits_mask; if (value && qm->err_ini->clear_dev_hw_err_status) qm->err_ini->clear_dev_hw_err_status(qm, value); /* clear QM ecc mbit error source */ - writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); /* clear AM Reorder Buffer ecc mbit source */ writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS); @@ -4495,6 +4565,34 @@ clear_flags: qm->err_status.is_dev_ecc_mbit = false; } +static void qm_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; + u32 val; + + val = ~(qm->error_mask & (~QM_RAS_AXI_ERROR)); + writel(val, qm->io_base + QM_ABNORMAL_INT_MASK); + if (qm->ver > QM_HW_V2) + writel(qm_err->shutdown_mask & (~QM_RAS_AXI_ERROR), + qm->io_base + QM_OOO_SHUTDOWN_SEL); + + if (qm->err_ini->disable_axi_error) + qm->err_ini->disable_axi_error(qm); +} + +static void qm_enable_axi_error(struct hisi_qm *qm) +{ + /* clear axi error source */ + writel(QM_RAS_AXI_ERROR, qm->io_base + QM_ABNORMAL_INT_SOURCE); + + writel(~qm->error_mask, qm->io_base + QM_ABNORMAL_INT_MASK); + if (qm->ver > QM_HW_V2) + writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); + + if (qm->err_ini->enable_axi_error) + qm->err_ini->enable_axi_error(qm); +} + static int qm_controller_reset_done(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -4528,6 +4626,7 @@ static int qm_controller_reset_done(struct hisi_qm *qm) qm_restart_prepare(qm); hisi_qm_dev_err_init(qm); + qm_disable_axi_error(qm); if (qm->err_ini->open_axi_master_ooo) qm->err_ini->open_axi_master_ooo(qm); @@ -4550,7 +4649,7 @@ static int qm_controller_reset_done(struct hisi_qm *qm) ret = qm_wait_vf_prepare_finish(qm); if (ret) pci_err(pdev, "failed to start by vfs in soft reset!\n"); - + qm_enable_axi_error(qm); qm_cmd_init(qm); qm_restart_done(qm); @@ -4731,6 +4830,15 @@ flr_done: } EXPORT_SYMBOL_GPL(hisi_qm_reset_done); +static irqreturn_t qm_rsvd_irq(int irq, void *data) +{ + struct hisi_qm *qm = data; + + dev_info(&qm->pdev->dev, "Reserved interrupt, ignore!\n"); + + return IRQ_HANDLED; +} + static irqreturn_t qm_abnormal_irq(int irq, void *data) { struct hisi_qm *qm = data; @@ -4760,8 +4868,6 @@ void hisi_qm_dev_shutdown(struct pci_dev *pdev) ret = hisi_qm_stop(qm, QM_DOWN); if (ret) dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); - - hisi_qm_cache_wb(qm); } EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown); @@ -5014,7 +5120,7 @@ static void qm_unregister_abnormal_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val; - if (qm->fun_type == QM_HW_VF) + if (qm->fun_type == QM_HW_VF && qm->ver < QM_HW_V3) return; val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; @@ -5031,17 +5137,28 @@ static int qm_register_abnormal_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret; - if (qm->fun_type == QM_HW_VF) - return 0; - val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) return 0; - irq_vector = val & QM_IRQ_VECTOR_MASK; + + /* For VF, this is a reserved interrupt in V3 version. */ + if (qm->fun_type == QM_HW_VF) { + if (qm->ver < QM_HW_V3) + return 0; + + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_rsvd_irq, + IRQF_NO_AUTOEN, qm->dev_name, qm); + if (ret) { + dev_err(&pdev->dev, "failed to request reserved irq, ret = %d!\n", ret); + return ret; + } + return 0; + } + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm); if (ret) - dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d", ret); + dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d!\n", ret); return ret; } @@ -5407,6 +5524,12 @@ static int hisi_qm_pci_init(struct hisi_qm *qm) pci_set_master(pdev); num_vec = qm_get_irq_num(qm); + if (!num_vec) { + dev_err(dev, "Device irq num is zero!\n"); + ret = -EINVAL; + goto err_get_pci_res; + } + num_vec = roundup_pow_of_two(num_vec); ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI); if (ret < 0) { dev_err(dev, "Failed to enable MSI vectors!\n"); diff --git a/drivers/crypto/hisilicon/sec/sec_drv.c b/drivers/crypto/hisilicon/sec/sec_drv.c index ef0cb733c92c..129cb6faa0b7 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.c +++ b/drivers/crypto/hisilicon/sec/sec_drv.c @@ -922,7 +922,8 @@ static int sec_hw_init(struct sec_dev_info *info) struct iommu_domain *domain; u32 sec_ipv4_mask = 0; u32 sec_ipv6_mask[10] = {}; - u32 i, ret; + int ret; + u32 i; domain = iommu_get_domain_for_dev(info->dev); diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index d044ded0f290..31590d01139a 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -1944,14 +1944,12 @@ static void sec_request_uninit(struct sec_req *req) static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) { struct sec_qp_ctx *qp_ctx; - int i; + int i = 0; - for (i = 0; i < ctx->sec->ctx_q_num; i++) { + do { qp_ctx = &ctx->qp_ctx[i]; req->req_id = sec_alloc_req_id(req, qp_ctx); - if (req->req_id >= 0) - break; - } + } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num); req->qp_ctx = qp_ctx; req->backlog = &qp_ctx->backlog; diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 72cf48d1f3ab..5eb2d6820742 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -47,6 +47,8 @@ #define SEC_RAS_FE_ENB_MSK 0x0 #define SEC_OOO_SHUTDOWN_SEL 0x301014 #define SEC_RAS_DISABLE 0x0 +#define SEC_AXI_ERROR_MASK (BIT(0) | BIT(1)) + #define SEC_MEM_START_INIT_REG 0x301100 #define SEC_MEM_INIT_DONE_REG 0x301104 @@ -93,6 +95,16 @@ #define SEC_PREFETCH_ENABLE (~(BIT(0) | BIT(1) | BIT(11))) #define SEC_PREFETCH_DISABLE BIT(1) #define SEC_SVA_DISABLE_READY (BIT(7) | BIT(11)) +#define SEC_SVA_PREFETCH_INFO 0x301ED4 +#define SEC_SVA_STALL_NUM GENMASK(23, 8) +#define SEC_SVA_PREFETCH_NUM GENMASK(2, 0) +#define SEC_WAIT_SVA_READY 500000 +#define SEC_READ_SVA_STATUS_TIMES 3 +#define SEC_WAIT_US_MIN 10 +#define SEC_WAIT_US_MAX 20 +#define SEC_WAIT_QP_US_MIN 1000 +#define SEC_WAIT_QP_US_MAX 2000 +#define SEC_MAX_WAIT_TIMES 2000 #define SEC_DELAY_10_US 10 #define SEC_POLL_TIMEOUT_US 1000 @@ -464,6 +476,81 @@ static void sec_set_endian(struct hisi_qm *qm) writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); } +static int sec_wait_sva_ready(struct hisi_qm *qm, __u32 offset, __u32 mask) +{ + u32 val, try_times = 0; + u8 count = 0; + + /* + * Read the register value every 10-20us. If the value is 0 for three + * consecutive times, the SVA module is ready. + */ + do { + val = readl(qm->io_base + offset); + if (val & mask) + count = 0; + else if (++count == SEC_READ_SVA_STATUS_TIMES) + break; + + usleep_range(SEC_WAIT_US_MIN, SEC_WAIT_US_MAX); + } while (++try_times < SEC_WAIT_SVA_READY); + + if (try_times == SEC_WAIT_SVA_READY) { + pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static void sec_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val |= SEC_PREFETCH_DISABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, + val, !(val & SEC_SVA_DISABLE_READY), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); + + (void)sec_wait_sva_ready(qm, SEC_SVA_PREFETCH_INFO, SEC_SVA_STALL_NUM); +} + +static void sec_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val &= SEC_PREFETCH_ENABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, + val, !(val & SEC_PREFETCH_DISABLE), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) { + pci_err(qm->pdev, "failed to open sva prefetch\n"); + sec_close_sva_prefetch(qm); + return; + } + + ret = sec_wait_sva_ready(qm, SEC_SVA_TRANS, SEC_SVA_PREFETCH_NUM); + if (ret) + sec_close_sva_prefetch(qm); +} + static void sec_engine_sva_config(struct hisi_qm *qm) { u32 reg; @@ -497,45 +584,7 @@ static void sec_engine_sva_config(struct hisi_qm *qm) writel_relaxed(reg, qm->io_base + SEC_INTERFACE_USER_CTRL1_REG); } -} - -static void sec_open_sva_prefetch(struct hisi_qm *qm) -{ - u32 val; - int ret; - - if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) - return; - - /* Enable prefetch */ - val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); - val &= SEC_PREFETCH_ENABLE; - writel(val, qm->io_base + SEC_PREFETCH_CFG); - - ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, - val, !(val & SEC_PREFETCH_DISABLE), - SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); - if (ret) - pci_err(qm->pdev, "failed to open sva prefetch\n"); -} - -static void sec_close_sva_prefetch(struct hisi_qm *qm) -{ - u32 val; - int ret; - - if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) - return; - - val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); - val |= SEC_PREFETCH_DISABLE; - writel(val, qm->io_base + SEC_PREFETCH_CFG); - - ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, - val, !(val & SEC_SVA_DISABLE_READY), - SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); - if (ret) - pci_err(qm->pdev, "failed to close sva prefetch\n"); + sec_open_sva_prefetch(qm); } static void sec_enable_clock_gate(struct hisi_qm *qm) @@ -666,8 +715,7 @@ static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + SEC_CONTROL_REG); if (enable) { val1 |= SEC_AXI_SHUTDOWN_ENABLE; - val2 = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + val2 = qm->err_info.dev_err.shutdown_mask; } else { val1 &= SEC_AXI_SHUTDOWN_DISABLE; val2 = 0x0; @@ -681,7 +729,8 @@ static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void sec_hw_error_enable(struct hisi_qm *qm) { - u32 ce, nfe; + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; if (qm->ver == QM_HW_V1) { writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); @@ -689,22 +738,19 @@ static void sec_hw_error_enable(struct hisi_qm *qm) return; } - ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver); - nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); - /* clear SEC hw error source if having */ - writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_SOURCE); + writel(err_mask, qm->io_base + SEC_CORE_INT_SOURCE); /* enable RAS int */ - writel(ce, qm->io_base + SEC_RAS_CE_REG); - writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG); - writel(nfe, qm->io_base + SEC_RAS_NFE_REG); + writel(dev_err->ce, qm->io_base + SEC_RAS_CE_REG); + writel(dev_err->fe, qm->io_base + SEC_RAS_FE_REG); + writel(dev_err->nfe, qm->io_base + SEC_RAS_NFE_REG); /* enable SEC block master OOO when nfe occurs on Kunpeng930 */ sec_master_ooo_ctrl(qm, true); /* enable SEC hw error interrupts */ - writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_MASK); + writel(err_mask, qm->io_base + SEC_CORE_INT_MASK); } static void sec_hw_error_disable(struct hisi_qm *qm) @@ -1061,12 +1107,20 @@ static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) static void sec_disable_error_report(struct hisi_qm *qm, u32 err_type) { - u32 nfe_mask; + u32 nfe_mask = qm->err_info.dev_err.nfe; - nfe_mask = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); writel(nfe_mask & (~err_type), qm->io_base + SEC_RAS_NFE_REG); } +static void sec_enable_error_report(struct hisi_qm *qm) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + u32 ce_mask = qm->err_info.dev_err.ce; + + writel(nfe_mask, qm->io_base + SEC_RAS_NFE_REG); + writel(ce_mask, qm->io_base + SEC_RAS_CE_REG); +} + static void sec_open_axi_master_ooo(struct hisi_qm *qm) { u32 val; @@ -1082,16 +1136,18 @@ static enum acc_err_result sec_get_err_result(struct hisi_qm *qm) err_status = sec_get_hw_err_status(qm); if (err_status) { - if (err_status & qm->err_info.ecc_2bits_mask) + if (err_status & qm->err_info.dev_err.ecc_2bits_mask) qm->err_status.is_dev_ecc_mbit = true; sec_log_hw_error(qm, err_status); - if (err_status & qm->err_info.dev_reset_mask) { + if (err_status & qm->err_info.dev_err.reset_mask) { /* Disable the same error reporting until device is recovered. */ sec_disable_error_report(qm, err_status); return ACC_ERR_NEED_RESET; } sec_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + sec_enable_error_report(qm); } return ACC_ERR_RECOVERED; @@ -1102,28 +1158,62 @@ static bool sec_dev_is_abnormal(struct hisi_qm *qm) u32 err_status; err_status = sec_get_hw_err_status(qm); - if (err_status & qm->err_info.dev_shutdown_mask) + if (err_status & qm->err_info.dev_err.shutdown_mask) return true; return false; } +static void sec_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + writel(err_mask & ~SEC_AXI_ERROR_MASK, qm->io_base + SEC_CORE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask & (~SEC_AXI_ERROR_MASK), + qm->io_base + SEC_OOO_SHUTDOWN_SEL); +} + +static void sec_enable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + /* clear axi error source */ + writel(SEC_AXI_ERROR_MASK, qm->io_base + SEC_CORE_INT_SOURCE); + + writel(err_mask, qm->io_base + SEC_CORE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask, qm->io_base + SEC_OOO_SHUTDOWN_SEL); +} + static void sec_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &err_info->qm_err; + struct hisi_qm_err_mask *dev_err = &err_info->dev_err; + + qm_err->fe = SEC_RAS_FE_ENB_MSK; + qm_err->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver); + qm_err->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver); + qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + qm_err->reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_QM_RESET_MASK_CAP, qm->cap_ver); + qm_err->ecc_2bits_mask = QM_ECC_MBIT; + + dev_err->fe = SEC_RAS_FE_ENB_MSK; + dev_err->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver); + dev_err->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); + dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + dev_err->reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_RESET_MASK_CAP, qm->cap_ver); + dev_err->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; - err_info->fe = SEC_RAS_FE_ENB_MSK; - err_info->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver); - err_info->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver); - err_info->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; - err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_QM_RESET_MASK_CAP, qm->cap_ver); - err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = BIT(0); err_info->acpi_rst = "SRST"; } @@ -1141,6 +1231,8 @@ static const struct hisi_qm_err_ini sec_err_ini = { .err_info_init = sec_err_info_init, .get_err_result = sec_get_err_result, .dev_is_abnormal = sec_dev_is_abnormal, + .disable_axi_error = sec_disable_axi_error, + .enable_axi_error = sec_enable_axi_error, }; static int sec_pf_probe_init(struct sec_dev *sec) @@ -1152,7 +1244,6 @@ static int sec_pf_probe_init(struct sec_dev *sec) if (ret) return ret; - sec_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); sec_debug_regs_clear(qm); ret = sec_show_last_regs_init(qm); @@ -1169,7 +1260,7 @@ static int sec_pre_store_cap_reg(struct hisi_qm *qm) size_t i, size; size = ARRAY_SIZE(sec_cap_query_info); - sec_cap = devm_kzalloc(&pdev->dev, sizeof(*sec_cap) * size, GFP_KERNEL); + sec_cap = devm_kcalloc(&pdev->dev, size, sizeof(*sec_cap), GFP_KERNEL); if (!sec_cap) return -ENOMEM; diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 7a9ef2a9972a..24c7b6ab285b 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -245,11 +245,6 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sgl, } curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma); - if (IS_ERR(curr_hw_sgl)) { - dev_err(dev, "Get SGL error!\n"); - ret = -ENOMEM; - goto err_unmap; - } curr_hw_sgl->entry_length_in_sgl = cpu_to_le16(pool->sge_nr); curr_hw_sge = curr_hw_sgl->sge_entries; diff --git a/drivers/crypto/hisilicon/zip/dae_main.c b/drivers/crypto/hisilicon/zip/dae_main.c index 6f22e4c36e49..68aebd02fc84 100644 --- a/drivers/crypto/hisilicon/zip/dae_main.c +++ b/drivers/crypto/hisilicon/zip/dae_main.c @@ -15,6 +15,7 @@ #define DAE_REG_RD_TMOUT_US USEC_PER_SEC #define DAE_ALG_NAME "hashagg" +#define DAE_V5_ALG_NAME "hashagg\nudma\nhashjoin\ngather" /* error */ #define DAE_AXI_CFG_OFFSET 0x331000 @@ -82,6 +83,7 @@ int hisi_dae_set_user_domain(struct hisi_qm *qm) int hisi_dae_set_alg(struct hisi_qm *qm) { + const char *alg_name; size_t len; if (!dae_is_support(qm)) @@ -90,9 +92,14 @@ int hisi_dae_set_alg(struct hisi_qm *qm) if (!qm->uacce) return 0; + if (qm->ver >= QM_HW_V5) + alg_name = DAE_V5_ALG_NAME; + else + alg_name = DAE_ALG_NAME; + len = strlen(qm->uacce->algs); /* A line break may be required */ - if (len + strlen(DAE_ALG_NAME) + 1 >= QM_DEV_ALG_MAX_LEN) { + if (len + strlen(alg_name) + 1 >= QM_DEV_ALG_MAX_LEN) { pci_err(qm->pdev, "algorithm name is too long!\n"); return -EINVAL; } @@ -100,7 +107,7 @@ int hisi_dae_set_alg(struct hisi_qm *qm) if (len) strcat((char *)qm->uacce->algs, "\n"); - strcat((char *)qm->uacce->algs, DAE_ALG_NAME); + strcat((char *)qm->uacce->algs, alg_name); return 0; } @@ -168,6 +175,12 @@ static void hisi_dae_disable_error_report(struct hisi_qm *qm, u32 err_type) writel(DAE_ERR_NFE_MASK & (~err_type), qm->io_base + DAE_ERR_NFE_OFFSET); } +static void hisi_dae_enable_error_report(struct hisi_qm *qm) +{ + writel(DAE_ERR_CE_MASK, qm->io_base + DAE_ERR_CE_OFFSET); + writel(DAE_ERR_NFE_MASK, qm->io_base + DAE_ERR_NFE_OFFSET); +} + static void hisi_dae_log_hw_error(struct hisi_qm *qm, u32 err_type) { const struct hisi_dae_hw_error *err = dae_hw_error; @@ -209,6 +222,8 @@ enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm) return ACC_ERR_NEED_RESET; } hisi_dae_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + hisi_dae_enable_error_report(qm); return ACC_ERR_RECOVERED; } diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index d8ba23b7cc7d..4fcbe6bada06 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -65,6 +65,7 @@ #define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 #define HZIP_CORE_INT_MASK_ALL GENMASK(12, 0) +#define HZIP_AXI_ERROR_MASK (BIT(2) | BIT(3)) #define HZIP_SQE_SIZE 128 #define HZIP_PF_DEF_Q_NUM 64 #define HZIP_PF_DEF_Q_BASE 0 @@ -80,6 +81,7 @@ #define HZIP_ALG_GZIP_BIT GENMASK(3, 2) #define HZIP_ALG_DEFLATE_BIT GENMASK(5, 4) #define HZIP_ALG_LZ77_BIT GENMASK(7, 6) +#define HZIP_ALG_LZ4_BIT GENMASK(9, 8) #define HZIP_BUF_SIZE 22 #define HZIP_SQE_MASK_OFFSET 64 @@ -95,10 +97,16 @@ #define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0))) #define HZIP_SVA_PREFETCH_DISABLE BIT(26) #define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30)) +#define HZIP_SVA_PREFETCH_NUM GENMASK(18, 16) +#define HZIP_SVA_STALL_NUM GENMASK(15, 0) #define HZIP_SHAPER_RATE_COMPRESS 750 #define HZIP_SHAPER_RATE_DECOMPRESS 140 -#define HZIP_DELAY_1_US 1 -#define HZIP_POLL_TIMEOUT_US 1000 +#define HZIP_DELAY_1_US 1 +#define HZIP_POLL_TIMEOUT_US 1000 +#define HZIP_WAIT_SVA_READY 500000 +#define HZIP_READ_SVA_STATUS_TIMES 3 +#define HZIP_WAIT_US_MIN 10 +#define HZIP_WAIT_US_MAX 20 /* clock gating */ #define HZIP_PEH_CFG_AUTO_GATE 0x3011A8 @@ -111,6 +119,9 @@ /* zip comp high performance */ #define HZIP_HIGH_PERF_OFFSET 0x301208 +#define HZIP_LIT_LEN_EN_OFFSET 0x301204 +#define HZIP_LIT_LEN_EN_EN BIT(4) + enum { HZIP_HIGH_COMP_RATE, HZIP_HIGH_COMP_PERF, @@ -141,6 +152,12 @@ static const struct qm_dev_alg zip_dev_algs[] = { { }, { .alg_msk = HZIP_ALG_LZ77_BIT, .alg = "lz77_zstd\n", + }, { + .alg_msk = HZIP_ALG_LZ77_BIT, + .alg = "lz77_only\n", + }, { + .alg_msk = HZIP_ALG_LZ4_BIT, + .alg = "lz4\n", }, }; @@ -448,10 +465,23 @@ bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) return false; } -static int hisi_zip_set_high_perf(struct hisi_qm *qm) +static void hisi_zip_literal_set(struct hisi_qm *qm) +{ + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + val = readl_relaxed(qm->io_base + HZIP_LIT_LEN_EN_OFFSET); + val &= ~HZIP_LIT_LEN_EN_EN; + + /* enable literal length in stream mode compression */ + writel(val, qm->io_base + HZIP_LIT_LEN_EN_OFFSET); +} + +static void hisi_zip_set_high_perf(struct hisi_qm *qm) { u32 val; - int ret; val = readl_relaxed(qm->io_base + HZIP_HIGH_PERF_OFFSET); if (perf_mode == HZIP_HIGH_COMP_PERF) @@ -461,16 +491,36 @@ static int hisi_zip_set_high_perf(struct hisi_qm *qm) /* Set perf mode */ writel(val, qm->io_base + HZIP_HIGH_PERF_OFFSET); - ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_HIGH_PERF_OFFSET, - val, val == perf_mode, HZIP_DELAY_1_US, - HZIP_POLL_TIMEOUT_US); - if (ret) - pci_err(qm->pdev, "failed to set perf mode\n"); +} - return ret; +static int hisi_zip_wait_sva_ready(struct hisi_qm *qm, __u32 offset, __u32 mask) +{ + u32 val, try_times = 0; + u8 count = 0; + + /* + * Read the register value every 10-20us. If the value is 0 for three + * consecutive times, the SVA module is ready. + */ + do { + val = readl(qm->io_base + offset); + if (val & mask) + count = 0; + else if (++count == HZIP_READ_SVA_STATUS_TIMES) + break; + + usleep_range(HZIP_WAIT_US_MIN, HZIP_WAIT_US_MAX); + } while (++try_times < HZIP_WAIT_SVA_READY); + + if (try_times == HZIP_WAIT_SVA_READY) { + pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); + return -ETIMEDOUT; + } + + return 0; } -static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) +static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) { u32 val; int ret; @@ -478,19 +528,20 @@ static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; - /* Enable prefetch */ val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); - val &= HZIP_PREFETCH_ENABLE; + val |= HZIP_SVA_PREFETCH_DISABLE; writel(val, qm->io_base + HZIP_PREFETCH_CFG); - ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG, - val, !(val & HZIP_SVA_PREFETCH_DISABLE), + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS, + val, !(val & HZIP_SVA_DISABLE_READY), HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US); if (ret) - pci_err(qm->pdev, "failed to open sva prefetch\n"); + pci_err(qm->pdev, "failed to close sva prefetch\n"); + + (void)hisi_zip_wait_sva_ready(qm, HZIP_SVA_TRANS, HZIP_SVA_STALL_NUM); } -static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) +static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) { u32 val; int ret; @@ -498,15 +549,23 @@ static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; + /* Enable prefetch */ val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); - val |= HZIP_SVA_PREFETCH_DISABLE; + val &= HZIP_PREFETCH_ENABLE; writel(val, qm->io_base + HZIP_PREFETCH_CFG); - ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS, - val, !(val & HZIP_SVA_DISABLE_READY), + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG, + val, !(val & HZIP_SVA_PREFETCH_DISABLE), HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US); + if (ret) { + pci_err(qm->pdev, "failed to open sva prefetch\n"); + hisi_zip_close_sva_prefetch(qm); + return; + } + + ret = hisi_zip_wait_sva_ready(qm, HZIP_SVA_TRANS, HZIP_SVA_PREFETCH_NUM); if (ret) - pci_err(qm->pdev, "failed to close sva prefetch\n"); + hisi_zip_close_sva_prefetch(qm); } static void hisi_zip_enable_clock_gate(struct hisi_qm *qm) @@ -530,6 +589,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) void __iomem *base = qm->io_base; u32 dcomp_bm, comp_bm; u32 zip_core_en; + int ret; /* qm user domain */ writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1); @@ -565,6 +625,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63); writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); } + hisi_zip_open_sva_prefetch(qm); /* let's open all compression/decompression cores */ @@ -580,9 +641,19 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL); + hisi_zip_set_high_perf(qm); + hisi_zip_literal_set(qm); hisi_zip_enable_clock_gate(qm); - return hisi_dae_set_user_domain(qm); + ret = hisi_dae_set_user_domain(qm); + if (ret) + goto close_sva_prefetch; + + return 0; + +close_sva_prefetch: + hisi_zip_close_sva_prefetch(qm); + return ret; } static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) @@ -592,8 +663,7 @@ static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); if (enable) { val1 |= HZIP_AXI_SHUTDOWN_ENABLE; - val2 = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + val2 = qm->err_info.dev_err.shutdown_mask; } else { val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE; val2 = 0x0; @@ -607,7 +677,8 @@ static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void hisi_zip_hw_error_enable(struct hisi_qm *qm) { - u32 nfe, ce; + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; if (qm->ver == QM_HW_V1) { writel(HZIP_CORE_INT_MASK_ALL, @@ -616,33 +687,29 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) return; } - nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); - ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); - /* clear ZIP hw error source if having */ - writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_SOURCE); + writel(err_mask, qm->io_base + HZIP_CORE_INT_SOURCE); /* configure error type */ - writel(ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); - writel(HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); - writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + writel(dev_err->ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); + writel(dev_err->fe, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); + writel(dev_err->nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); hisi_zip_master_ooo_ctrl(qm, true); /* enable ZIP hw error interrupts */ - writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); + writel(~err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG); hisi_dae_hw_error_enable(qm); } static void hisi_zip_hw_error_disable(struct hisi_qm *qm) { - u32 nfe, ce; + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; /* disable ZIP hw error interrupts */ - nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); - ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); - writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_MASK_REG); + writel(err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG); hisi_zip_master_ooo_ctrl(qm, false); @@ -1116,12 +1183,20 @@ static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) static void hisi_zip_disable_error_report(struct hisi_qm *qm, u32 err_type) { - u32 nfe_mask; + u32 nfe_mask = qm->err_info.dev_err.nfe; - nfe_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); writel(nfe_mask & (~err_type), qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); } +static void hisi_zip_enable_error_report(struct hisi_qm *qm) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + u32 ce_mask = qm->err_info.dev_err.ce; + + writel(nfe_mask, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + writel(ce_mask, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); +} + static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm) { u32 val; @@ -1160,16 +1235,18 @@ static enum acc_err_result hisi_zip_get_err_result(struct hisi_qm *qm) /* Get device hardware new error status */ err_status = hisi_zip_get_hw_err_status(qm); if (err_status) { - if (err_status & qm->err_info.ecc_2bits_mask) + if (err_status & qm->err_info.dev_err.ecc_2bits_mask) qm->err_status.is_dev_ecc_mbit = true; hisi_zip_log_hw_error(qm, err_status); - if (err_status & qm->err_info.dev_reset_mask) { + if (err_status & qm->err_info.dev_err.reset_mask) { /* Disable the same error reporting until device is recovered. */ hisi_zip_disable_error_report(qm, err_status); - return ACC_ERR_NEED_RESET; + zip_result = ACC_ERR_NEED_RESET; } else { hisi_zip_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + hisi_zip_enable_error_report(qm); } } @@ -1185,7 +1262,7 @@ static bool hisi_zip_dev_is_abnormal(struct hisi_qm *qm) u32 err_status; err_status = hisi_zip_get_hw_err_status(qm); - if (err_status & qm->err_info.dev_shutdown_mask) + if (err_status & qm->err_info.dev_err.shutdown_mask) return true; return hisi_dae_dev_is_abnormal(qm); @@ -1196,23 +1273,59 @@ static int hisi_zip_set_priv_status(struct hisi_qm *qm) return hisi_dae_close_axi_master_ooo(qm); } +static void hisi_zip_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + u32 val; + + val = ~(err_mask & (~HZIP_AXI_ERROR_MASK)); + writel(val, qm->io_base + HZIP_CORE_INT_MASK_REG); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask & (~HZIP_AXI_ERROR_MASK), + qm->io_base + HZIP_OOO_SHUTDOWN_SEL); +} + +static void hisi_zip_enable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + /* clear axi error source */ + writel(HZIP_AXI_ERROR_MASK, qm->io_base + HZIP_CORE_INT_SOURCE); + + writel(~err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask, qm->io_base + HZIP_OOO_SHUTDOWN_SEL); +} + static void hisi_zip_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &err_info->qm_err; + struct hisi_qm_err_mask *dev_err = &err_info->dev_err; + + qm_err->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK; + qm_err->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver); + qm_err->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_NFE_MASK_CAP, qm->cap_ver); + qm_err->ecc_2bits_mask = QM_ECC_MBIT; + qm_err->reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_RESET_MASK_CAP, qm->cap_ver); + qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + + dev_err->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK; + dev_err->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); + dev_err->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); + dev_err->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC; + dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + dev_err->reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_RESET_MASK_CAP, qm->cap_ver); - err_info->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK; - err_info->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver); - err_info->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_QM_NFE_MASK_CAP, qm->cap_ver); - err_info->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC; - err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_QM_RESET_MASK_CAP, qm->cap_ver); - err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = HZIP_WR_PORT; err_info->acpi_rst = "ZRST"; } @@ -1232,6 +1345,8 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = { .get_err_result = hisi_zip_get_err_result, .set_priv_status = hisi_zip_set_priv_status, .dev_is_abnormal = hisi_zip_dev_is_abnormal, + .disable_axi_error = hisi_zip_disable_axi_error, + .enable_axi_error = hisi_zip_enable_axi_error, }; static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) @@ -1251,11 +1366,6 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) if (ret) return ret; - ret = hisi_zip_set_high_perf(qm); - if (ret) - return ret; - - hisi_zip_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); hisi_zip_debug_regs_clear(qm); @@ -1273,7 +1383,7 @@ static int zip_pre_store_cap_reg(struct hisi_qm *qm) size_t i, size; size = ARRAY_SIZE(zip_cap_query_info); - zip_cap = devm_kzalloc(&pdev->dev, sizeof(*zip_cap) * size, GFP_KERNEL); + zip_cap = devm_kcalloc(&pdev->dev, size, sizeof(*zip_cap), GFP_KERNEL); if (!zip_cap) return -ENOMEM; diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 23f585219fb4..d0058757b000 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -805,7 +805,7 @@ static int save_iaa_wq(struct idxd_wq *wq) if (!cpus_per_iaa) cpus_per_iaa = 1; out: - return 0; + return ret; } static void remove_iaa_wq(struct idxd_wq *wq) diff --git a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c index 8f9e21ced0fe..48281d882260 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c @@ -232,7 +232,7 @@ static int kmb_ocs_dma_prepare(struct ahash_request *req) struct device *dev = rctx->hcu_dev->dev; unsigned int remainder = 0; unsigned int total; - size_t nents; + int nents; size_t count; int rc; int i; @@ -253,6 +253,9 @@ static int kmb_ocs_dma_prepare(struct ahash_request *req) /* Determine the number of scatter gather list entries to process. */ nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder); + if (nents < 0) + return nents; + /* If there are entries to process, map them. */ if (nents) { rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents, diff --git a/drivers/crypto/intel/qat/Kconfig b/drivers/crypto/intel/qat/Kconfig index 359c61f0c8a1..4b4861460dd4 100644 --- a/drivers/crypto/intel/qat/Kconfig +++ b/drivers/crypto/intel/qat/Kconfig @@ -6,12 +6,11 @@ config CRYPTO_DEV_QAT select CRYPTO_SKCIPHER select CRYPTO_AKCIPHER select CRYPTO_DH - select CRYPTO_HMAC select CRYPTO_RSA - select CRYPTO_SHA1 - select CRYPTO_SHA256 - select CRYPTO_SHA512 select CRYPTO_LIB_AES + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 select FW_LOADER select CRC8 diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index 35679b21ff63..667d5e320f50 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -276,11 +276,11 @@ int adf_notify_fatal_error(struct adf_accel_dev *accel_dev) int adf_init_aer(void) { device_reset_wq = alloc_workqueue("qat_device_reset_wq", - WQ_MEM_RECLAIM, 0); + WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!device_reset_wq) return -EFAULT; - device_sriov_wq = alloc_workqueue("qat_device_sriov_wq", 0, 0); + device_sriov_wq = alloc_workqueue("qat_device_sriov_wq", WQ_PERCPU, 0); if (!device_sriov_wq) { destroy_workqueue(device_reset_wq); device_reset_wq = NULL; diff --git a/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c index 48c62a14a6a7..c2e6f0cb7480 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c @@ -89,26 +89,14 @@ err_chrdev_unreg: return -EFAULT; } -static int adf_ctl_alloc_resources(struct adf_user_cfg_ctl_data **ctl_data, - unsigned long arg) +static struct adf_user_cfg_ctl_data *adf_ctl_alloc_resources(unsigned long arg) { struct adf_user_cfg_ctl_data *cfg_data; - cfg_data = kzalloc(sizeof(*cfg_data), GFP_KERNEL); - if (!cfg_data) - return -ENOMEM; - - /* Initialize device id to NO DEVICE as 0 is a valid device id */ - cfg_data->device_id = ADF_CFG_NO_DEVICE; - - if (copy_from_user(cfg_data, (void __user *)arg, sizeof(*cfg_data))) { + cfg_data = memdup_user((void __user *)arg, sizeof(*cfg_data)); + if (IS_ERR(cfg_data)) pr_err("QAT: failed to copy from user cfg_data.\n"); - kfree(cfg_data); - return -EIO; - } - - *ctl_data = cfg_data; - return 0; + return cfg_data; } static int adf_add_key_value_data(struct adf_accel_dev *accel_dev, @@ -188,13 +176,13 @@ out_err: static int adf_ctl_ioctl_dev_config(struct file *fp, unsigned int cmd, unsigned long arg) { - int ret; struct adf_user_cfg_ctl_data *ctl_data; struct adf_accel_dev *accel_dev; + int ret = 0; - ret = adf_ctl_alloc_resources(&ctl_data, arg); - if (ret) - return ret; + ctl_data = adf_ctl_alloc_resources(arg); + if (IS_ERR(ctl_data)) + return PTR_ERR(ctl_data); accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); if (!accel_dev) { @@ -267,9 +255,9 @@ static int adf_ctl_ioctl_dev_stop(struct file *fp, unsigned int cmd, int ret; struct adf_user_cfg_ctl_data *ctl_data; - ret = adf_ctl_alloc_resources(&ctl_data, arg); - if (ret) - return ret; + ctl_data = adf_ctl_alloc_resources(arg); + if (IS_ERR(ctl_data)) + return PTR_ERR(ctl_data); if (adf_devmgr_verify_id(ctl_data->device_id)) { pr_err("QAT: Device %d not found\n", ctl_data->device_id); @@ -301,9 +289,9 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, struct adf_user_cfg_ctl_data *ctl_data; struct adf_accel_dev *accel_dev; - ret = adf_ctl_alloc_resources(&ctl_data, arg); - if (ret) - return ret; + ctl_data = adf_ctl_alloc_resources(arg); + if (IS_ERR(ctl_data)) + return PTR_ERR(ctl_data); ret = -ENODEV; accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c index cf804f95838a..faa60b04c406 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c @@ -21,6 +21,25 @@ #define SLICE_IDX(sl) offsetof(struct icp_qat_fw_init_admin_slice_cnt, sl##_cnt) +#define ADF_GEN6_TL_CMDQ_WAIT_COUNTER(_name) \ + ADF_TL_COUNTER("cmdq_wait_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_CMDQ_REG_OFF(_name, reg_tm_cmdq_wait_cnt, gen6)) +#define ADF_GEN6_TL_CMDQ_EXEC_COUNTER(_name) \ + ADF_TL_COUNTER("cmdq_exec_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_CMDQ_REG_OFF(_name, reg_tm_cmdq_exec_cnt, gen6)) +#define ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(_name) \ + ADF_TL_COUNTER("cmdq_drain_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_CMDQ_REG_OFF(_name, reg_tm_cmdq_drain_cnt, \ + gen6)) + +#define CPR_QUEUE_COUNT 5 +#define DCPR_QUEUE_COUNT 3 +#define PKE_QUEUE_COUNT 1 +#define WAT_QUEUE_COUNT 7 +#define WCP_QUEUE_COUNT 7 +#define USC_QUEUE_COUNT 3 +#define ATH_QUEUE_COUNT 2 + /* Device level counters. */ static const struct adf_tl_dbg_counter dev_counters[] = { /* PCIe partial transactions. */ @@ -57,6 +76,10 @@ static const struct adf_tl_dbg_counter dev_counters[] = { /* Maximum uTLB used. */ ADF_TL_COUNTER(AT_MAX_UTLB_USED_NAME, ADF_TL_SIMPLE_COUNT, ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_max_utlb_used)), + /* Ring Empty average[ns] across all rings */ + ADF_TL_COUNTER_LATENCY(RE_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_re_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_re_cnt)), }; /* Accelerator utilization counters */ @@ -95,6 +118,80 @@ static const struct adf_tl_dbg_counter sl_exec_counters[ADF_TL_SL_CNT_COUNT] = { [SLICE_IDX(ath)] = ADF_GEN6_TL_SL_EXEC_COUNTER(ath), }; +static const struct adf_tl_dbg_counter cnv_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(cnv), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(cnv), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(cnv) +}; + +#define NUM_CMDQ_COUNTERS ARRAY_SIZE(cnv_cmdq_counters) + +static const struct adf_tl_dbg_counter dcprz_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(dcprz), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(dcprz), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(dcprz) +}; + +static_assert(ARRAY_SIZE(dcprz_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter pke_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(pke), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(pke), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(pke) +}; + +static_assert(ARRAY_SIZE(pke_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter wat_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(wat), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(wat), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(wat) +}; + +static_assert(ARRAY_SIZE(wat_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter wcp_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(wcp), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(wcp), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(wcp) +}; + +static_assert(ARRAY_SIZE(wcp_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter ucs_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(ucs), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(ucs), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(ucs) +}; + +static_assert(ARRAY_SIZE(ucs_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter ath_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(ath), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(ath), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(ath) +}; + +static_assert(ARRAY_SIZE(ath_cmdq_counters) == NUM_CMDQ_COUNTERS); + +/* CMDQ drain counters. */ +static const struct adf_tl_dbg_counter *cmdq_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression accelerator execution count. */ + [SLICE_IDX(cpr)] = cnv_cmdq_counters, + /* Decompression accelerator execution count. */ + [SLICE_IDX(dcpr)] = dcprz_cmdq_counters, + /* PKE execution count. */ + [SLICE_IDX(pke)] = pke_cmdq_counters, + /* Wireless Authentication accelerator execution count. */ + [SLICE_IDX(wat)] = wat_cmdq_counters, + /* Wireless Cipher accelerator execution count. */ + [SLICE_IDX(wcp)] = wcp_cmdq_counters, + /* UCS accelerator execution count. */ + [SLICE_IDX(ucs)] = ucs_cmdq_counters, + /* Authentication accelerator execution count. */ + [SLICE_IDX(ath)] = ath_cmdq_counters, +}; + /* Ring pair counters. */ static const struct adf_tl_dbg_counter rp_counters[] = { /* PCIe partial transactions. */ @@ -122,12 +219,17 @@ static const struct adf_tl_dbg_counter rp_counters[] = { /* Payload DevTLB miss rate. */ ADF_TL_COUNTER(AT_PAYLD_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT, ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_miss)), + /* Ring Empty average[ns]. */ + ADF_TL_COUNTER_LATENCY(RE_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_re_acc), + ADF_GEN6_TL_RP_REG_OFF(reg_tl_re_cnt)), }; void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data) { tl_data->layout_sz = ADF_GEN6_TL_LAYOUT_SZ; tl_data->slice_reg_sz = ADF_GEN6_TL_SLICE_REG_SZ; + tl_data->cmdq_reg_sz = ADF_GEN6_TL_CMDQ_REG_SZ; tl_data->rp_reg_sz = ADF_GEN6_TL_RP_REG_SZ; tl_data->num_hbuff = ADF_GEN6_TL_NUM_HIST_BUFFS; tl_data->max_rp = ADF_GEN6_TL_MAX_RP_NUM; @@ -139,8 +241,18 @@ void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data) tl_data->num_dev_counters = ARRAY_SIZE(dev_counters); tl_data->sl_util_counters = sl_util_counters; tl_data->sl_exec_counters = sl_exec_counters; + tl_data->cmdq_counters = cmdq_counters; + tl_data->num_cmdq_counters = NUM_CMDQ_COUNTERS; tl_data->rp_counters = rp_counters; tl_data->num_rp_counters = ARRAY_SIZE(rp_counters); tl_data->max_sl_cnt = ADF_GEN6_TL_MAX_SLICES_PER_TYPE; + + tl_data->multiplier.cpr_cnt = CPR_QUEUE_COUNT; + tl_data->multiplier.dcpr_cnt = DCPR_QUEUE_COUNT; + tl_data->multiplier.pke_cnt = PKE_QUEUE_COUNT; + tl_data->multiplier.wat_cnt = WAT_QUEUE_COUNT; + tl_data->multiplier.wcp_cnt = WCP_QUEUE_COUNT; + tl_data->multiplier.ucs_cnt = USC_QUEUE_COUNT; + tl_data->multiplier.ath_cnt = ATH_QUEUE_COUNT; } EXPORT_SYMBOL_GPL(adf_gen6_init_tl_data); diff --git a/drivers/crypto/intel/qat/qat_common/adf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_isr.c index 12e565613661..4639d7fd93e6 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_isr.c @@ -384,7 +384,8 @@ EXPORT_SYMBOL_GPL(adf_isr_resource_alloc); */ int __init adf_init_misc_wq(void) { - adf_misc_wq = alloc_workqueue("qat_misc_wq", WQ_MEM_RECLAIM, 0); + adf_misc_wq = alloc_workqueue("qat_misc_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); return !adf_misc_wq ? -ENOMEM : 0; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_sriov.c b/drivers/crypto/intel/qat/qat_common/adf_sriov.c index 31d1ef0cb1f5..bb904ba4bf84 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sriov.c @@ -299,7 +299,8 @@ EXPORT_SYMBOL_GPL(adf_sriov_configure); int __init adf_init_pf_wq(void) { /* Workqueue for PF2VF responses */ - pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq", WQ_MEM_RECLAIM, 0); + pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); return !pf2vf_resp_wq ? -ENOMEM : 0; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.c b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c index 74fb0c2ed241..b64142db1f0d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_telemetry.c +++ b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c @@ -212,6 +212,23 @@ int adf_tl_halt(struct adf_accel_dev *accel_dev) return ret; } +static void adf_set_cmdq_cnt(struct adf_accel_dev *accel_dev, + struct adf_tl_hw_data *tl_data) +{ + struct icp_qat_fw_init_admin_slice_cnt *slice_cnt, *cmdq_cnt; + + slice_cnt = &accel_dev->telemetry->slice_cnt; + cmdq_cnt = &accel_dev->telemetry->cmdq_cnt; + + cmdq_cnt->cpr_cnt = slice_cnt->cpr_cnt * tl_data->multiplier.cpr_cnt; + cmdq_cnt->dcpr_cnt = slice_cnt->dcpr_cnt * tl_data->multiplier.dcpr_cnt; + cmdq_cnt->pke_cnt = slice_cnt->pke_cnt * tl_data->multiplier.pke_cnt; + cmdq_cnt->wat_cnt = slice_cnt->wat_cnt * tl_data->multiplier.wat_cnt; + cmdq_cnt->wcp_cnt = slice_cnt->wcp_cnt * tl_data->multiplier.wcp_cnt; + cmdq_cnt->ucs_cnt = slice_cnt->ucs_cnt * tl_data->multiplier.ucs_cnt; + cmdq_cnt->ath_cnt = slice_cnt->ath_cnt * tl_data->multiplier.ath_cnt; +} + int adf_tl_run(struct adf_accel_dev *accel_dev, int state) { struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); @@ -235,6 +252,8 @@ int adf_tl_run(struct adf_accel_dev *accel_dev, int state) return ret; } + adf_set_cmdq_cnt(accel_dev, tl_data); + telemetry->hbuffs = state; atomic_set(&telemetry->state, state); diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.h b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h index e54a406cc1b4..02d75c3c214a 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_telemetry.h +++ b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h @@ -28,19 +28,23 @@ struct dentry; struct adf_tl_hw_data { size_t layout_sz; size_t slice_reg_sz; + size_t cmdq_reg_sz; size_t rp_reg_sz; size_t msg_cnt_off; const struct adf_tl_dbg_counter *dev_counters; const struct adf_tl_dbg_counter *sl_util_counters; const struct adf_tl_dbg_counter *sl_exec_counters; + const struct adf_tl_dbg_counter **cmdq_counters; const struct adf_tl_dbg_counter *rp_counters; u8 num_hbuff; u8 cpp_ns_per_cycle; u8 bw_units_to_bytes; u8 num_dev_counters; u8 num_rp_counters; + u8 num_cmdq_counters; u8 max_rp; u8 max_sl_cnt; + struct icp_qat_fw_init_admin_slice_cnt multiplier; }; struct adf_telemetry { @@ -69,6 +73,7 @@ struct adf_telemetry { struct mutex wr_lock; struct delayed_work work_ctx; struct icp_qat_fw_init_admin_slice_cnt slice_cnt; + struct icp_qat_fw_init_admin_slice_cnt cmdq_cnt; }; #ifdef CONFIG_DEBUG_FS diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c index a32db273842a..b81f70576683 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c @@ -339,6 +339,48 @@ static int tl_calc_and_print_sl_counters(struct adf_accel_dev *accel_dev, return 0; } +static int tl_print_cmdq_counter(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct seq_file *s, u8 cnt_id, u8 counter) +{ + size_t cmdq_regs_sz = GET_TL_DATA(telemetry->accel_dev).cmdq_reg_sz; + size_t offset_inc = cnt_id * cmdq_regs_sz; + struct adf_tl_dbg_counter slice_ctr; + char cnt_name[MAX_COUNT_NAME_SIZE]; + + slice_ctr = *(ctr + counter); + slice_ctr.offset1 += offset_inc; + snprintf(cnt_name, MAX_COUNT_NAME_SIZE, "%s%d", slice_ctr.name, cnt_id); + + return tl_calc_and_print_counter(telemetry, s, &slice_ctr, cnt_name); +} + +static int tl_calc_and_print_cmdq_counters(struct adf_accel_dev *accel_dev, + struct seq_file *s, u8 cnt_type, + u8 cnt_id) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + const struct adf_tl_dbg_counter **cmdq_tl_counters; + const struct adf_tl_dbg_counter *ctr; + u8 counter; + int ret; + + cmdq_tl_counters = tl_data->cmdq_counters; + ctr = cmdq_tl_counters[cnt_type]; + + for (counter = 0; counter < tl_data->num_cmdq_counters; counter++) { + ret = tl_print_cmdq_counter(telemetry, ctr, s, cnt_id, counter); + if (ret) { + dev_notice(&GET_DEV(accel_dev), + "invalid slice utilization counter type\n"); + return ret; + } + } + + return 0; +} + static void tl_print_msg_cnt(struct seq_file *s, u32 msg_cnt) { seq_printf(s, "%-*s", TL_KEY_MIN_PADDING, SNAPSHOT_CNT_MSG); @@ -352,6 +394,7 @@ static int tl_print_dev_data(struct adf_accel_dev *accel_dev, struct adf_telemetry *telemetry = accel_dev->telemetry; const struct adf_tl_dbg_counter *dev_tl_counters; u8 num_dev_counters = tl_data->num_dev_counters; + u8 *cmdq_cnt = (u8 *)&telemetry->cmdq_cnt; u8 *sl_cnt = (u8 *)&telemetry->slice_cnt; const struct adf_tl_dbg_counter *ctr; unsigned int i; @@ -387,6 +430,15 @@ static int tl_print_dev_data(struct adf_accel_dev *accel_dev, } } + /* Print per command queue telemetry. */ + for (i = 0; i < ADF_TL_SL_CNT_COUNT; i++) { + for (j = 0; j < cmdq_cnt[i]; j++) { + ret = tl_calc_and_print_cmdq_counters(accel_dev, s, i, j); + if (ret) + return ret; + } + } + return 0; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h index 11cc9eae19b3..97c5eeaa1b17 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h +++ b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h @@ -17,6 +17,7 @@ struct adf_accel_dev; #define LAT_ACC_NAME "gp_lat_acc_avg" #define BW_IN_NAME "bw_in" #define BW_OUT_NAME "bw_out" +#define RE_ACC_NAME "re_acc_avg" #define PAGE_REQ_LAT_NAME "at_page_req_lat_avg" #define AT_TRANS_LAT_NAME "at_trans_lat_avg" #define AT_MAX_UTLB_USED_NAME "at_max_tlb_used" @@ -43,6 +44,10 @@ struct adf_accel_dev; (ADF_TL_DEV_REG_OFF(slice##_slices[0], qat_gen) + \ offsetof(struct adf_##qat_gen##_tl_slice_data_regs, reg)) +#define ADF_TL_CMDQ_REG_OFF(slice, reg, qat_gen) \ + (ADF_TL_DEV_REG_OFF(slice##_cmdq[0], qat_gen) + \ + offsetof(struct adf_##qat_gen##_tl_cmdq_data_regs, reg)) + #define ADF_TL_RP_REG_OFF(reg, qat_gen) \ (ADF_TL_DATA_REG_OFF(tl_ring_pairs_data_regs[0], qat_gen) + \ offsetof(struct adf_##qat_gen##_tl_ring_pair_data_regs, reg)) diff --git a/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c index a4636ec9f9ca..d0fef20a3df4 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c @@ -299,7 +299,8 @@ EXPORT_SYMBOL_GPL(adf_flush_vf_wq); */ int __init adf_init_vf_wq(void) { - adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0); + adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); return !adf_vf_stop_wq ? -EFAULT : 0; } diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs.c b/drivers/crypto/intel/qat/qat_common/qat_algs.c index 43e6dd9b77b7..7f638a62e3ad 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_algs.c @@ -5,12 +5,10 @@ #include <linux/crypto.h> #include <crypto/internal/aead.h> #include <crypto/internal/cipher.h> -#include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/aes.h> #include <crypto/sha1.h> #include <crypto/sha2.h> -#include <crypto/hmac.h> #include <crypto/algapi.h> #include <crypto/authenc.h> #include <crypto/scatterwalk.h> @@ -68,16 +66,10 @@ struct qat_alg_aead_ctx { dma_addr_t dec_cd_paddr; struct icp_qat_fw_la_bulk_req enc_fw_req; struct icp_qat_fw_la_bulk_req dec_fw_req; - struct crypto_shash *hash_tfm; enum icp_qat_hw_auth_algo qat_hash_alg; + unsigned int hash_digestsize; + unsigned int hash_blocksize; struct qat_crypto_instance *inst; - union { - struct sha1_state sha1; - struct sha256_state sha256; - struct sha512_state sha512; - }; - char ipad[SHA512_BLOCK_SIZE]; /* sufficient for SHA-1/SHA-256 as well */ - char opad[SHA512_BLOCK_SIZE]; }; struct qat_alg_skcipher_ctx { @@ -94,125 +86,57 @@ struct qat_alg_skcipher_ctx { int mode; }; -static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) -{ - switch (qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - return ICP_QAT_HW_SHA1_STATE1_SZ; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - return ICP_QAT_HW_SHA256_STATE1_SZ; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - return ICP_QAT_HW_SHA512_STATE1_SZ; - default: - return -EFAULT; - } -} - static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, struct qat_alg_aead_ctx *ctx, const u8 *auth_key, unsigned int auth_keylen) { - SHASH_DESC_ON_STACK(shash, ctx->hash_tfm); - int block_size = crypto_shash_blocksize(ctx->hash_tfm); - int digest_size = crypto_shash_digestsize(ctx->hash_tfm); - __be32 *hash_state_out; - __be64 *hash512_state_out; - int i, offset; - - memset(ctx->ipad, 0, block_size); - memset(ctx->opad, 0, block_size); - shash->tfm = ctx->hash_tfm; - - if (auth_keylen > block_size) { - int ret = crypto_shash_digest(shash, auth_key, - auth_keylen, ctx->ipad); - if (ret) - return ret; - - memcpy(ctx->opad, ctx->ipad, digest_size); - } else { - memcpy(ctx->ipad, auth_key, auth_keylen); - memcpy(ctx->opad, auth_key, auth_keylen); + switch (ctx->qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: { + struct hmac_sha1_key key; + __be32 *istate = (__be32 *)hash->sha.state1; + __be32 *ostate = (__be32 *)(hash->sha.state1 + + round_up(sizeof(key.istate.h), 8)); + + hmac_sha1_preparekey(&key, auth_key, auth_keylen); + for (int i = 0; i < ARRAY_SIZE(key.istate.h); i++) { + istate[i] = cpu_to_be32(key.istate.h[i]); + ostate[i] = cpu_to_be32(key.ostate.h[i]); + } + memzero_explicit(&key, sizeof(key)); + return 0; } - - for (i = 0; i < block_size; i++) { - char *ipad_ptr = ctx->ipad + i; - char *opad_ptr = ctx->opad + i; - *ipad_ptr ^= HMAC_IPAD_VALUE; - *opad_ptr ^= HMAC_OPAD_VALUE; + case ICP_QAT_HW_AUTH_ALGO_SHA256: { + struct hmac_sha256_key key; + __be32 *istate = (__be32 *)hash->sha.state1; + __be32 *ostate = (__be32 *)(hash->sha.state1 + + sizeof(key.key.istate.h)); + + hmac_sha256_preparekey(&key, auth_key, auth_keylen); + for (int i = 0; i < ARRAY_SIZE(key.key.istate.h); i++) { + istate[i] = cpu_to_be32(key.key.istate.h[i]); + ostate[i] = cpu_to_be32(key.key.ostate.h[i]); + } + memzero_explicit(&key, sizeof(key)); + return 0; } - - if (crypto_shash_init(shash)) - return -EFAULT; - - if (crypto_shash_update(shash, ctx->ipad, block_size)) - return -EFAULT; - - hash_state_out = (__be32 *)hash->sha.state1; - hash512_state_out = (__be64 *)hash_state_out; - - switch (ctx->qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - if (crypto_shash_export_core(shash, &ctx->sha1)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha1.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - if (crypto_shash_export_core(shash, &ctx->sha256)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha256.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - if (crypto_shash_export_core(shash, &ctx->sha512)) - return -EFAULT; - for (i = 0; i < digest_size >> 3; i++, hash512_state_out++) - *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]); - break; - default: - return -EFAULT; + case ICP_QAT_HW_AUTH_ALGO_SHA512: { + struct hmac_sha512_key key; + __be64 *istate = (__be64 *)hash->sha.state1; + __be64 *ostate = (__be64 *)(hash->sha.state1 + + sizeof(key.key.istate.h)); + + hmac_sha512_preparekey(&key, auth_key, auth_keylen); + for (int i = 0; i < ARRAY_SIZE(key.key.istate.h); i++) { + istate[i] = cpu_to_be64(key.key.istate.h[i]); + ostate[i] = cpu_to_be64(key.key.ostate.h[i]); + } + memzero_explicit(&key, sizeof(key)); + return 0; } - - if (crypto_shash_init(shash)) - return -EFAULT; - - if (crypto_shash_update(shash, ctx->opad, block_size)) - return -EFAULT; - - offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8); - if (offset < 0) - return -EFAULT; - - hash_state_out = (__be32 *)(hash->sha.state1 + offset); - hash512_state_out = (__be64 *)hash_state_out; - - switch (ctx->qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - if (crypto_shash_export_core(shash, &ctx->sha1)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha1.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - if (crypto_shash_export_core(shash, &ctx->sha256)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha256.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - if (crypto_shash_export_core(shash, &ctx->sha512)) - return -EFAULT; - for (i = 0; i < digest_size >> 3; i++, hash512_state_out++) - *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]); - break; default: return -EFAULT; } - memzero_explicit(ctx->ipad, block_size); - memzero_explicit(ctx->opad, block_size); - return 0; } static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) @@ -259,7 +183,7 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, ctx->qat_hash_alg, digestsize); hash->sha.inner_setup.auth_counter.counter = - cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); + cpu_to_be32(ctx->hash_blocksize); if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen)) return -EFAULT; @@ -326,7 +250,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, struct icp_qat_hw_cipher_algo_blk *cipher = (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx + sizeof(struct icp_qat_hw_auth_setup) + - roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2); + roundup(ctx->hash_digestsize, 8) * 2); struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; @@ -346,7 +270,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, ctx->qat_hash_alg, digestsize); hash->sha.inner_setup.auth_counter.counter = - cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); + cpu_to_be32(ctx->hash_blocksize); if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen)) return -EFAULT; @@ -368,7 +292,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; cipher_cd_ctrl->cipher_cfg_offset = (sizeof(struct icp_qat_hw_auth_setup) + - roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3; + roundup(ctx->hash_digestsize, 8) * 2) >> 3; ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); @@ -1150,32 +1074,35 @@ static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req) } static int qat_alg_aead_init(struct crypto_aead *tfm, - enum icp_qat_hw_auth_algo hash, - const char *hash_name) + enum icp_qat_hw_auth_algo hash_alg, + unsigned int hash_digestsize, + unsigned int hash_blocksize) { struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); - ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); - if (IS_ERR(ctx->hash_tfm)) - return PTR_ERR(ctx->hash_tfm); - ctx->qat_hash_alg = hash; + ctx->qat_hash_alg = hash_alg; + ctx->hash_digestsize = hash_digestsize; + ctx->hash_blocksize = hash_blocksize; crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request)); return 0; } static int qat_alg_aead_sha1_init(struct crypto_aead *tfm) { - return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, + SHA1_DIGEST_SIZE, SHA1_BLOCK_SIZE); } static int qat_alg_aead_sha256_init(struct crypto_aead *tfm) { - return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, + SHA256_DIGEST_SIZE, SHA256_BLOCK_SIZE); } static int qat_alg_aead_sha512_init(struct crypto_aead *tfm) { - return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, + SHA512_DIGEST_SIZE, SHA512_BLOCK_SIZE); } static void qat_alg_aead_exit(struct crypto_aead *tfm) @@ -1184,8 +1111,6 @@ static void qat_alg_aead_exit(struct crypto_aead *tfm) struct qat_crypto_instance *inst = ctx->inst; struct device *dev; - crypto_free_shash(ctx->hash_tfm); - if (!inst) return; diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 21d652a1c8ef..06d49cb781ae 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -200,20 +200,12 @@ qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_parse_num(char *str, unsigned int *num) { - char buf[16] = {0}; - unsigned long ae = 0; - int i; - - strscpy(buf, str, sizeof(buf)); - for (i = 0; i < 16; i++) { - if (!isdigit(buf[i])) { - buf[i] = '\0'; - break; - } - } - if ((kstrtoul(buf, 10, &ae))) - return -EFAULT; + unsigned long long ae; + char *end; + ae = simple_strtoull(str, &end, 10); + if (ae > UINT_MAX || str == end || (end - str) > 19) + return -EINVAL; *num = (unsigned int)ae; return 0; } @@ -1900,7 +1892,7 @@ static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) if (sobj_hdr) sobj_chunk_num = sobj_hdr->num_chunks; - mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) * + mobj_hdr = kcalloc(size_add(uobj_chunk_num, sobj_chunk_num), sizeof(*mobj_hdr), GFP_KERNEL); if (!mobj_hdr) return -ENOMEM; diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index 9c21f5d835d2..301bdf239e7d 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -420,7 +420,6 @@ static int mv_cesa_probe(struct platform_device *pdev) { const struct mv_cesa_caps *caps = &orion_caps; const struct mbus_dram_target_info *dram; - const struct of_device_id *match; struct device *dev = &pdev->dev; struct mv_cesa_dev *cesa; struct mv_cesa_engine *engines; @@ -433,11 +432,9 @@ static int mv_cesa_probe(struct platform_device *pdev) } if (dev->of_node) { - match = of_match_node(mv_cesa_of_match_table, dev->of_node); - if (!match || !match->data) + caps = of_device_get_match_data(dev); + if (!caps) return -ENOTSUPP; - - caps = match->data; } cesa = devm_kzalloc(dev, sizeof(*cesa), GFP_KERNEL); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index cc47e361089a..b5cc5401f704 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -3,6 +3,7 @@ #include <linux/ctype.h> #include <linux/firmware.h> +#include <linux/string.h> #include <linux/string_choices.h> #include "otx2_cptpf_ucode.h" #include "otx2_cpt_common.h" @@ -458,13 +459,13 @@ static int cpt_ucode_load_fw(struct pci_dev *pdev, struct fw_info_t *fw_info, u16 rid) { char filename[OTX2_CPT_NAME_LENGTH]; - char eng_type[8] = {0}; + char eng_type[8]; int ret, e, i; INIT_LIST_HEAD(&fw_info->ucodes); for (e = 1; e < OTX2_CPT_MAX_ENG_TYPES; e++) { - strcpy(eng_type, get_eng_type_str(e)); + strscpy(eng_type, get_eng_type_str(e)); for (i = 0; i < strlen(eng_type); i++) eng_type[i] = tolower(eng_type[i]); @@ -1615,7 +1616,7 @@ int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, return -EINVAL; } err_msg = "Invalid engine group format"; - strscpy(tmp_buf, ctx->val.vstr, strlen(ctx->val.vstr) + 1); + strscpy(tmp_buf, ctx->val.vstr); start = tmp_buf; has_se = has_ie = has_ae = false; diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c index fd0a98b2fb1b..0493041ea088 100644 --- a/drivers/crypto/nx/nx-common-powernv.c +++ b/drivers/crypto/nx/nx-common-powernv.c @@ -1043,8 +1043,10 @@ static struct scomp_alg nx842_powernv_alg = { .base.cra_priority = 300, .base.cra_module = THIS_MODULE, - .alloc_ctx = nx842_powernv_crypto_alloc_ctx, - .free_ctx = nx842_crypto_free_ctx, + .streams = { + .alloc_ctx = nx842_powernv_crypto_alloc_ctx, + .free_ctx = nx842_crypto_free_ctx, + }, .compress = nx842_crypto_compress, .decompress = nx842_crypto_decompress, }; diff --git a/drivers/crypto/nx/nx-common-pseries.c b/drivers/crypto/nx/nx-common-pseries.c index f528e072494a..fc0222ebe807 100644 --- a/drivers/crypto/nx/nx-common-pseries.c +++ b/drivers/crypto/nx/nx-common-pseries.c @@ -1020,8 +1020,10 @@ static struct scomp_alg nx842_pseries_alg = { .base.cra_priority = 300, .base.cra_module = THIS_MODULE, - .alloc_ctx = nx842_pseries_crypto_alloc_ctx, - .free_ctx = nx842_crypto_free_ctx, + .streams = { + .alloc_ctx = nx842_pseries_crypto_alloc_ctx, + .free_ctx = nx842_crypto_free_ctx, + }, .compress = nx842_crypto_compress, .decompress = nx842_crypto_decompress, }; diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 244e24e52987..3cc802622dd5 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -32,6 +32,7 @@ #include <linux/pm_runtime.h> #include <linux/scatterlist.h> #include <linux/string.h> +#include <linux/workqueue.h> #include "omap-crypto.h" #include "omap-aes.h" @@ -221,7 +222,7 @@ static void omap_aes_dma_out_callback(void *data) struct omap_aes_dev *dd = data; /* dma_lch_out - completed */ - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); } static int omap_aes_dma_init(struct omap_aes_dev *dd) @@ -494,9 +495,9 @@ static void omap_aes_copy_ivout(struct omap_aes_dev *dd, u8 *ivbuf) ((u32 *)ivbuf)[i] = omap_aes_read(dd, AES_REG_IV(dd, i)); } -static void omap_aes_done_task(unsigned long data) +static void omap_aes_done_task(struct work_struct *t) { - struct omap_aes_dev *dd = (struct omap_aes_dev *)data; + struct omap_aes_dev *dd = from_work(dd, t, done_task); pr_debug("enter done_task\n"); @@ -925,7 +926,7 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) if (!dd->total) /* All bytes read! */ - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); else /* Enable DATA_IN interrupt for next block */ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2); @@ -1140,7 +1141,7 @@ static int omap_aes_probe(struct platform_device *pdev) (reg & dd->pdata->major_mask) >> dd->pdata->major_shift, (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift); - tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd); + INIT_WORK(&dd->done_task, omap_aes_done_task); err = omap_aes_dma_init(dd); if (err == -EPROBE_DEFER) { @@ -1229,7 +1230,7 @@ err_engine: omap_aes_dma_cleanup(dd); err_irq: - tasklet_kill(&dd->done_task); + cancel_work_sync(&dd->done_task); err_pm_disable: pm_runtime_disable(dev); err_res: @@ -1264,7 +1265,7 @@ static void omap_aes_remove(struct platform_device *pdev) crypto_engine_exit(dd->engine); - tasklet_kill(&dd->done_task); + cancel_work_sync(&dd->done_task); omap_aes_dma_cleanup(dd); pm_runtime_disable(dd->dev); } diff --git a/drivers/crypto/omap-aes.h b/drivers/crypto/omap-aes.h index 41d67780fd45..99c36a777e97 100644 --- a/drivers/crypto/omap-aes.h +++ b/drivers/crypto/omap-aes.h @@ -159,7 +159,7 @@ struct omap_aes_dev { unsigned long flags; int err; - struct tasklet_struct done_task; + struct work_struct done_task; struct aead_queue aead_queue; spinlock_t lock; diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 9c5538ae17db..149ebd77710b 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -32,6 +32,7 @@ #include <linux/pm_runtime.h> #include <linux/scatterlist.h> #include <linux/string.h> +#include <linux/workqueue.h> #include "omap-crypto.h" @@ -130,7 +131,7 @@ struct omap_des_dev { unsigned long flags; int err; - struct tasklet_struct done_task; + struct work_struct done_task; struct skcipher_request *req; struct crypto_engine *engine; @@ -325,7 +326,7 @@ static void omap_des_dma_out_callback(void *data) struct omap_des_dev *dd = data; /* dma_lch_out - completed */ - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); } static int omap_des_dma_init(struct omap_des_dev *dd) @@ -580,9 +581,9 @@ static int omap_des_crypt_req(struct crypto_engine *engine, omap_des_crypt_dma_start(dd); } -static void omap_des_done_task(unsigned long data) +static void omap_des_done_task(struct work_struct *t) { - struct omap_des_dev *dd = (struct omap_des_dev *)data; + struct omap_des_dev *dd = from_work(dd, t, done_task); int i; pr_debug("enter done_task\n"); @@ -890,7 +891,7 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) if (!dd->total) /* All bytes read! */ - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); else /* Enable DATA_IN interrupt for next block */ omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2); @@ -986,7 +987,7 @@ static int omap_des_probe(struct platform_device *pdev) (reg & dd->pdata->major_mask) >> dd->pdata->major_shift, (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift); - tasklet_init(&dd->done_task, omap_des_done_task, (unsigned long)dd); + INIT_WORK(&dd->done_task, omap_des_done_task); err = omap_des_dma_init(dd); if (err == -EPROBE_DEFER) { @@ -1053,7 +1054,7 @@ err_engine: omap_des_dma_cleanup(dd); err_irq: - tasklet_kill(&dd->done_task); + cancel_work_sync(&dd->done_task); err_get: pm_runtime_disable(dev); err_res: @@ -1077,7 +1078,7 @@ static void omap_des_remove(struct platform_device *pdev) crypto_engine_unregister_skcipher( &dd->pdata->algs_info[i].algs_list[j]); - tasklet_kill(&dd->done_task); + cancel_work_sync(&dd->done_task); omap_des_dma_cleanup(dd); pm_runtime_disable(dd->dev); } diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 6328e8026b91..ff8aac02994a 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -37,6 +37,7 @@ #include <linux/scatterlist.h> #include <linux/slab.h> #include <linux/string.h> +#include <linux/workqueue.h> #define MD5_DIGEST_SIZE 16 @@ -217,7 +218,7 @@ struct omap_sham_dev { int irq; int err; struct dma_chan *dma_lch; - struct tasklet_struct done_task; + struct work_struct done_task; u8 polling_mode; u8 xmit_buf[BUFLEN] OMAP_ALIGNED; @@ -561,7 +562,7 @@ static void omap_sham_dma_callback(void *param) struct omap_sham_dev *dd = param; set_bit(FLAGS_DMA_READY, &dd->flags); - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); } static int omap_sham_xmit_dma(struct omap_sham_dev *dd, size_t length, @@ -1703,9 +1704,9 @@ static struct ahash_engine_alg algs_sha384_sha512[] = { }, }; -static void omap_sham_done_task(unsigned long data) +static void omap_sham_done_task(struct work_struct *t) { - struct omap_sham_dev *dd = (struct omap_sham_dev *)data; + struct omap_sham_dev *dd = from_work(dd, t, done_task); int err = 0; dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags); @@ -1739,7 +1740,7 @@ finish: static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd) { set_bit(FLAGS_OUTPUT_READY, &dd->flags); - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); return IRQ_HANDLED; } @@ -2059,7 +2060,7 @@ static int omap_sham_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dd); INIT_LIST_HEAD(&dd->list); - tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd); + INIT_WORK(&dd->done_task, omap_sham_done_task); crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH); err = (dev->of_node) ? omap_sham_get_res_of(dd, dev, &res) : @@ -2194,7 +2195,7 @@ static void omap_sham_remove(struct platform_device *pdev) &dd->pdata->algs_info[i].algs_list[j]); dd->pdata->algs_info[i].registered--; } - tasklet_kill(&dd->done_task); + cancel_work_sync(&dd->done_task); pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_disable(&pdev->dev); diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index e95e84486d9a..b966f3365b7d 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -21,7 +21,6 @@ #include "sha.h" #include "aead.h" -#define QCE_MAJOR_VERSION5 0x05 #define QCE_QUEUE_LENGTH 1 #define QCE_DEFAULT_MEM_BANDWIDTH 393600 @@ -161,7 +160,7 @@ static int qce_check_version(struct qce_device *qce) * the driver does not support v5 with minor 0 because it has special * alignment requirements. */ - if (major != QCE_MAJOR_VERSION5 || minor == 0) + if (major == 5 && minor == 0) return -ENODEV; qce->burst_size = QCE_BAM_BURST_SIZE; diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c index 1dec7aea852d..68cafd4741ad 100644 --- a/drivers/crypto/qce/dma.c +++ b/drivers/crypto/qce/dma.c @@ -24,11 +24,13 @@ int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma) dma->txchan = dma_request_chan(dev, "tx"); if (IS_ERR(dma->txchan)) - return PTR_ERR(dma->txchan); + return dev_err_probe(dev, PTR_ERR(dma->txchan), + "Failed to get TX DMA channel\n"); dma->rxchan = dma_request_chan(dev, "rx"); if (IS_ERR(dma->rxchan)) { - ret = PTR_ERR(dma->rxchan); + ret = dev_err_probe(dev, PTR_ERR(dma->rxchan), + "Failed to get RX DMA channel\n"); goto error_rx; } diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index d6928ebe9526..b9f5a8b42e66 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -254,7 +254,7 @@ static void rk_hash_unprepare(struct crypto_engine *engine, void *breq) struct rk_ahash_rctx *rctx = ahash_request_ctx(areq); struct rk_crypto_info *rkc = rctx->dev; - dma_unmap_sg(rkc->dev, areq->src, rctx->nrsg, DMA_TO_DEVICE); + dma_unmap_sg(rkc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); } static int rk_hash_run(struct crypto_engine *engine, void *breq) diff --git a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c index 9393e10671c2..e80f9148c012 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c +++ b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c @@ -321,8 +321,7 @@ static int rk_cipher_run(struct crypto_engine *engine, void *async_req) algt->stat_req++; rkc->nreq++; - ivsize = crypto_skcipher_ivsize(tfm); - if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { + if (areq->iv && ivsize > 0) { if (rctx->mode & RK_CRYPTO_DEC) { offset = areq->cryptlen - ivsize; scatterwalk_map_and_copy(rctx->backup_iv, areq->src, diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 86a1a1fa9f8f..426b24889af8 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -511,8 +511,7 @@ static int starfive_aes_map_sg(struct starfive_cryp_dev *cryp, stsg = sg_next(stsg), dtsg = sg_next(dtsg)) { src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL); if (src_nents == 0) - return dev_err_probe(cryp->dev, -ENOMEM, - "dma_map_sg error\n"); + return -ENOMEM; dst_nents = src_nents; len = min(sg_dma_len(stsg), remain); @@ -528,13 +527,11 @@ static int starfive_aes_map_sg(struct starfive_cryp_dev *cryp, for (stsg = src, dtsg = dst;;) { src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE); if (src_nents == 0) - return dev_err_probe(cryp->dev, -ENOMEM, - "dma_map_sg src error\n"); + return -ENOMEM; dst_nents = dma_map_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE); if (dst_nents == 0) - return dev_err_probe(cryp->dev, -ENOMEM, - "dma_map_sg dst error\n"); + return -ENOMEM; len = min(sg_dma_len(stsg), sg_dma_len(dtsg)); len = min(len, remain); @@ -669,8 +666,7 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq if (cryp->assoclen) { rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL); if (!rctx->adata) - return dev_err_probe(cryp->dev, -ENOMEM, - "Failed to alloc memory for adata"); + return -ENOMEM; if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), rctx->adata, cryp->assoclen) != cryp->assoclen) diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index 6cfe0238f615..54b7af4a7aee 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -229,8 +229,7 @@ static int starfive_hash_one_request(struct crypto_engine *engine, void *areq) for_each_sg(rctx->in_sg, tsg, rctx->in_sg_len, i) { src_nents = dma_map_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE); if (src_nents == 0) - return dev_err_probe(cryp->dev, -ENOMEM, - "dma_map_sg error\n"); + return -ENOMEM; ret = starfive_hash_dma_xfer(cryp, tsg); dma_unmap_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE); @@ -326,6 +325,7 @@ static int starfive_hash_digest(struct ahash_request *req) struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); struct starfive_cryp_dev *cryp = ctx->cryp; + int sg_len; memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx)); @@ -334,7 +334,10 @@ static int starfive_hash_digest(struct ahash_request *req) rctx->in_sg = req->src; rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); rctx->digsize = crypto_ahash_digestsize(tfm); - rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + if (sg_len < 0) + return sg_len; + rctx->in_sg_len = sg_len; ctx->rctx = rctx; return crypto_transfer_hash_request_to_engine(cryp->engine, req); diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index a89b4c5d62a0..5e82e8a1f71a 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -2781,5 +2781,5 @@ static struct platform_driver stm32_cryp_driver = { module_platform_driver(stm32_cryp_driver); MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>"); -MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver"); +MODULE_DESCRIPTION("STMicroelectronics STM32 CRYP hardware driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c index d09b4aaeecef..4a298ace6e9f 100644 --- a/drivers/crypto/tegra/tegra-se-hash.c +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -400,8 +400,9 @@ static int tegra_sha_do_update(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); struct tegra_se *se = ctx->se; - unsigned int nblks, nresidue, size, ret; + unsigned int nblks, nresidue, size; u32 *cpuvaddr = se->cmdbuf->addr; + int ret; nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size; nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size; diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c index 1c94f1de0546..7237f14eaf5a 100644 --- a/drivers/crypto/tegra/tegra-se-main.c +++ b/drivers/crypto/tegra/tegra-se-main.c @@ -310,7 +310,7 @@ static int tegra_se_probe(struct platform_device *pdev) se->engine = crypto_engine_alloc_init(dev, 0); if (!se->engine) - return dev_err_probe(dev, -ENOMEM, "failed to init crypto engine\n"); + return -ENOMEM; ret = crypto_engine_start(se->engine); if (ret) { diff --git a/drivers/crypto/ti/Kconfig b/drivers/crypto/ti/Kconfig new file mode 100644 index 000000000000..a3692ceec49b --- /dev/null +++ b/drivers/crypto/ti/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CRYPTO_DEV_TI_DTHEV2 + tristate "Support for TI DTHE V2 cryptography engine" + depends on ARCH_K3 || COMPILE_TEST + select CRYPTO_ENGINE + select CRYPTO_SKCIPHER + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_XTS + help + This enables support for the TI DTHE V2 hw cryptography engine + which can be found on TI K3 SOCs. Selecting this enables use + of hardware offloading for cryptographic algorithms on + these devices, providing enhanced resistance against side-channel + attacks. diff --git a/drivers/crypto/ti/Makefile b/drivers/crypto/ti/Makefile new file mode 100644 index 000000000000..b883078f203d --- /dev/null +++ b/drivers/crypto/ti/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_TI_DTHEV2) += dthev2.o +dthev2-objs := dthev2-common.o dthev2-aes.o diff --git a/drivers/crypto/ti/dthev2-aes.c b/drivers/crypto/ti/dthev2-aes.c new file mode 100644 index 000000000000..156729ccc50e --- /dev/null +++ b/drivers/crypto/ti/dthev2-aes.c @@ -0,0 +1,538 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * K3 DTHE V2 crypto accelerator driver + * + * Copyright (C) Texas Instruments 2025 - https://www.ti.com + * Author: T Pratham <t-pratham@ti.com> + */ + +#include <crypto/aead.h> +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/engine.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/skcipher.h> + +#include "dthev2-common.h" + +#include <linux/delay.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/scatterlist.h> + +/* Registers */ + +// AES Engine +#define DTHE_P_AES_BASE 0x7000 + +#define DTHE_P_AES_KEY1_0 0x0038 +#define DTHE_P_AES_KEY1_1 0x003C +#define DTHE_P_AES_KEY1_2 0x0030 +#define DTHE_P_AES_KEY1_3 0x0034 +#define DTHE_P_AES_KEY1_4 0x0028 +#define DTHE_P_AES_KEY1_5 0x002C +#define DTHE_P_AES_KEY1_6 0x0020 +#define DTHE_P_AES_KEY1_7 0x0024 + +#define DTHE_P_AES_KEY2_0 0x0018 +#define DTHE_P_AES_KEY2_1 0x001C +#define DTHE_P_AES_KEY2_2 0x0010 +#define DTHE_P_AES_KEY2_3 0x0014 +#define DTHE_P_AES_KEY2_4 0x0008 +#define DTHE_P_AES_KEY2_5 0x000C +#define DTHE_P_AES_KEY2_6 0x0000 +#define DTHE_P_AES_KEY2_7 0x0004 + +#define DTHE_P_AES_IV_IN_0 0x0040 +#define DTHE_P_AES_IV_IN_1 0x0044 +#define DTHE_P_AES_IV_IN_2 0x0048 +#define DTHE_P_AES_IV_IN_3 0x004C +#define DTHE_P_AES_CTRL 0x0050 +#define DTHE_P_AES_C_LENGTH_0 0x0054 +#define DTHE_P_AES_C_LENGTH_1 0x0058 +#define DTHE_P_AES_AUTH_LENGTH 0x005C +#define DTHE_P_AES_DATA_IN_OUT 0x0060 + +#define DTHE_P_AES_SYSCONFIG 0x0084 +#define DTHE_P_AES_IRQSTATUS 0x008C +#define DTHE_P_AES_IRQENABLE 0x0090 + +/* Register write values and macros */ + +enum aes_ctrl_mode_masks { + AES_CTRL_ECB_MASK = 0x00, + AES_CTRL_CBC_MASK = BIT(5), + AES_CTRL_XTS_MASK = BIT(12) | BIT(11), +}; + +#define DTHE_AES_CTRL_MODE_CLEAR_MASK ~GENMASK(28, 5) + +#define DTHE_AES_CTRL_DIR_ENC BIT(2) + +#define DTHE_AES_CTRL_KEYSIZE_16B BIT(3) +#define DTHE_AES_CTRL_KEYSIZE_24B BIT(4) +#define DTHE_AES_CTRL_KEYSIZE_32B (BIT(3) | BIT(4)) + +#define DTHE_AES_CTRL_SAVE_CTX_SET BIT(29) + +#define DTHE_AES_CTRL_OUTPUT_READY BIT_MASK(0) +#define DTHE_AES_CTRL_INPUT_READY BIT_MASK(1) +#define DTHE_AES_CTRL_SAVED_CTX_READY BIT_MASK(30) +#define DTHE_AES_CTRL_CTX_READY BIT_MASK(31) + +#define DTHE_AES_SYSCONFIG_DMA_DATA_IN_OUT_EN GENMASK(6, 5) +#define DTHE_AES_IRQENABLE_EN_ALL GENMASK(3, 0) + +/* Misc */ +#define AES_IV_SIZE AES_BLOCK_SIZE +#define AES_BLOCK_WORDS (AES_BLOCK_SIZE / sizeof(u32)) +#define AES_IV_WORDS AES_BLOCK_WORDS + +static int dthe_cipher_init_tfm(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct dthe_data *dev_data = dthe_get_dev(ctx); + + ctx->dev_data = dev_data; + ctx->keylen = 0; + + return 0; +} + +static int dthe_cipher_xts_init_tfm(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct dthe_data *dev_data = dthe_get_dev(ctx); + + ctx->dev_data = dev_data; + ctx->keylen = 0; + + ctx->skcipher_fb = crypto_alloc_sync_skcipher("xts(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->skcipher_fb)) { + dev_err(dev_data->dev, "fallback driver xts(aes) couldn't be loaded\n"); + return PTR_ERR(ctx->skcipher_fb); + } + + return 0; +} + +static void dthe_cipher_xts_exit_tfm(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_sync_skcipher(ctx->skcipher_fb); +} + +static int dthe_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) + return -EINVAL; + + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + + return 0; +} + +static int dthe_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->aes_mode = DTHE_AES_ECB; + + return dthe_aes_setkey(tfm, key, keylen); +} + +static int dthe_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->aes_mode = DTHE_AES_CBC; + + return dthe_aes_setkey(tfm, key, keylen); +} + +static int dthe_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != 2 * AES_KEYSIZE_128 && + keylen != 2 * AES_KEYSIZE_192 && + keylen != 2 * AES_KEYSIZE_256) + return -EINVAL; + + ctx->aes_mode = DTHE_AES_XTS; + ctx->keylen = keylen / 2; + memcpy(ctx->key, key, keylen); + + crypto_sync_skcipher_clear_flags(ctx->skcipher_fb, CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(ctx->skcipher_fb, + crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(ctx->skcipher_fb, key, keylen); +} + +static void dthe_aes_set_ctrl_key(struct dthe_tfm_ctx *ctx, + struct dthe_aes_req_ctx *rctx, + u32 *iv_in) +{ + struct dthe_data *dev_data = dthe_get_dev(ctx); + void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; + u32 ctrl_val = 0; + + writel_relaxed(ctx->key[0], aes_base_reg + DTHE_P_AES_KEY1_0); + writel_relaxed(ctx->key[1], aes_base_reg + DTHE_P_AES_KEY1_1); + writel_relaxed(ctx->key[2], aes_base_reg + DTHE_P_AES_KEY1_2); + writel_relaxed(ctx->key[3], aes_base_reg + DTHE_P_AES_KEY1_3); + + if (ctx->keylen > AES_KEYSIZE_128) { + writel_relaxed(ctx->key[4], aes_base_reg + DTHE_P_AES_KEY1_4); + writel_relaxed(ctx->key[5], aes_base_reg + DTHE_P_AES_KEY1_5); + } + if (ctx->keylen == AES_KEYSIZE_256) { + writel_relaxed(ctx->key[6], aes_base_reg + DTHE_P_AES_KEY1_6); + writel_relaxed(ctx->key[7], aes_base_reg + DTHE_P_AES_KEY1_7); + } + + if (ctx->aes_mode == DTHE_AES_XTS) { + size_t key2_offset = ctx->keylen / sizeof(u32); + + writel_relaxed(ctx->key[key2_offset + 0], aes_base_reg + DTHE_P_AES_KEY2_0); + writel_relaxed(ctx->key[key2_offset + 1], aes_base_reg + DTHE_P_AES_KEY2_1); + writel_relaxed(ctx->key[key2_offset + 2], aes_base_reg + DTHE_P_AES_KEY2_2); + writel_relaxed(ctx->key[key2_offset + 3], aes_base_reg + DTHE_P_AES_KEY2_3); + + if (ctx->keylen > AES_KEYSIZE_128) { + writel_relaxed(ctx->key[key2_offset + 4], aes_base_reg + DTHE_P_AES_KEY2_4); + writel_relaxed(ctx->key[key2_offset + 5], aes_base_reg + DTHE_P_AES_KEY2_5); + } + if (ctx->keylen == AES_KEYSIZE_256) { + writel_relaxed(ctx->key[key2_offset + 6], aes_base_reg + DTHE_P_AES_KEY2_6); + writel_relaxed(ctx->key[key2_offset + 7], aes_base_reg + DTHE_P_AES_KEY2_7); + } + } + + if (rctx->enc) + ctrl_val |= DTHE_AES_CTRL_DIR_ENC; + + if (ctx->keylen == AES_KEYSIZE_128) + ctrl_val |= DTHE_AES_CTRL_KEYSIZE_16B; + else if (ctx->keylen == AES_KEYSIZE_192) + ctrl_val |= DTHE_AES_CTRL_KEYSIZE_24B; + else + ctrl_val |= DTHE_AES_CTRL_KEYSIZE_32B; + + // Write AES mode + ctrl_val &= DTHE_AES_CTRL_MODE_CLEAR_MASK; + switch (ctx->aes_mode) { + case DTHE_AES_ECB: + ctrl_val |= AES_CTRL_ECB_MASK; + break; + case DTHE_AES_CBC: + ctrl_val |= AES_CTRL_CBC_MASK; + break; + case DTHE_AES_XTS: + ctrl_val |= AES_CTRL_XTS_MASK; + break; + } + + if (iv_in) { + ctrl_val |= DTHE_AES_CTRL_SAVE_CTX_SET; + for (int i = 0; i < AES_IV_WORDS; ++i) + writel_relaxed(iv_in[i], + aes_base_reg + DTHE_P_AES_IV_IN_0 + (DTHE_REG_SIZE * i)); + } + + writel_relaxed(ctrl_val, aes_base_reg + DTHE_P_AES_CTRL); +} + +static void dthe_aes_dma_in_callback(void *data) +{ + struct skcipher_request *req = (struct skcipher_request *)data; + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + complete(&rctx->aes_compl); +} + +static int dthe_aes_run(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = container_of(areq, struct skcipher_request, base); + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct dthe_data *dev_data = dthe_get_dev(ctx); + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + unsigned int len = req->cryptlen; + struct scatterlist *src = req->src; + struct scatterlist *dst = req->dst; + + int src_nents = sg_nents_for_len(src, len); + int dst_nents; + + int src_mapped_nents; + int dst_mapped_nents; + + bool diff_dst; + enum dma_data_direction src_dir, dst_dir; + + struct device *tx_dev, *rx_dev; + struct dma_async_tx_descriptor *desc_in, *desc_out; + + int ret; + + void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; + + u32 aes_irqenable_val = readl_relaxed(aes_base_reg + DTHE_P_AES_IRQENABLE); + u32 aes_sysconfig_val = readl_relaxed(aes_base_reg + DTHE_P_AES_SYSCONFIG); + + aes_sysconfig_val |= DTHE_AES_SYSCONFIG_DMA_DATA_IN_OUT_EN; + writel_relaxed(aes_sysconfig_val, aes_base_reg + DTHE_P_AES_SYSCONFIG); + + aes_irqenable_val |= DTHE_AES_IRQENABLE_EN_ALL; + writel_relaxed(aes_irqenable_val, aes_base_reg + DTHE_P_AES_IRQENABLE); + + if (src == dst) { + diff_dst = false; + src_dir = DMA_BIDIRECTIONAL; + dst_dir = DMA_BIDIRECTIONAL; + } else { + diff_dst = true; + src_dir = DMA_TO_DEVICE; + dst_dir = DMA_FROM_DEVICE; + } + + tx_dev = dmaengine_get_dma_device(dev_data->dma_aes_tx); + rx_dev = dmaengine_get_dma_device(dev_data->dma_aes_rx); + + src_mapped_nents = dma_map_sg(tx_dev, src, src_nents, src_dir); + if (src_mapped_nents == 0) { + ret = -EINVAL; + goto aes_err; + } + + if (!diff_dst) { + dst_nents = src_nents; + dst_mapped_nents = src_mapped_nents; + } else { + dst_nents = sg_nents_for_len(dst, len); + dst_mapped_nents = dma_map_sg(rx_dev, dst, dst_nents, dst_dir); + if (dst_mapped_nents == 0) { + dma_unmap_sg(tx_dev, src, src_nents, src_dir); + ret = -EINVAL; + goto aes_err; + } + } + + desc_in = dmaengine_prep_slave_sg(dev_data->dma_aes_rx, dst, dst_mapped_nents, + DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc_in) { + dev_err(dev_data->dev, "IN prep_slave_sg() failed\n"); + ret = -EINVAL; + goto aes_prep_err; + } + + desc_out = dmaengine_prep_slave_sg(dev_data->dma_aes_tx, src, src_mapped_nents, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc_out) { + dev_err(dev_data->dev, "OUT prep_slave_sg() failed\n"); + ret = -EINVAL; + goto aes_prep_err; + } + + desc_in->callback = dthe_aes_dma_in_callback; + desc_in->callback_param = req; + + init_completion(&rctx->aes_compl); + + if (ctx->aes_mode == DTHE_AES_ECB) + dthe_aes_set_ctrl_key(ctx, rctx, NULL); + else + dthe_aes_set_ctrl_key(ctx, rctx, (u32 *)req->iv); + + writel_relaxed(lower_32_bits(req->cryptlen), aes_base_reg + DTHE_P_AES_C_LENGTH_0); + writel_relaxed(upper_32_bits(req->cryptlen), aes_base_reg + DTHE_P_AES_C_LENGTH_1); + + dmaengine_submit(desc_in); + dmaengine_submit(desc_out); + + dma_async_issue_pending(dev_data->dma_aes_rx); + dma_async_issue_pending(dev_data->dma_aes_tx); + + // Need to do a timeout to ensure finalise gets called if DMA callback fails for any reason + ret = wait_for_completion_timeout(&rctx->aes_compl, msecs_to_jiffies(DTHE_DMA_TIMEOUT_MS)); + if (!ret) { + ret = -ETIMEDOUT; + dmaengine_terminate_sync(dev_data->dma_aes_rx); + dmaengine_terminate_sync(dev_data->dma_aes_tx); + + for (int i = 0; i < AES_BLOCK_WORDS; ++i) + readl_relaxed(aes_base_reg + DTHE_P_AES_DATA_IN_OUT + (DTHE_REG_SIZE * i)); + } else { + ret = 0; + } + + // For modes other than ECB, read IV_OUT + if (ctx->aes_mode != DTHE_AES_ECB) { + u32 *iv_out = (u32 *)req->iv; + + for (int i = 0; i < AES_IV_WORDS; ++i) + iv_out[i] = readl_relaxed(aes_base_reg + + DTHE_P_AES_IV_IN_0 + + (DTHE_REG_SIZE * i)); + } + +aes_prep_err: + dma_unmap_sg(tx_dev, src, src_nents, src_dir); + if (dst_dir != DMA_BIDIRECTIONAL) + dma_unmap_sg(rx_dev, dst, dst_nents, dst_dir); + +aes_err: + local_bh_disable(); + crypto_finalize_skcipher_request(dev_data->engine, req, ret); + local_bh_enable(); + return 0; +} + +static int dthe_aes_crypt(struct skcipher_request *req) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + struct dthe_data *dev_data = dthe_get_dev(ctx); + struct crypto_engine *engine; + + /* + * If data is not a multiple of AES_BLOCK_SIZE: + * - need to return -EINVAL for ECB, CBC as they are block ciphers + * - need to fallback to software as H/W doesn't support Ciphertext Stealing for XTS + */ + if (req->cryptlen % AES_BLOCK_SIZE) { + if (ctx->aes_mode == DTHE_AES_XTS) { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->skcipher_fb); + + skcipher_request_set_callback(subreq, skcipher_request_flags(req), + req->base.complete, req->base.data); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + return rctx->enc ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + } + return -EINVAL; + } + + /* + * If data length input is zero, no need to do any operation. + * Except for XTS mode, where data length should be non-zero. + */ + if (req->cryptlen == 0) { + if (ctx->aes_mode == DTHE_AES_XTS) + return -EINVAL; + return 0; + } + + engine = dev_data->engine; + return crypto_transfer_skcipher_request_to_engine(engine, req); +} + +static int dthe_aes_encrypt(struct skcipher_request *req) +{ + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + rctx->enc = 1; + return dthe_aes_crypt(req); +} + +static int dthe_aes_decrypt(struct skcipher_request *req) +{ + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + rctx->enc = 0; + return dthe_aes_crypt(req); +} + +static struct skcipher_engine_alg cipher_algs[] = { + { + .base.init = dthe_cipher_init_tfm, + .base.setkey = dthe_aes_ecb_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_alignmask = AES_BLOCK_SIZE - 1, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* ECB AES */ + { + .base.init = dthe_cipher_init_tfm, + .base.setkey = dthe_aes_cbc_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.ivsize = AES_IV_SIZE, + .base.base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_alignmask = AES_BLOCK_SIZE - 1, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* CBC AES */ + { + .base.init = dthe_cipher_xts_init_tfm, + .base.exit = dthe_cipher_xts_exit_tfm, + .base.setkey = dthe_aes_xts_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE * 2, + .base.max_keysize = AES_MAX_KEY_SIZE * 2, + .base.ivsize = AES_IV_SIZE, + .base.base = { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_alignmask = AES_BLOCK_SIZE - 1, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* XTS AES */ +}; + +int dthe_register_aes_algs(void) +{ + return crypto_engine_register_skciphers(cipher_algs, ARRAY_SIZE(cipher_algs)); +} + +void dthe_unregister_aes_algs(void) +{ + crypto_engine_unregister_skciphers(cipher_algs, ARRAY_SIZE(cipher_algs)); +} diff --git a/drivers/crypto/ti/dthev2-common.c b/drivers/crypto/ti/dthev2-common.c new file mode 100644 index 000000000000..c39d37933b9e --- /dev/null +++ b/drivers/crypto/ti/dthev2-common.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * K3 DTHE V2 crypto accelerator driver + * + * Copyright (C) Texas Instruments 2025 - https://www.ti.com + * Author: T Pratham <t-pratham@ti.com> + */ + +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/engine.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/skcipher.h> + +#include "dthev2-common.h" + +#include <linux/delay.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/platform_device.h> +#include <linux/scatterlist.h> + +#define DRIVER_NAME "dthev2" + +static struct dthe_list dthe_dev_list = { + .dev_list = LIST_HEAD_INIT(dthe_dev_list.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(dthe_dev_list.lock), +}; + +struct dthe_data *dthe_get_dev(struct dthe_tfm_ctx *ctx) +{ + struct dthe_data *dev_data; + + if (ctx->dev_data) + return ctx->dev_data; + + spin_lock_bh(&dthe_dev_list.lock); + dev_data = list_first_entry(&dthe_dev_list.dev_list, struct dthe_data, list); + if (dev_data) + list_move_tail(&dev_data->list, &dthe_dev_list.dev_list); + spin_unlock_bh(&dthe_dev_list.lock); + + return dev_data; +} + +static int dthe_dma_init(struct dthe_data *dev_data) +{ + int ret; + struct dma_slave_config cfg; + + dev_data->dma_aes_rx = NULL; + dev_data->dma_aes_tx = NULL; + dev_data->dma_sha_tx = NULL; + + dev_data->dma_aes_rx = dma_request_chan(dev_data->dev, "rx"); + if (IS_ERR(dev_data->dma_aes_rx)) { + return dev_err_probe(dev_data->dev, PTR_ERR(dev_data->dma_aes_rx), + "Unable to request rx DMA channel\n"); + } + + dev_data->dma_aes_tx = dma_request_chan(dev_data->dev, "tx1"); + if (IS_ERR(dev_data->dma_aes_tx)) { + ret = dev_err_probe(dev_data->dev, PTR_ERR(dev_data->dma_aes_tx), + "Unable to request tx1 DMA channel\n"); + goto err_dma_aes_tx; + } + + dev_data->dma_sha_tx = dma_request_chan(dev_data->dev, "tx2"); + if (IS_ERR(dev_data->dma_sha_tx)) { + ret = dev_err_probe(dev_data->dev, PTR_ERR(dev_data->dma_sha_tx), + "Unable to request tx2 DMA channel\n"); + goto err_dma_sha_tx; + } + + memzero_explicit(&cfg, sizeof(cfg)); + + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_maxburst = 4; + + ret = dmaengine_slave_config(dev_data->dma_aes_rx, &cfg); + if (ret) { + dev_err(dev_data->dev, "Can't configure IN dmaengine slave: %d\n", ret); + goto err_dma_config; + } + + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_maxburst = 4; + + ret = dmaengine_slave_config(dev_data->dma_aes_tx, &cfg); + if (ret) { + dev_err(dev_data->dev, "Can't configure OUT dmaengine slave: %d\n", ret); + goto err_dma_config; + } + + return 0; + +err_dma_config: + dma_release_channel(dev_data->dma_sha_tx); +err_dma_sha_tx: + dma_release_channel(dev_data->dma_aes_tx); +err_dma_aes_tx: + dma_release_channel(dev_data->dma_aes_rx); + + return ret; +} + +static int dthe_register_algs(void) +{ + return dthe_register_aes_algs(); +} + +static void dthe_unregister_algs(void) +{ + dthe_unregister_aes_algs(); +} + +static int dthe_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dthe_data *dev_data; + int ret; + + dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return -ENOMEM; + + dev_data->dev = dev; + dev_data->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dev_data->regs)) + return PTR_ERR(dev_data->regs); + + platform_set_drvdata(pdev, dev_data); + + spin_lock(&dthe_dev_list.lock); + list_add(&dev_data->list, &dthe_dev_list.dev_list); + spin_unlock(&dthe_dev_list.lock); + + ret = dthe_dma_init(dev_data); + if (ret) + goto probe_dma_err; + + dev_data->engine = crypto_engine_alloc_init(dev, 1); + if (!dev_data->engine) { + ret = -ENOMEM; + goto probe_engine_err; + } + + ret = crypto_engine_start(dev_data->engine); + if (ret) { + dev_err(dev, "Failed to start crypto engine\n"); + goto probe_engine_start_err; + } + + ret = dthe_register_algs(); + if (ret) { + dev_err(dev, "Failed to register algs\n"); + goto probe_engine_start_err; + } + + return 0; + +probe_engine_start_err: + crypto_engine_exit(dev_data->engine); +probe_engine_err: + dma_release_channel(dev_data->dma_aes_rx); + dma_release_channel(dev_data->dma_aes_tx); + dma_release_channel(dev_data->dma_sha_tx); +probe_dma_err: + spin_lock(&dthe_dev_list.lock); + list_del(&dev_data->list); + spin_unlock(&dthe_dev_list.lock); + + return ret; +} + +static void dthe_remove(struct platform_device *pdev) +{ + struct dthe_data *dev_data = platform_get_drvdata(pdev); + + spin_lock(&dthe_dev_list.lock); + list_del(&dev_data->list); + spin_unlock(&dthe_dev_list.lock); + + dthe_unregister_algs(); + + crypto_engine_exit(dev_data->engine); + + dma_release_channel(dev_data->dma_aes_rx); + dma_release_channel(dev_data->dma_aes_tx); + dma_release_channel(dev_data->dma_sha_tx); +} + +static const struct of_device_id dthe_of_match[] = { + { .compatible = "ti,am62l-dthev2", }, + {}, +}; +MODULE_DEVICE_TABLE(of, dthe_of_match); + +static struct platform_driver dthe_driver = { + .probe = dthe_probe, + .remove = dthe_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = dthe_of_match, + }, +}; + +module_platform_driver(dthe_driver); + +MODULE_AUTHOR("T Pratham <t-pratham@ti.com>"); +MODULE_DESCRIPTION("Texas Instruments DTHE V2 driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/ti/dthev2-common.h b/drivers/crypto/ti/dthev2-common.h new file mode 100644 index 000000000000..c7a06a4c353f --- /dev/null +++ b/drivers/crypto/ti/dthev2-common.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * K3 DTHE V2 crypto accelerator driver + * + * Copyright (C) Texas Instruments 2025 - https://www.ti.com + * Author: T Pratham <t-pratham@ti.com> + */ + +#ifndef __TI_DTHEV2_H__ +#define __TI_DTHEV2_H__ + +#include <crypto/aead.h> +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/engine.h> +#include <crypto/hash.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> + +#include <linux/delay.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/scatterlist.h> + +#define DTHE_REG_SIZE 4 +#define DTHE_DMA_TIMEOUT_MS 2000 +/* + * Size of largest possible key (of all algorithms) to be stored in dthe_tfm_ctx + * This is currently the keysize of XTS-AES-256 which is 512 bits (64 bytes) + */ +#define DTHE_MAX_KEYSIZE (AES_MAX_KEY_SIZE * 2) + +enum dthe_aes_mode { + DTHE_AES_ECB = 0, + DTHE_AES_CBC, + DTHE_AES_XTS, +}; + +/* Driver specific struct definitions */ + +/** + * struct dthe_data - DTHE_V2 driver instance data + * @dev: Device pointer + * @regs: Base address of the register space + * @list: list node for dev + * @engine: Crypto engine instance + * @dma_aes_rx: AES Rx DMA Channel + * @dma_aes_tx: AES Tx DMA Channel + * @dma_sha_tx: SHA Tx DMA Channel + */ +struct dthe_data { + struct device *dev; + void __iomem *regs; + struct list_head list; + struct crypto_engine *engine; + + struct dma_chan *dma_aes_rx; + struct dma_chan *dma_aes_tx; + + struct dma_chan *dma_sha_tx; +}; + +/** + * struct dthe_list - device data list head + * @dev_list: linked list head + * @lock: Spinlock protecting accesses to the list + */ +struct dthe_list { + struct list_head dev_list; + spinlock_t lock; +}; + +/** + * struct dthe_tfm_ctx - Transform ctx struct containing ctx for all sub-components of DTHE V2 + * @dev_data: Device data struct pointer + * @keylen: AES key length + * @key: AES key + * @aes_mode: AES mode + * @skcipher_fb: Fallback crypto skcipher handle for AES-XTS mode + */ +struct dthe_tfm_ctx { + struct dthe_data *dev_data; + unsigned int keylen; + u32 key[DTHE_MAX_KEYSIZE / sizeof(u32)]; + enum dthe_aes_mode aes_mode; + struct crypto_sync_skcipher *skcipher_fb; +}; + +/** + * struct dthe_aes_req_ctx - AES engine req ctx struct + * @enc: flag indicating encryption or decryption operation + * @aes_compl: Completion variable for use in manual completion in case of DMA callback failure + */ +struct dthe_aes_req_ctx { + int enc; + struct completion aes_compl; +}; + +/* Struct definitions end */ + +struct dthe_data *dthe_get_dev(struct dthe_tfm_ctx *ctx); + +int dthe_register_aes_algs(void); +void dthe_unregister_aes_algs(void); + +#endif diff --git a/drivers/crypto/xilinx/Makefile b/drivers/crypto/xilinx/Makefile index 730feff5b5f2..9b51636ef75e 100644 --- a/drivers/crypto/xilinx/Makefile +++ b/drivers/crypto/xilinx/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_XILINX_TRNG) += xilinx-trng.o obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_AES) += zynqmp-aes-gcm.o obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_SHA3) += zynqmp-sha.o diff --git a/drivers/crypto/xilinx/xilinx-trng.c b/drivers/crypto/xilinx/xilinx-trng.c new file mode 100644 index 000000000000..db0fbb28ff32 --- /dev/null +++ b/drivers/crypto/xilinx/xilinx-trng.c @@ -0,0 +1,430 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Versal True Random Number Generator driver + * Copyright (c) 2024 - 2025 Advanced Micro Devices, Inc. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/crypto.h> +#include <linux/delay.h> +#include <linux/firmware/xlnx-zynqmp.h> +#include <linux/hw_random.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/mod_devicetable.h> +#include <linux/platform_device.h> +#include <crypto/aes.h> +#include <crypto/df_sp80090a.h> +#include <crypto/internal/drbg.h> +#include <crypto/internal/cipher.h> +#include <crypto/internal/rng.h> + +/* TRNG Registers Offsets */ +#define TRNG_STATUS_OFFSET 0x4U +#define TRNG_CTRL_OFFSET 0x8U +#define TRNG_EXT_SEED_OFFSET 0x40U +#define TRNG_PER_STRNG_OFFSET 0x80U +#define TRNG_CORE_OUTPUT_OFFSET 0xC0U +#define TRNG_RESET_OFFSET 0xD0U +#define TRNG_OSC_EN_OFFSET 0xD4U + +/* Mask values */ +#define TRNG_RESET_VAL_MASK BIT(0) +#define TRNG_OSC_EN_VAL_MASK BIT(0) +#define TRNG_CTRL_PRNGSRST_MASK BIT(0) +#define TRNG_CTRL_EUMODE_MASK BIT(8) +#define TRNG_CTRL_TRSSEN_MASK BIT(2) +#define TRNG_CTRL_PRNGSTART_MASK BIT(5) +#define TRNG_CTRL_PRNGXS_MASK BIT(3) +#define TRNG_CTRL_PRNGMODE_MASK BIT(7) +#define TRNG_STATUS_DONE_MASK BIT(0) +#define TRNG_STATUS_QCNT_MASK GENMASK(11, 9) +#define TRNG_STATUS_QCNT_16_BYTES 0x800 + +/* Sizes in bytes */ +#define TRNG_SEED_LEN_BYTES 48U +#define TRNG_ENTROPY_SEED_LEN_BYTES 64U +#define TRNG_SEC_STRENGTH_SHIFT 5U +#define TRNG_SEC_STRENGTH_BYTES BIT(TRNG_SEC_STRENGTH_SHIFT) +#define TRNG_BYTES_PER_REG 4U +#define TRNG_RESET_DELAY 10 +#define TRNG_NUM_INIT_REGS 12U +#define TRNG_READ_4_WORD 4 +#define TRNG_DATA_READ_DELAY 8000 + +struct xilinx_rng { + void __iomem *rng_base; + struct device *dev; + unsigned char *scratchpadbuf; + struct crypto_aes_ctx *aesctx; + struct mutex lock; /* Protect access to TRNG device */ + struct hwrng trng; +}; + +struct xilinx_rng_ctx { + struct xilinx_rng *rng; +}; + +static struct xilinx_rng *xilinx_rng_dev; + +static void xtrng_readwrite32(void __iomem *addr, u32 mask, u8 value) +{ + u32 val; + + val = ioread32(addr); + val = (val & (~mask)) | (mask & value); + iowrite32(val, addr); +} + +static void xtrng_trng_reset(void __iomem *addr) +{ + xtrng_readwrite32(addr + TRNG_RESET_OFFSET, TRNG_RESET_VAL_MASK, TRNG_RESET_VAL_MASK); + udelay(TRNG_RESET_DELAY); + xtrng_readwrite32(addr + TRNG_RESET_OFFSET, TRNG_RESET_VAL_MASK, 0); +} + +static void xtrng_hold_reset(void __iomem *addr) +{ + xtrng_readwrite32(addr + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSRST_MASK, + TRNG_CTRL_PRNGSRST_MASK); + iowrite32(TRNG_RESET_VAL_MASK, addr + TRNG_RESET_OFFSET); + udelay(TRNG_RESET_DELAY); +} + +static void xtrng_softreset(struct xilinx_rng *rng) +{ + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSRST_MASK, + TRNG_CTRL_PRNGSRST_MASK); + udelay(TRNG_RESET_DELAY); + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSRST_MASK, 0); +} + +/* Return no. of bytes read */ +static size_t xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks32, bool wait) +{ + int read = 0, ret; + int timeout = 1; + int i, idx; + u32 val; + + if (wait) + timeout = TRNG_DATA_READ_DELAY; + + for (i = 0; i < (blocks32 * 2); i++) { + /* TRNG core generate data in 16 bytes. Read twice to complete 32 bytes read */ + ret = readl_poll_timeout(rng_base + TRNG_STATUS_OFFSET, val, + (val & TRNG_STATUS_QCNT_MASK) == + TRNG_STATUS_QCNT_16_BYTES, !!wait, timeout); + if (ret) + break; + + for (idx = 0; idx < TRNG_READ_4_WORD; idx++) { + *(buf + read) = cpu_to_be32(ioread32(rng_base + TRNG_CORE_OUTPUT_OFFSET)); + read += 1; + } + } + return read * 4; +} + +static int xtrng_collect_random_data(struct xilinx_rng *rng, u8 *rand_gen_buf, + int no_of_random_bytes, bool wait) +{ + u8 randbuf[TRNG_SEC_STRENGTH_BYTES]; + int byteleft, blocks, count = 0; + int ret; + + byteleft = no_of_random_bytes & (TRNG_SEC_STRENGTH_BYTES - 1); + blocks = no_of_random_bytes >> TRNG_SEC_STRENGTH_SHIFT; + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSTART_MASK, + TRNG_CTRL_PRNGSTART_MASK); + if (blocks) { + ret = xtrng_readblock32(rng->rng_base, (__be32 *)rand_gen_buf, blocks, wait); + if (!ret) + return 0; + count += ret; + } + + if (byteleft) { + ret = xtrng_readblock32(rng->rng_base, (__be32 *)randbuf, 1, wait); + if (!ret) + return count; + memcpy(rand_gen_buf + (blocks * TRNG_SEC_STRENGTH_BYTES), randbuf, byteleft); + count += byteleft; + } + + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, + TRNG_CTRL_PRNGMODE_MASK | TRNG_CTRL_PRNGSTART_MASK, 0U); + + return count; +} + +static void xtrng_write_multiple_registers(void __iomem *base_addr, u32 *values, size_t n) +{ + void __iomem *reg_addr; + size_t i; + + /* Write seed value into EXTERNAL_SEED Registers in big endian format */ + for (i = 0; i < n; i++) { + reg_addr = (base_addr + ((n - 1 - i) * TRNG_BYTES_PER_REG)); + iowrite32((u32 __force)(cpu_to_be32(values[i])), reg_addr); + } +} + +static void xtrng_enable_entropy(struct xilinx_rng *rng) +{ + iowrite32(TRNG_OSC_EN_VAL_MASK, rng->rng_base + TRNG_OSC_EN_OFFSET); + xtrng_softreset(rng); + iowrite32(TRNG_CTRL_EUMODE_MASK | TRNG_CTRL_TRSSEN_MASK, rng->rng_base + TRNG_CTRL_OFFSET); +} + +static int xtrng_reseed_internal(struct xilinx_rng *rng) +{ + u8 entropy[TRNG_ENTROPY_SEED_LEN_BYTES]; + struct drbg_string data; + LIST_HEAD(seedlist); + u32 val; + int ret; + + drbg_string_fill(&data, entropy, TRNG_SEED_LEN_BYTES); + list_add_tail(&data.list, &seedlist); + memset(entropy, 0, sizeof(entropy)); + xtrng_enable_entropy(rng); + + /* collect random data to use it as entropy (input for DF) */ + ret = xtrng_collect_random_data(rng, entropy, TRNG_SEED_LEN_BYTES, true); + if (ret != TRNG_SEED_LEN_BYTES) + return -EINVAL; + ret = crypto_drbg_ctr_df(rng->aesctx, rng->scratchpadbuf, + TRNG_SEED_LEN_BYTES, &seedlist, AES_BLOCK_SIZE, + TRNG_SEED_LEN_BYTES); + if (ret) + return ret; + + xtrng_write_multiple_registers(rng->rng_base + TRNG_EXT_SEED_OFFSET, + (u32 *)rng->scratchpadbuf, TRNG_NUM_INIT_REGS); + /* select reseed operation */ + iowrite32(TRNG_CTRL_PRNGXS_MASK, rng->rng_base + TRNG_CTRL_OFFSET); + + /* Start the reseed operation with above configuration and wait for STATUS.Done bit to be + * set. Monitor STATUS.CERTF bit, if set indicates SP800-90B entropy health test has failed. + */ + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSTART_MASK, + TRNG_CTRL_PRNGSTART_MASK); + + ret = readl_poll_timeout(rng->rng_base + TRNG_STATUS_OFFSET, val, + (val & TRNG_STATUS_DONE_MASK) == TRNG_STATUS_DONE_MASK, + 1U, 15000U); + if (ret) + return ret; + + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSTART_MASK, 0U); + + return 0; +} + +static int xtrng_random_bytes_generate(struct xilinx_rng *rng, u8 *rand_buf_ptr, + u32 rand_buf_size, int wait) +{ + int nbytes; + int ret; + + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, + TRNG_CTRL_PRNGMODE_MASK | TRNG_CTRL_PRNGXS_MASK, + TRNG_CTRL_PRNGMODE_MASK | TRNG_CTRL_PRNGXS_MASK); + nbytes = xtrng_collect_random_data(rng, rand_buf_ptr, rand_buf_size, wait); + + ret = xtrng_reseed_internal(rng); + if (ret) { + dev_err(rng->dev, "Re-seed fail\n"); + return ret; + } + + return nbytes; +} + +static int xtrng_trng_generate(struct crypto_rng *tfm, const u8 *src, u32 slen, + u8 *dst, u32 dlen) +{ + struct xilinx_rng_ctx *ctx = crypto_rng_ctx(tfm); + int ret; + + mutex_lock(&ctx->rng->lock); + ret = xtrng_random_bytes_generate(ctx->rng, dst, dlen, true); + mutex_unlock(&ctx->rng->lock); + + return ret < 0 ? ret : 0; +} + +static int xtrng_trng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) +{ + return 0; +} + +static int xtrng_trng_init(struct crypto_tfm *rtfm) +{ + struct xilinx_rng_ctx *ctx = crypto_tfm_ctx(rtfm); + + ctx->rng = xilinx_rng_dev; + + return 0; +} + +static struct rng_alg xtrng_trng_alg = { + .generate = xtrng_trng_generate, + .seed = xtrng_trng_seed, + .seedsize = 0, + .base = { + .cra_name = "stdrng", + .cra_driver_name = "xilinx-trng", + .cra_priority = 300, + .cra_ctxsize = sizeof(struct xilinx_rng_ctx), + .cra_module = THIS_MODULE, + .cra_init = xtrng_trng_init, + }, +}; + +static int xtrng_hwrng_trng_read(struct hwrng *hwrng, void *data, size_t max, bool wait) +{ + u8 buf[TRNG_SEC_STRENGTH_BYTES]; + struct xilinx_rng *rng; + int ret = -EINVAL, i = 0; + + rng = container_of(hwrng, struct xilinx_rng, trng); + /* Return in case wait not set and lock not available. */ + if (!mutex_trylock(&rng->lock) && !wait) + return 0; + else if (!mutex_is_locked(&rng->lock) && wait) + mutex_lock(&rng->lock); + + while (i < max) { + ret = xtrng_random_bytes_generate(rng, buf, TRNG_SEC_STRENGTH_BYTES, wait); + if (ret < 0) + break; + + memcpy(data + i, buf, min_t(int, ret, (max - i))); + i += min_t(int, ret, (max - i)); + } + mutex_unlock(&rng->lock); + + return ret; +} + +static int xtrng_hwrng_register(struct hwrng *trng) +{ + int ret; + + trng->name = "Xilinx Versal Crypto Engine TRNG"; + trng->read = xtrng_hwrng_trng_read; + + ret = hwrng_register(trng); + if (ret) + pr_err("Fail to register the TRNG\n"); + + return ret; +} + +static void xtrng_hwrng_unregister(struct hwrng *trng) +{ + hwrng_unregister(trng); +} + +static int xtrng_probe(struct platform_device *pdev) +{ + struct xilinx_rng *rng; + size_t sb_size; + int ret; + + rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); + if (!rng) + return -ENOMEM; + + rng->dev = &pdev->dev; + rng->rng_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(rng->rng_base)) { + dev_err(&pdev->dev, "Failed to map resource %pe\n", rng->rng_base); + return PTR_ERR(rng->rng_base); + } + + rng->aesctx = devm_kzalloc(&pdev->dev, sizeof(*rng->aesctx), GFP_KERNEL); + if (!rng->aesctx) + return -ENOMEM; + + sb_size = crypto_drbg_ctr_df_datalen(TRNG_SEED_LEN_BYTES, AES_BLOCK_SIZE); + rng->scratchpadbuf = devm_kzalloc(&pdev->dev, sb_size, GFP_KERNEL); + if (!rng->scratchpadbuf) { + ret = -ENOMEM; + goto end; + } + + xtrng_trng_reset(rng->rng_base); + ret = xtrng_reseed_internal(rng); + if (ret) { + dev_err(&pdev->dev, "TRNG Seed fail\n"); + goto end; + } + + xilinx_rng_dev = rng; + mutex_init(&rng->lock); + ret = crypto_register_rng(&xtrng_trng_alg); + if (ret) { + dev_err(&pdev->dev, "Crypto Random device registration failed: %d\n", ret); + goto end; + } + + ret = xtrng_hwrng_register(&rng->trng); + if (ret) { + dev_err(&pdev->dev, "HWRNG device registration failed: %d\n", ret); + goto crypto_rng_free; + } + platform_set_drvdata(pdev, rng); + + return 0; + +crypto_rng_free: + crypto_unregister_rng(&xtrng_trng_alg); + +end: + return ret; +} + +static void xtrng_remove(struct platform_device *pdev) +{ + struct xilinx_rng *rng; + u32 zero[TRNG_NUM_INIT_REGS] = { }; + + rng = platform_get_drvdata(pdev); + xtrng_hwrng_unregister(&rng->trng); + crypto_unregister_rng(&xtrng_trng_alg); + xtrng_write_multiple_registers(rng->rng_base + TRNG_EXT_SEED_OFFSET, zero, + TRNG_NUM_INIT_REGS); + xtrng_write_multiple_registers(rng->rng_base + TRNG_PER_STRNG_OFFSET, zero, + TRNG_NUM_INIT_REGS); + xtrng_hold_reset(rng->rng_base); + xilinx_rng_dev = NULL; +} + +static const struct of_device_id xtrng_of_match[] = { + { .compatible = "xlnx,versal-trng", }, + {}, +}; + +MODULE_DEVICE_TABLE(of, xtrng_of_match); + +static struct platform_driver xtrng_driver = { + .driver = { + .name = "xlnx,versal-trng", + .of_match_table = xtrng_of_match, + }, + .probe = xtrng_probe, + .remove = xtrng_remove, +}; + +module_platform_driver(xtrng_driver); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Harsh Jain <h.jain@amd.com>"); +MODULE_AUTHOR("Mounika Botcha <mounika.botcha@amd.com>"); +MODULE_DESCRIPTION("True Random Number Generator Driver"); |