xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Browse Source 
Pull crypto updates from Herbert Xu:
 "API:
   - Introduce crypto_shash_tfm_digest() and use it wherever possible.
   - Fix use-after-free and race in crypto_spawn_alg.
   - Add support for parallel and batch requests to crypto_engine.

  Algorithms:
   - Update jitter RNG for SP800-90B compliance.
   - Always use jitter RNG as seed in drbg.

  Drivers:
   - Add Arm CryptoCell driver cctrng.
   - Add support for SEV-ES to the PSP driver in ccp"

* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (114 commits)
  crypto: hisilicon - fix driver compatibility issue with different versions of devices
  crypto: engine - do not requeue in case of fatal error
  crypto: cavium/nitrox - Fix a typo in a comment
  crypto: hisilicon/qm - change debugfs file name from qm_regs to regs
  crypto: hisilicon/qm - add DebugFS for xQC and xQE dump
  crypto: hisilicon/zip - add debugfs for Hisilicon ZIP
  crypto: hisilicon/hpre - add debugfs for Hisilicon HPRE
  crypto: hisilicon/sec2 - add debugfs for Hisilicon SEC
  crypto: hisilicon/qm - add debugfs to the QM state machine
  crypto: hisilicon/qm - add debugfs for QM
  crypto: stm32/crc32 - protect from concurrent accesses
  crypto: stm32/crc32 - don't sleep in runtime pm
  crypto: stm32/crc32 - fix multi-instance
  crypto: stm32/crc32 - fix run-time self test issue.
  crypto: stm32/crc32 - fix ext4 chksum BUG_ON()
  crypto: hisilicon/zip - Use temporary sqe when doing work
  crypto: hisilicon - add device error report through abnormal irq
  crypto: hisilicon - remove codes of directly report device errors through MSI
  crypto: hisilicon - QM memory management optimization
  crypto: hisilicon - unify initial value assignment into QM
  ...
This commit is contained in:
Linus Torvalds
2020-06-01 12:00:10 -07:00
parent 729ea4e064 58ca0060ec
commit 81e8c10dac
127 changed files with 5021 additions and 1849 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
View File

@@ -537,10 +537,8 @@ static int sun8i_ss_probe(struct platform_device *pdev)
return err;
irq = platform_get_irq(pdev, 0);
if (irq < 0) {
dev_err(ss->dev, "Cannot get SecuritySystem IRQ\n");
if (irq < 0)
return irq;
}
ss->reset = devm_reset_control_get(&pdev->dev, NULL);
if (IS_ERR(ss->reset)) {

4
drivers/crypto/amlogic/amlogic-gxl-core.c
View File

@@ -253,10 +253,8 @@ static int meson_crypto_probe(struct platform_device *pdev)
mc->irqs = devm_kcalloc(mc->dev, MAXFLOW, sizeof(int), GFP_KERNEL);
for (i = 0; i < MAXFLOW; i++) {
mc->irqs[i] = platform_get_irq(pdev, i);
if (mc->irqs[i] < 0) {
dev_err(mc->dev, "Cannot get IRQ for flow %d\n", i);
if (mc->irqs[i] < 0)
return mc->irqs[i];
}
err = devm_request_irq(&pdev->dev, mc->irqs[i], meson_irq_handler, 0,
"gxl-crypto", mc);

1
drivers/crypto/atmel-sha.c
View File

@@ -31,7 +31,6 @@
#include <linux/of_device.h>
#include <linux/delay.h>
#include <linux/crypto.h>
#include <linux/cryptohash.h>
#include <crypto/scatterwalk.h>
#include <crypto/algapi.h>
#include <crypto/sha.h>

10
drivers/crypto/axis/artpec6_crypto.c
View File

@@ -2239,16 +2239,12 @@ artpec6_crypto_hash_set_key(struct crypto_ahash *tfm,
blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
if (keylen > blocksize) {
SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash);
hdesc->tfm = tfm_ctx->child_hash;
tfm_ctx->hmac_key_length = blocksize;
ret = crypto_shash_digest(hdesc, key, keylen,
tfm_ctx->hmac_key);
ret = crypto_shash_tfm_digest(tfm_ctx->child_hash, key, keylen,
tfm_ctx->hmac_key);
if (ret)
return ret;
} else {
memcpy(tfm_ctx->hmac_key, key, keylen);
tfm_ctx->hmac_key_length = keylen;

27
drivers/crypto/bcm/cipher.c
View File

@@ -308,9 +308,9 @@ static int handle_skcipher_req(struct iproc_reqctx_s *rctx)
container_of(areq, struct skcipher_request, base);
struct iproc_ctx_s *ctx = rctx->ctx;
struct spu_cipher_parms cipher_parms;
int err = 0;
unsigned int chunksize = 0; /* Num bytes of request to submit */
int remaining = 0; /* Bytes of request still to process */
int err;
unsigned int chunksize; /* Num bytes of request to submit */
int remaining; /* Bytes of request still to process */
int chunk_start; /* Beginning of data for current SPU msg */
/* IV or ctr value to use in this SPU msg */
@@ -698,7 +698,7 @@ static int handle_ahash_req(struct iproc_reqctx_s *rctx)
/* number of bytes still to be hashed in this req */
unsigned int nbytes_to_hash = 0;
int err = 0;
int err;
unsigned int chunksize = 0; /* length of hash carry + new data */
/*
* length of new data, not from hash carry, to be submitted in
@@ -1664,7 +1664,7 @@ static void spu_rx_callback(struct mbox_client *cl, void *msg)
struct spu_hw *spu = &iproc_priv.spu;
struct brcm_message *mssg = msg;
struct iproc_reqctx_s *rctx;
int err = 0;
int err;
rctx = mssg->ctx;
if (unlikely(!rctx)) {
@@ -1967,7 +1967,7 @@ static int ahash_enqueue(struct ahash_request *req)
struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
int err = 0;
int err;
const char *alg_name;
flow_log("ahash_enqueue() nbytes:%u\n", req->nbytes);
@@ -2299,7 +2299,7 @@ ahash_finup_exit:
static int ahash_digest(struct ahash_request *req)
{
int err = 0;
int err;
flow_log("ahash_digest() nbytes:%u\n", req->nbytes);
@@ -4436,7 +4436,7 @@ static int spu_mb_init(struct device *dev)
for (i = 0; i < iproc_priv.spu.num_chan; i++) {
iproc_priv.mbox[i] = mbox_request_channel(mcl, i);
if (IS_ERR(iproc_priv.mbox[i])) {
err = (int)PTR_ERR(iproc_priv.mbox[i]);
err = PTR_ERR(iproc_priv.mbox[i]);
dev_err(dev,
"Mbox channel %d request failed with err %d",
i, err);
@@ -4717,21 +4717,20 @@ static int spu_dt_read(struct platform_device *pdev)
matched_spu_type = of_device_get_match_data(dev);
if (!matched_spu_type) {
dev_err(&pdev->dev, "Failed to match device\n");
dev_err(dev, "Failed to match device\n");
return -ENODEV;
}
spu->spu_type = matched_spu_type->type;
spu->spu_subtype = matched_spu_type->subtype;
i = 0;
for (i = 0; (i < MAX_SPUS) && ((spu_ctrl_regs =
platform_get_resource(pdev, IORESOURCE_MEM, i)) != NULL); i++) {
spu->reg_vbase[i] = devm_ioremap_resource(dev, spu_ctrl_regs);
if (IS_ERR(spu->reg_vbase[i])) {
err = PTR_ERR(spu->reg_vbase[i]);
dev_err(&pdev->dev, "Failed to map registers: %d\n",
dev_err(dev, "Failed to map registers: %d\n",
err);
spu->reg_vbase[i] = NULL;
return err;
@@ -4747,7 +4746,7 @@ static int bcm_spu_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct spu_hw *spu = &iproc_priv.spu;
int err = 0;
int err;
iproc_priv.pdev = pdev;
platform_set_drvdata(iproc_priv.pdev,
@@ -4757,7 +4756,7 @@ static int bcm_spu_probe(struct platform_device *pdev)
if (err < 0)
goto failure;
err = spu_mb_init(&pdev->dev);
err = spu_mb_init(dev);
if (err < 0)
goto failure;
@@ -4766,7 +4765,7 @@ static int bcm_spu_probe(struct platform_device *pdev)
else if (spu->spu_type == SPU_TYPE_SPU2)
iproc_priv.bcm_hdr_len = 0;
spu_functions_register(&pdev->dev, spu->spu_type, spu->spu_subtype);
spu_functions_register(dev, spu->spu_type, spu->spu_subtype);
spu_counters_init();

2
drivers/crypto/cavium/nitrox/nitrox_main.c
View File

@@ -346,7 +346,7 @@ static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
}
/**
* nitrox_bist_check - Check NITORX BIST registers status
* nitrox_bist_check - Check NITROX BIST registers status
* @ndev: NITROX device
*/
static int nitrox_bist_check(struct nitrox_device *ndev)

3
drivers/crypto/ccp/Kconfig
View File

@@ -10,10 +10,9 @@ config CRYPTO_DEV_CCP_DD
config CRYPTO_DEV_SP_CCP
bool "Cryptographic Coprocessor device"
default y
depends on CRYPTO_DEV_CCP_DD
depends on CRYPTO_DEV_CCP_DD && DMADEVICES
select HW_RANDOM
select DMA_ENGINE
select DMADEVICES
select CRYPTO_SHA1
select CRYPTO_SHA256
help

9
drivers/crypto/ccp/ccp-crypto-sha.c
View File

@@ -272,9 +272,6 @@ static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
{
struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
struct crypto_shash *shash = ctx->u.sha.hmac_tfm;
SHASH_DESC_ON_STACK(sdesc, shash);
unsigned int block_size = crypto_shash_blocksize(shash);
unsigned int digest_size = crypto_shash_digestsize(shash);
int i, ret;
@@ -289,10 +286,8 @@ static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
if (key_len > block_size) {
/* Must hash the input key */
sdesc->tfm = shash;
ret = crypto_shash_digest(sdesc, key, key_len,
ctx->u.sha.key);
ret = crypto_shash_tfm_digest(shash, key, key_len,
ctx->u.sha.key);
if (ret)
return -EINVAL;

43
drivers/crypto/ccp/sev-dev.c
View File

@@ -20,6 +20,7 @@
#include <linux/hw_random.h>
#include <linux/ccp.h>
#include <linux/firmware.h>
#include <linux/gfp.h>
#include <asm/smp.h>
@@ -44,6 +45,14 @@ MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during
static bool psp_dead;
static int psp_timeout;
/* Trusted Memory Region (TMR):
* The TMR is a 1MB area that must be 1MB aligned. Use the page allocator
* to allocate the memory, which will return aligned memory for the specified
* allocation order.
*/
#define SEV_ES_TMR_SIZE (1024 * 1024)
static void *sev_es_tmr;
static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
{
struct sev_device *sev = psp_master->sev_data;
@@ -214,6 +223,20 @@ static int __sev_platform_init_locked(int *error)
if (sev->state == SEV_STATE_INIT)
return 0;
if (sev_es_tmr) {
u64 tmr_pa;
/*
* Do not include the encryption mask on the physical
* address of the TMR (firmware should clear it anyway).
*/
tmr_pa = __pa(sev_es_tmr);
sev->init_cmd_buf.flags |= SEV_INIT_FLAGS_SEV_ES;
sev->init_cmd_buf.tmr_address = tmr_pa;
sev->init_cmd_buf.tmr_len = SEV_ES_TMR_SIZE;
}
rc = __sev_do_cmd_locked(SEV_CMD_INIT, &sev->init_cmd_buf, error);
if (rc)
return rc;
@@ -1012,6 +1035,7 @@ EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
void sev_pci_init(void)
{
struct sev_device *sev = psp_master->sev_data;
struct page *tmr_page;
int error, rc;
if (!sev)
@@ -1041,6 +1065,16 @@ void sev_pci_init(void)
sev_update_firmware(sev->dev) == 0)
sev_get_api_version();
/* Obtain the TMR memory area for SEV-ES use */
tmr_page = alloc_pages(GFP_KERNEL, get_order(SEV_ES_TMR_SIZE));
if (tmr_page) {
sev_es_tmr = page_address(tmr_page);
} else {
sev_es_tmr = NULL;
dev_warn(sev->dev,
"SEV: TMR allocation failed, SEV-ES support unavailable\n");
}
/* Initialize the platform */
rc = sev_platform_init(&error);
if (rc && (error == SEV_RET_SECURE_DATA_INVALID)) {
@@ -1075,4 +1109,13 @@ void sev_pci_exit(void)
return;
sev_platform_shutdown(NULL);
if (sev_es_tmr) {
/* The TMR area was encrypted, flush it from the cache */
wbinvd_on_all_cpus();
free_pages((unsigned long)sev_es_tmr,
get_order(SEV_ES_TMR_SIZE));
sev_es_tmr = NULL;
}
}

9
drivers/crypto/ccree/cc_cipher.c
View File

@@ -427,12 +427,9 @@ static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key,
int key_len = keylen >> 1;
int err;
SHASH_DESC_ON_STACK(desc, ctx_p->shash_tfm);
desc->tfm = ctx_p->shash_tfm;
err = crypto_shash_digest(desc, ctx_p->user.key, key_len,
ctx_p->user.key + key_len);
err = crypto_shash_tfm_digest(ctx_p->shash_tfm,
ctx_p->user.key, key_len,
ctx_p->user.key + key_len);
if (err) {
dev_err(dev, "Failed to hash ESSIV key.\n");
return err;

4
drivers/crypto/ccree/cc_debugfs.c
View File

@@ -26,7 +26,7 @@ static struct debugfs_reg32 ver_sig_regs[] = {
{ .name = "VERSION" }, /* Must be 1st */
};
static struct debugfs_reg32 pid_cid_regs[] = {
static const struct debugfs_reg32 pid_cid_regs[] = {
CC_DEBUG_REG(PERIPHERAL_ID_0),
CC_DEBUG_REG(PERIPHERAL_ID_1),
CC_DEBUG_REG(PERIPHERAL_ID_2),
@@ -38,7 +38,7 @@ static struct debugfs_reg32 pid_cid_regs[] = {
CC_DEBUG_REG(COMPONENT_ID_3),
};
static struct debugfs_reg32 debug_regs[] = {
static const struct debugfs_reg32 debug_regs[] = {
CC_DEBUG_REG(HOST_IRR),
CC_DEBUG_REG(HOST_POWER_DOWN_EN),
CC_DEBUG_REG(AXIM_MON_ERR),

3
drivers/crypto/chelsio/chcr_algo.c
View File

@@ -44,7 +44,6 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/crypto.h>
#include <linux/cryptohash.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/highmem.h>
@@ -1757,7 +1756,7 @@ static int chcr_ahash_final(struct ahash_request *req)
struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
struct chcr_context *ctx = h_ctx(rtfm);
u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
int error = -EINVAL;
int error;
unsigned int cpu;
cpu = get_cpu();

1
drivers/crypto/chelsio/chcr_ipsec.c
View File

@@ -40,7 +40,6 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/crypto.h>
#include <linux/cryptohash.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/highmem.h>

4
drivers/crypto/hisilicon/Kconfig
View File

@@ -29,6 +29,7 @@ config CRYPTO_DEV_HISI_SEC2
depends on PCI && PCI_MSI
depends on UACCE || UACCE=n
depends on ARM64 || (COMPILE_TEST && 64BIT)
depends on ACPI
help
Support for HiSilicon SEC Engine of version 2 in crypto subsystem.
It provides AES, SM4, and 3DES algorithms with ECB
@@ -42,6 +43,7 @@ config CRYPTO_DEV_HISI_QM
depends on ARM64 || COMPILE_TEST
depends on PCI && PCI_MSI
depends on UACCE || UACCE=n
depends on ACPI
help
HiSilicon accelerator engines use a common queue management
interface. Specific engine driver may use this module.
@@ -52,6 +54,7 @@ config CRYPTO_DEV_HISI_ZIP
depends on ARM64 || (COMPILE_TEST && 64BIT)
depends on !CPU_BIG_ENDIAN || COMPILE_TEST
depends on UACCE || UACCE=n
depends on ACPI
select CRYPTO_DEV_HISI_QM
help
Support for HiSilicon ZIP Driver
@@ -61,6 +64,7 @@ config CRYPTO_DEV_HISI_HPRE
depends on PCI && PCI_MSI
depends on UACCE || UACCE=n
depends on ARM64 || (COMPILE_TEST && 64BIT)
depends on ACPI
select CRYPTO_DEV_HISI_QM
select CRYPTO_DH
select CRYPTO_RSA

18
drivers/crypto/hisilicon/hpre/hpre.h
View File

@@ -25,6 +25,17 @@ enum hpre_ctrl_dbgfs_file {
HPRE_DEBUG_FILE_NUM,
};
enum hpre_dfx_dbgfs_file {
HPRE_SEND_CNT,
HPRE_RECV_CNT,
HPRE_SEND_FAIL_CNT,
HPRE_SEND_BUSY_CNT,
HPRE_OVER_THRHLD_CNT,
HPRE_OVERTIME_THRHLD,
HPRE_INVALID_REQ_CNT,
HPRE_DFX_FILE_NUM
};
#define HPRE_DEBUGFS_FILE_NUM (HPRE_DEBUG_FILE_NUM + HPRE_CLUSTERS_NUM - 1)
struct hpre_debugfs_file {
@@ -34,6 +45,11 @@ struct hpre_debugfs_file {
struct hpre_debug *debug;
};
struct hpre_dfx {
atomic64_t value;
enum hpre_dfx_dbgfs_file type;
};
/*
* One HPRE controller has one PF and multiple VFs, some global configurations
* which PF has need this structure.
@@ -41,13 +57,13 @@ struct hpre_debugfs_file {
*/
struct hpre_debug {
struct dentry *debug_root;
struct hpre_dfx dfx[HPRE_DFX_FILE_NUM];
struct hpre_debugfs_file files[HPRE_DEBUGFS_FILE_NUM];
};
struct hpre {
struct hisi_qm qm;
struct hpre_debug debug;
u32 num_vfs;
unsigned long status;
};

99
drivers/crypto/hisilicon/hpre/hpre_crypto.c
View File

@@ -10,6 +10,7 @@
#include <linux/dma-mapping.h>
#include <linux/fips.h>
#include <linux/module.h>
#include <linux/time.h>
#include "hpre.h"
struct hpre_ctx;
@@ -32,6 +33,9 @@ struct hpre_ctx;
#define HPRE_SQE_DONE_SHIFT 30
#define HPRE_DH_MAX_P_SZ 512
#define HPRE_DFX_SEC_TO_US 1000000
#define HPRE_DFX_US_TO_NS 1000
typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
struct hpre_rsa_ctx {
@@ -68,6 +72,7 @@ struct hpre_dh_ctx {
struct hpre_ctx {
struct hisi_qp *qp;
struct hpre_asym_request **req_list;
struct hpre *hpre;
spinlock_t req_lock;
unsigned int key_sz;
bool crt_g2_mode;
@@ -90,6 +95,7 @@ struct hpre_asym_request {
int err;
int req_id;
hpre_cb cb;
struct timespec64 req_time;
};
static DEFINE_MUTEX(hpre_alg_lock);
@@ -119,6 +125,7 @@ static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)
static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
{
struct hpre_ctx *ctx;
struct hpre_dfx *dfx;
int id;
ctx = hpre_req->ctx;
@@ -129,6 +136,10 @@ static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
ctx->req_list[id] = hpre_req;
hpre_req->req_id = id;
dfx = ctx->hpre->debug.dfx;
if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value))
ktime_get_ts64(&hpre_req->req_time);
return id;
}
@@ -309,12 +320,16 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
{
struct hpre *hpre;
if (!ctx || !qp || qlen < 0)
return -EINVAL;
spin_lock_init(&ctx->req_lock);
ctx->qp = qp;
hpre = container_of(ctx->qp->qm, struct hpre, qm);
ctx->hpre = hpre;
ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);
if (!ctx->req_list)
return -ENOMEM;
@@ -337,38 +352,80 @@ static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all)
ctx->key_sz = 0;
}
static bool hpre_is_bd_timeout(struct hpre_asym_request *req,
u64 overtime_thrhld)
{
struct timespec64 reply_time;
u64 time_use_us;
ktime_get_ts64(&reply_time);
time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) *
HPRE_DFX_SEC_TO_US +
(reply_time.tv_nsec - req->req_time.tv_nsec) /
HPRE_DFX_US_TO_NS;
if (time_use_us <= overtime_thrhld)
return false;
return true;
}
static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
{
struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
struct hpre_asym_request *req;
struct kpp_request *areq;
u64 overtime_thrhld;
int ret;
ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
areq = req->areq.dh;
areq->dst_len = ctx->key_sz;
overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
kpp_request_complete(areq, ret);
atomic64_inc(&dfx[HPRE_RECV_CNT].value);
}
static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)
{
struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
struct hpre_asym_request *req;
struct akcipher_request *areq;
u64 overtime_thrhld;
int ret;
ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
areq = req->areq.rsa;
areq->dst_len = ctx->key_sz;
hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
akcipher_request_complete(areq, ret);
atomic64_inc(&dfx[HPRE_RECV_CNT].value);
}
static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
{
struct hpre_ctx *ctx = qp->qp_ctx;
struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
struct hpre_sqe *sqe = resp;
struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)];
ctx->req_list[le16_to_cpu(sqe->tag)]->cb(ctx, resp);
if (unlikely(!req)) {
atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value);
return;
}
req->cb(ctx, resp);
}
static int hpre_ctx_init(struct hpre_ctx *ctx)
@@ -436,6 +493,29 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
return 0;
}
static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
{
struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
int ctr = 0;
int ret;
do {
atomic64_inc(&dfx[HPRE_SEND_CNT].value);
ret = hisi_qp_send(ctx->qp, msg);
if (ret != -EBUSY)
break;
atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
} while (ctr++ < HPRE_TRY_SEND_TIMES);
if (likely(!ret))
return ret;
if (ret != -EBUSY)
atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
return ret;
}
#ifdef CONFIG_CRYPTO_DH
static int hpre_dh_compute_value(struct kpp_request *req)
{
@@ -444,7 +524,6 @@ static int hpre_dh_compute_value(struct kpp_request *req)
void *tmp = kpp_request_ctx(req);
struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
struct hpre_sqe *msg = &hpre_req->req;
int ctr = 0;
int ret;
ret = hpre_msg_request_set(ctx, req, false);
@@ -465,11 +544,9 @@ static int hpre_dh_compute_value(struct kpp_request *req)
msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2);
else
msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
do {
ret = hisi_qp_send(ctx->qp, msg);
} while (ret == -EBUSY && ctr++ < HPRE_TRY_SEND_TIMES);
/* success */
ret = hpre_send(ctx, msg);
if (likely(!ret))
return -EINPROGRESS;
@@ -647,7 +724,6 @@ static int hpre_rsa_enc(struct akcipher_request *req)
void *tmp = akcipher_request_ctx(req);
struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
struct hpre_sqe *msg = &hpre_req->req;
int ctr = 0;
int ret;
/* For 512 and 1536 bits key size, use soft tfm instead */
@@ -677,11 +753,8 @@ static int hpre_rsa_enc(struct akcipher_request *req)
if (unlikely(ret))
goto clear_all;
do {
ret = hisi_qp_send(ctx->qp, msg);
} while (ret == -EBUSY && ctr++ < HPRE_TRY_SEND_TIMES);
/* success */
ret = hpre_send(ctx, msg);
if (likely(!ret))
return -EINPROGRESS;
@@ -699,7 +772,6 @@ static int hpre_rsa_dec(struct akcipher_request *req)
void *tmp = akcipher_request_ctx(req);
struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
struct hpre_sqe *msg = &hpre_req->req;
int ctr = 0;
int ret;
/* For 512 and 1536 bits key size, use soft tfm instead */
@@ -736,11 +808,8 @@ static int hpre_rsa_dec(struct akcipher_request *req)
if (unlikely(ret))
goto clear_all;
do {
ret = hisi_qp_send(ctx->qp, msg);
} while (ret == -EBUSY && ctr++ < HPRE_TRY_SEND_TIMES);
/* success */
ret = hpre_send(ctx, msg);
if (likely(!ret))
return -EINPROGRESS;

361
drivers/crypto/hisilicon/hpre/hpre_main.c
View File

@@ -59,10 +59,6 @@
#define HPRE_HAC_ECC2_CNT 0x301a08
#define HPRE_HAC_INT_STATUS 0x301800
#define HPRE_HAC_SOURCE_INT 0x301600
#define MASTER_GLOBAL_CTRL_SHUTDOWN 1
#define MASTER_TRANS_RETURN_RW 3
#define HPRE_MASTER_TRANS_RETURN 0x300150
#define HPRE_MASTER_GLOBAL_CTRL 0x300000
#define HPRE_CLSTR_ADDR_INTRVL 0x1000
#define HPRE_CLUSTER_INQURY 0x100
#define HPRE_CLSTR_ADDR_INQRY_RSLT 0x104
@@ -80,7 +76,16 @@
#define HPRE_BD_USR_MASK 0x3
#define HPRE_CLUSTER_CORE_MASK 0xf
#define HPRE_AM_OOO_SHUTDOWN_ENB 0x301044
#define HPRE_AM_OOO_SHUTDOWN_ENABLE BIT(0)
#define HPRE_WR_MSI_PORT BIT(2)
#define HPRE_CORE_ECC_2BIT_ERR BIT(1)
#define HPRE_OOO_ECC_2BIT_ERR BIT(5)
#define HPRE_VIA_MSI_DSM 1
#define HPRE_SQE_MASK_OFFSET 8
#define HPRE_SQE_MASK_LEN 24
static struct hisi_qm_list hpre_devices;
static const char hpre_name[] = "hisi_hpre";
@@ -131,7 +136,7 @@ static const u64 hpre_cluster_offsets[] = {
HPRE_CLSTR_BASE + HPRE_CLUSTER3 * HPRE_CLSTR_ADDR_INTRVL,
};
static struct debugfs_reg32 hpre_cluster_dfx_regs[] = {
static const struct debugfs_reg32 hpre_cluster_dfx_regs[] = {
{"CORES_EN_STATUS ", HPRE_CORE_EN_OFFSET},
{"CORES_INI_CFG ", HPRE_CORE_INI_CFG_OFFSET},
{"CORES_INI_STATUS ", HPRE_CORE_INI_STATUS_OFFSET},
@@ -139,7 +144,7 @@ static struct debugfs_reg32 hpre_cluster_dfx_regs[] = {
{"CORES_IS_SCHD ", HPRE_CORE_IS_SCHD_OFFSET},
};
static struct debugfs_reg32 hpre_com_dfx_regs[] = {
static const struct debugfs_reg32 hpre_com_dfx_regs[] = {
{"READ_CLR_EN ", HPRE_CTRL_CNT_CLR_CE},
{"AXQOS ", HPRE_VFG_AXQOS},
{"AWUSR_CFG ", HPRE_AWUSR_FP_CFG},
@@ -156,44 +161,38 @@ static struct debugfs_reg32 hpre_com_dfx_regs[] = {
{"INT_STATUS ", HPRE_INT_STATUS},
};
static int hpre_pf_q_num_set(const char *val, const struct kernel_param *kp)
static const char *hpre_dfx_files[HPRE_DFX_FILE_NUM] = {
"send_cnt",
"recv_cnt",
"send_fail_cnt",
"send_busy_cnt",
"over_thrhld_cnt",
"overtime_thrhld",
"invalid_req_cnt"
};
static int pf_q_num_set(const char *val, const struct kernel_param *kp)
{
struct pci_dev *pdev;
u32 n, q_num;
u8 rev_id;
int ret;
if (!val)
return -EINVAL;
pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, HPRE_PCI_DEVICE_ID, NULL);
if (!pdev) {
q_num = HPRE_QUEUE_NUM_V2;
pr_info("No device found currently, suppose queue number is %d\n",
q_num);
} else {
rev_id = pdev->revision;
if (rev_id != QM_HW_V2)
return -EINVAL;
q_num = HPRE_QUEUE_NUM_V2;
}
ret = kstrtou32(val, 10, &n);
if (ret != 0 || n == 0 || n > q_num)
return -EINVAL;
return param_set_int(val, kp);
return q_num_set(val, kp, HPRE_PCI_DEVICE_ID);
}
static const struct kernel_param_ops hpre_pf_q_num_ops = {
.set = hpre_pf_q_num_set,
.set = pf_q_num_set,
.get = param_get_int,
};
static u32 hpre_pf_q_num = HPRE_PF_DEF_Q_NUM;
module_param_cb(hpre_pf_q_num, &hpre_pf_q_num_ops, &hpre_pf_q_num, 0444);
MODULE_PARM_DESC(hpre_pf_q_num, "Number of queues in PF of CS(1-1024)");
static u32 pf_q_num = HPRE_PF_DEF_Q_NUM;
module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444);
MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(1-1024)");
static const struct kernel_param_ops vfs_num_ops = {
.set = vfs_num_set,
.get = param_get_int,
};
static u32 vfs_num;
module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
struct hisi_qp *hpre_create_qp(void)
{
@@ -232,9 +231,8 @@ static int hpre_cfg_by_dsm(struct hisi_qm *qm)
return 0;
}
static int hpre_set_user_domain_and_cache(struct hpre *hpre)
static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)
{
struct hisi_qm *qm = &hpre->qm;
struct device *dev = &qm->pdev->dev;
unsigned long offset;
int ret, i;
@@ -324,17 +322,34 @@ static void hpre_cnt_regs_clear(struct hisi_qm *qm)
static void hpre_hw_error_disable(struct hisi_qm *qm)
{
u32 val;
/* disable hpre hw error interrupts */
writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK);
/* disable HPRE block master OOO when m-bit error occur */
val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
val &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE;
writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
}
static void hpre_hw_error_enable(struct hisi_qm *qm)
{
u32 val;
/* clear HPRE hw error source if having */
writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_HAC_SOURCE_INT);
/* enable hpre hw error interrupts */
writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK);
writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB);
writel(HPRE_HAC_RAS_NFE_ENABLE, qm->io_base + HPRE_RAS_NFE_ENB);
writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB);
/* enable HPRE block master OOO when m-bit error occur */
val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
val |= HPRE_AM_OOO_SHUTDOWN_ENABLE;
writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
}
static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file)
@@ -354,9 +369,7 @@ static u32 hpre_current_qm_read(struct hpre_debugfs_file *file)
static int hpre_current_qm_write(struct hpre_debugfs_file *file, u32 val)
{
struct hisi_qm *qm = hpre_file_to_qm(file);
struct hpre_debug *debug = file->debug;
struct hpre *hpre = container_of(debug, struct hpre, debug);
u32 num_vfs = hpre->num_vfs;
u32 num_vfs = qm->vfs_num;
u32 vfq_num, tmp;
@@ -523,6 +536,33 @@ static const struct file_operations hpre_ctrl_debug_fops = {
.write = hpre_ctrl_debug_write,
};
static int hpre_debugfs_atomic64_get(void *data, u64 *val)
{
struct hpre_dfx *dfx_item = data;
*val = atomic64_read(&dfx_item->value);
return 0;
}
static int hpre_debugfs_atomic64_set(void *data, u64 val)
{
struct hpre_dfx *dfx_item = data;
struct hpre_dfx *hpre_dfx = dfx_item - HPRE_OVERTIME_THRHLD;
if (val)
return -EINVAL;
if (dfx_item->type == HPRE_OVERTIME_THRHLD)
atomic64_set(&hpre_dfx[HPRE_OVER_THRHLD_CNT].value, 0);
atomic64_set(&dfx_item->value, val);
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(hpre_atomic64_ops, hpre_debugfs_atomic64_get,
hpre_debugfs_atomic64_set, "%llu\n");
static int hpre_create_debugfs_file(struct hpre_debug *dbg, struct dentry *dir,
enum hpre_ctrl_dbgfs_file type, int indx)
{
@@ -620,6 +660,22 @@ static int hpre_ctrl_debug_init(struct hpre_debug *debug)
return hpre_cluster_debugfs_init(debug);
}
static void hpre_dfx_debug_init(struct hpre_debug *debug)
{
struct hpre *hpre = container_of(debug, struct hpre, debug);
struct hpre_dfx *dfx = hpre->debug.dfx;
struct hisi_qm *qm = &hpre->qm;
struct dentry *parent;
int i;
parent = debugfs_create_dir("hpre_dfx", qm->debug.debug_root);
for (i = 0; i < HPRE_DFX_FILE_NUM; i++) {
dfx[i].type = i;
debugfs_create_file(hpre_dfx_files[i], 0644, parent, &dfx[i],
&hpre_atomic64_ops);
}
}
static int hpre_debugfs_init(struct hpre *hpre)
{
struct hisi_qm *qm = &hpre->qm;
@@ -629,6 +685,8 @@ static int hpre_debugfs_init(struct hpre *hpre)
dir = debugfs_create_dir(dev_name(dev), hpre_debugfs_root);
qm->debug.debug_root = dir;
qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET;
qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN;
ret = hisi_qm_debug_init(qm);
if (ret)
@@ -640,6 +698,9 @@ static int hpre_debugfs_init(struct hpre *hpre)
if (ret)
goto failed_to_create;
}
hpre_dfx_debug_init(&hpre->debug);
return 0;
failed_to_create:
@@ -654,32 +715,27 @@ static void hpre_debugfs_exit(struct hpre *hpre)
debugfs_remove_recursive(qm->debug.debug_root);
}
static int hpre_qm_pre_init(struct hisi_qm *qm, struct pci_dev *pdev)
static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
{
enum qm_hw_ver rev_id;
rev_id = hisi_qm_get_hw_version(pdev);
if (rev_id < 0)
return -ENODEV;
if (rev_id == QM_HW_V1) {
if (pdev->revision == QM_HW_V1) {
pci_warn(pdev, "HPRE version 1 is not supported!\n");
return -EINVAL;
}
qm->pdev = pdev;
qm->ver = rev_id;
qm->ver = pdev->revision;
qm->sqe_size = HPRE_SQE_SIZE;
qm->dev_name = hpre_name;
qm->fun_type = (pdev->device == HPRE_PCI_DEVICE_ID) ?
QM_HW_PF : QM_HW_VF;
if (pdev->is_physfn) {
qm->qp_base = HPRE_PF_DEF_Q_BASE;
qm->qp_num = hpre_pf_q_num;
}
qm->use_dma_api = true;
return 0;
qm->fun_type = (pdev->device == HPRE_PCI_DEVICE_ID) ?
QM_HW_PF : QM_HW_VF;
if (qm->fun_type == QM_HW_PF) {
qm->qp_base = HPRE_PF_DEF_Q_BASE;
qm->qp_num = pf_q_num;
qm->qm_list = &hpre_devices;
}
return hisi_qm_init(qm);
}
static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts)
@@ -693,8 +749,6 @@ static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts)
err->msg, err->int_msk);
err++;
}
writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT);
}
static u32 hpre_get_hw_err_status(struct hisi_qm *qm)
@@ -702,16 +756,38 @@ static u32 hpre_get_hw_err_status(struct hisi_qm *qm)
return readl(qm->io_base + HPRE_HAC_INT_STATUS);
}
static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
{
writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT);
}
static void hpre_open_axi_master_ooo(struct hisi_qm *qm)
{
u32 value;
value = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
writel(value & ~HPRE_AM_OOO_SHUTDOWN_ENABLE,
HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB));
writel(value | HPRE_AM_OOO_SHUTDOWN_ENABLE,
HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB));
}
static const struct hisi_qm_err_ini hpre_err_ini = {
.hw_init = hpre_set_user_domain_and_cache,
.hw_err_enable = hpre_hw_error_enable,
.hw_err_disable = hpre_hw_error_disable,
.get_dev_hw_err_status = hpre_get_hw_err_status,
.clear_dev_hw_err_status = hpre_clear_hw_err_status,
.log_dev_hw_err = hpre_log_hw_error,
.open_axi_master_ooo = hpre_open_axi_master_ooo,
.err_info = {
.ce = QM_BASE_CE,
.nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT,
.fe = 0,
.msi = QM_DB_RANDOM_INVALID,
.ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR |
HPRE_OOO_ECC_2BIT_ERR,
.msi_wr_port = HPRE_WR_MSI_PORT,
.acpi_rst = "HRST",
}
};
@@ -722,7 +798,7 @@ static int hpre_pf_probe_init(struct hpre *hpre)
qm->ctrl_qp_num = HPRE_QUEUE_NUM_V2;
ret = hpre_set_user_domain_and_cache(hpre);
ret = hpre_set_user_domain_and_cache(qm);
if (ret)
return ret;
@@ -732,6 +808,20 @@ static int hpre_pf_probe_init(struct hpre *hpre)
return 0;
}
static int hpre_probe_init(struct hpre *hpre)
{
struct hisi_qm *qm = &hpre->qm;
int ret;
if (qm->fun_type == QM_HW_PF) {
ret = hpre_pf_probe_init(hpre);
if (ret)
return ret;
}
return 0;
}
static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct hisi_qm *qm;
@@ -742,26 +832,17 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (!hpre)
return -ENOMEM;
pci_set_drvdata(pdev, hpre);
qm = &hpre->qm;
ret = hpre_qm_pre_init(qm, pdev);
if (ret)
ret = hpre_qm_init(qm, pdev);
if (ret) {
pci_err(pdev, "Failed to init HPRE QM (%d)!\n", ret);
return ret;
}
ret = hisi_qm_init(qm);
if (ret)
return ret;
if (pdev->is_physfn) {
ret = hpre_pf_probe_init(hpre);
if (ret)
goto err_with_qm_init;
} else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V2) {
/* v2 starts to support get vft by mailbox */
ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
if (ret)
goto err_with_qm_init;
ret = hpre_probe_init(hpre);
if (ret) {
pci_err(pdev, "Failed to probe (%d)!\n", ret);
goto err_with_qm_init;
}
ret = hisi_qm_start(qm);
@@ -779,8 +860,18 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id)
pci_err(pdev, "fail to register algs to crypto!\n");
goto err_with_qm_start;
}
if (qm->fun_type == QM_HW_PF && vfs_num) {
ret = hisi_qm_sriov_enable(pdev, vfs_num);
if (ret < 0)
goto err_with_crypto_register;
}
return 0;
err_with_crypto_register:
hpre_algs_unregister();
err_with_qm_start:
hisi_qm_del_from_list(qm, &hpre_devices);
hisi_qm_stop(qm);
@@ -794,107 +885,6 @@ err_with_qm_init:
return ret;
}
static int hpre_vf_q_assign(struct hpre *hpre, int num_vfs)
{
struct hisi_qm *qm = &hpre->qm;
u32 qp_num = qm->qp_num;
int q_num, remain_q_num, i;
u32 q_base = qp_num;
int ret;
if (!num_vfs)
return -EINVAL;
remain_q_num = qm->ctrl_qp_num - qp_num;
/* If remaining queues are not enough, return error. */
if (remain_q_num < num_vfs)
return -EINVAL;
q_num = remain_q_num / num_vfs;
for (i = 1; i <= num_vfs; i++) {
if (i == num_vfs)
q_num += remain_q_num % num_vfs;
ret = hisi_qm_set_vft(qm, i, q_base, (u32)q_num);
if (ret)
return ret;
q_base += q_num;
}
return 0;
}
static int hpre_clear_vft_config(struct hpre *hpre)
{
struct hisi_qm *qm = &hpre->qm;
u32 num_vfs = hpre->num_vfs;
int ret;
u32 i;
for (i = 1; i <= num_vfs; i++) {
ret = hisi_qm_set_vft(qm, i, 0, 0);
if (ret)
return ret;
}
hpre->num_vfs = 0;
return 0;
}
static int hpre_sriov_enable(struct pci_dev *pdev, int max_vfs)
{
struct hpre *hpre = pci_get_drvdata(pdev);
int pre_existing_vfs, num_vfs, ret;
pre_existing_vfs = pci_num_vf(pdev);
if (pre_existing_vfs) {
pci_err(pdev,
"Can't enable VF. Please disable pre-enabled VFs!\n");
return 0;
}
num_vfs = min_t(int, max_vfs, HPRE_VF_NUM);
ret = hpre_vf_q_assign(hpre, num_vfs);
if (ret) {
pci_err(pdev, "Can't assign queues for VF!\n");
return ret;
}
hpre->num_vfs = num_vfs;
ret = pci_enable_sriov(pdev, num_vfs);
if (ret) {
pci_err(pdev, "Can't enable VF!\n");
hpre_clear_vft_config(hpre);
return ret;
}
return num_vfs;
}
static int hpre_sriov_disable(struct pci_dev *pdev)
{
struct hpre *hpre = pci_get_drvdata(pdev);
if (pci_vfs_assigned(pdev)) {
pci_err(pdev, "Failed to disable VFs while VFs are assigned!\n");
return -EPERM;
}
/* remove in hpre_pci_driver will be called to free VF resources */
pci_disable_sriov(pdev);
return hpre_clear_vft_config(hpre);
}
static int hpre_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
if (num_vfs)
return hpre_sriov_enable(pdev, num_vfs);
else
return hpre_sriov_disable(pdev);
}
static void hpre_remove(struct pci_dev *pdev)
{
struct hpre *hpre = pci_get_drvdata(pdev);
@@ -903,8 +893,8 @@ static void hpre_remove(struct pci_dev *pdev)
hpre_algs_unregister();
hisi_qm_del_from_list(qm, &hpre_devices);
if (qm->fun_type == QM_HW_PF && hpre->num_vfs != 0) {
ret = hpre_sriov_disable(pdev);
if (qm->fun_type == QM_HW_PF && qm->vfs_num) {
ret = hisi_qm_sriov_disable(pdev);
if (ret) {
pci_err(pdev, "Disable SRIOV fail!\n");
return;
@@ -924,6 +914,9 @@ static void hpre_remove(struct pci_dev *pdev)
static const struct pci_error_handlers hpre_err_handler = {
.error_detected = hisi_qm_dev_err_detected,
.slot_reset = hisi_qm_dev_slot_reset,
.reset_prepare = hisi_qm_reset_prepare,
.reset_done = hisi_qm_reset_done,
};
static struct pci_driver hpre_pci_driver = {
@@ -931,7 +924,7 @@ static struct pci_driver hpre_pci_driver = {
.id_table = hpre_dev_ids,
.probe = hpre_probe,
.remove = hpre_remove,
.sriov_configure = hpre_sriov_configure,
.sriov_configure = hisi_qm_sriov_configure,
.err_handler = &hpre_err_handler,
};

2451
drivers/crypto/hisilicon/qm.c
View File

File diff suppressed because it is too large Load Diff

120
drivers/crypto/hisilicon/qm.h
View File

@@ -8,6 +8,10 @@
#include <linux/module.h>
#include <linux/pci.h>
#define QM_QNUM_V1 4096
#define QM_QNUM_V2 1024
#define QM_MAX_VFS_NUM_V2 63
/* qm user domain */
#define QM_ARUSER_M_CFG_1 0x100088
#define AXUSER_SNOOP_ENABLE BIT(30)
@@ -70,7 +74,7 @@
#define QM_BASE_NFE (QM_AXI_RRESP | QM_AXI_BRESP | QM_ECC_MBIT | \
QM_ACC_GET_TASK_TIMEOUT | QM_DB_TIMEOUT | \
QM_OF_FIFO_OF)
QM_OF_FIFO_OF | QM_DB_RANDOM_INVALID)
#define QM_BASE_CE QM_ECC_1BIT
#define QM_Q_DEPTH 1024
@@ -80,14 +84,31 @@
/* page number for queue file region */
#define QM_DOORBELL_PAGE_NR 1
enum qm_stop_reason {
QM_NORMAL,
QM_SOFT_RESET,
QM_FLR,
};
enum qm_state {
QM_INIT = 0,
QM_START,
QM_CLOSE,
QM_STOP,
};
enum qp_state {
QP_INIT = 1,
QP_START,
QP_STOP,
QP_CLOSE,
};
enum qm_hw_ver {
QM_HW_UNKNOWN = -1,
QM_HW_V1 = 0x20,
QM_HW_V2 = 0x21,
QM_HW_V3 = 0x30,
};
enum qm_fun_type {
@@ -101,6 +122,14 @@ enum qm_debug_file {
DEBUG_FILE_NUM,
};
struct qm_dfx {
atomic64_t err_irq_cnt;
atomic64_t aeq_irq_cnt;
atomic64_t abnormal_irq_cnt;
atomic64_t create_qp_err_cnt;
atomic64_t mb_err_cnt;
};
struct debugfs_file {
enum qm_debug_file index;
struct mutex lock;
@@ -109,6 +138,9 @@ struct debugfs_file {
struct qm_debug {
u32 curr_qm_qp_num;
u32 sqe_mask_offset;
u32 sqe_mask_len;
struct qm_dfx dfx;
struct dentry *debug_root;
struct dentry *qm_d;
struct debugfs_file files[DEBUG_FILE_NUM];
@@ -125,22 +157,34 @@ struct hisi_qm_status {
bool eqc_phase;
u32 aeq_head;
bool aeqc_phase;
unsigned long flags;
atomic_t flags;
int stop_reason;
};
struct hisi_qm;
struct hisi_qm_err_info {
char *acpi_rst;
u32 msi_wr_port;
u32 ecc_2bits_mask;
u32 ce;
u32 nfe;
u32 fe;
u32 msi;
};
struct hisi_qm_err_status {
u32 is_qm_ecc_mbit;
u32 is_dev_ecc_mbit;
};
struct hisi_qm_err_ini {
int (*hw_init)(struct hisi_qm *qm);
void (*hw_err_enable)(struct hisi_qm *qm);
void (*hw_err_disable)(struct hisi_qm *qm);
u32 (*get_dev_hw_err_status)(struct hisi_qm *qm);
void (*clear_dev_hw_err_status)(struct hisi_qm *qm, u32 err_sts);
void (*open_axi_master_ooo)(struct hisi_qm *qm);
void (*close_axi_master_ooo)(struct hisi_qm *qm);
void (*log_dev_hw_err)(struct hisi_qm *qm, u32 err_sts);
struct hisi_qm_err_info err_info;
};
@@ -161,7 +205,9 @@ struct hisi_qm {
u32 qp_num;
u32 qp_in_used;
u32 ctrl_qp_num;
u32 vfs_num;
struct list_head list;
struct hisi_qm_list *qm_list;
struct qm_dma qdma;
struct qm_sqc *sqc;
@@ -175,10 +221,12 @@ struct hisi_qm {
struct hisi_qm_status status;
const struct hisi_qm_err_ini *err_ini;
struct hisi_qm_err_status err_status;
unsigned long reset_flag;
rwlock_t qps_lock;
unsigned long *qp_bitmap;
struct hisi_qp **qp_array;
struct rw_semaphore qps_lock;
struct idr qp_idr;
struct hisi_qp *qp_array;
struct mutex mailbox_lock;
@@ -187,13 +235,12 @@ struct hisi_qm {
struct qm_debug debug;
u32 error_mask;
u32 msi_mask;
struct workqueue_struct *wq;
struct work_struct work;
struct work_struct rst_work;
const char *algs;
bool use_dma_api;
bool use_sva;
resource_size_t phys_base;
resource_size_t phys_size;
@@ -205,7 +252,7 @@ struct hisi_qp_status {
u16 sq_tail;
u16 cq_head;
bool cqc_phase;
unsigned long flags;
atomic_t flags;
};
struct hisi_qp_ops {
@@ -230,10 +277,58 @@ struct hisi_qp {
void (*event_cb)(struct hisi_qp *qp);
struct hisi_qm *qm;
bool is_resetting;
u16 pasid;
struct uacce_queue *uacce_q;
};
static inline int q_num_set(const char *val, const struct kernel_param *kp,
unsigned int device)
{
struct pci_dev *pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI,
device, NULL);
u32 n, q_num;
int ret;
if (!val)
return -EINVAL;
if (!pdev) {
q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2);
pr_info("No device found currently, suppose queue number is %d\n",
q_num);
} else {
if (pdev->revision == QM_HW_V1)
q_num = QM_QNUM_V1;
else
q_num = QM_QNUM_V2;
}
ret = kstrtou32(val, 10, &n);
if (ret || !n || n > q_num)
return -EINVAL;
return param_set_int(val, kp);
}
static inline int vfs_num_set(const char *val, const struct kernel_param *kp)
{
u32 n;
int ret;
if (!val)
return -EINVAL;
ret = kstrtou32(val, 10, &n);
if (ret < 0)
return ret;
if (n > QM_MAX_VFS_NUM_V2)
return -EINVAL;
return param_set_int(val, kp);
}
static inline void hisi_qm_init_list(struct hisi_qm_list *qm_list)
{
INIT_LIST_HEAD(&qm_list->list);
@@ -267,14 +362,19 @@ void hisi_qm_release_qp(struct hisi_qp *qp);
int hisi_qp_send(struct hisi_qp *qp, const void *msg);
int hisi_qm_get_free_qp_num(struct hisi_qm *qm);
int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number);
int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base, u32 number);
int hisi_qm_debug_init(struct hisi_qm *qm);
enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev);
void hisi_qm_debug_regs_clear(struct hisi_qm *qm);
int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs);
int hisi_qm_sriov_disable(struct pci_dev *pdev);
int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs);
void hisi_qm_dev_err_init(struct hisi_qm *qm);
void hisi_qm_dev_err_uninit(struct hisi_qm *qm);
pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,
pci_channel_state_t state);
pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev);
void hisi_qm_reset_prepare(struct pci_dev *pdev);
void hisi_qm_reset_done(struct pci_dev *pdev);
struct hisi_acc_sgl_pool;
struct hisi_acc_hw_sgl *hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,

5
drivers/crypto/hisilicon/sec2/sec.h
View File

@@ -160,6 +160,10 @@ struct sec_debug_file {
struct sec_dfx {
atomic64_t send_cnt;
atomic64_t recv_cnt;
atomic64_t send_busy_cnt;
atomic64_t err_bd_cnt;
atomic64_t invalid_req_cnt;
atomic64_t done_flag_cnt;
};
struct sec_debug {
@@ -172,7 +176,6 @@ struct sec_dev {
struct sec_debug debug;
u32 ctx_q_num;
bool iommu_used;
u32 num_vfs;
unsigned long status;
};

20
drivers/crypto/hisilicon/sec2/sec_crypto.c
View File

@@ -148,6 +148,7 @@ static int sec_aead_verify(struct sec_req *req)
static void sec_req_cb(struct hisi_qp *qp, void *resp)
{
struct sec_qp_ctx *qp_ctx = qp->qp_ctx;
struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx;
struct sec_sqe *bd = resp;
struct sec_ctx *ctx;
struct sec_req *req;
@@ -157,11 +158,16 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp)
type = bd->type_cipher_auth & SEC_TYPE_MASK;
if (unlikely(type != SEC_BD_TYPE2)) {
atomic64_inc(&dfx->err_bd_cnt);
pr_err("err bd type [%d]\n", type);
return;
}
req = qp_ctx->req_list[le16_to_cpu(bd->type2.tag)];
if (unlikely(!req)) {
atomic64_inc(&dfx->invalid_req_cnt);
return;
}
req->err_type = bd->type2.error_type;
ctx = req->ctx;
done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK;
@@ -174,12 +180,13 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp)
"err_type[%d],done[%d],flag[%d]\n",
req->err_type, done, flag);
err = -EIO;
atomic64_inc(&dfx->done_flag_cnt);
}
if (ctx->alg_type == SEC_AEAD && !req->c_req.encrypt)
err = sec_aead_verify(req);
atomic64_inc(&ctx->sec->debug.dfx.recv_cnt);
atomic64_inc(&dfx->recv_cnt);
ctx->req_op->buf_unmap(ctx, req);
@@ -200,10 +207,12 @@ static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
return -ENOBUFS;
if (!ret) {
if (req->fake_busy)
if (req->fake_busy) {
atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt);
ret = -EBUSY;
else
} else {
ret = -EINPROGRESS;
}
}
return ret;
@@ -832,7 +841,6 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx,
struct crypto_authenc_keys *keys)
{
struct crypto_shash *hash_tfm = ctx->hash_tfm;
SHASH_DESC_ON_STACK(shash, hash_tfm);
int blocksize, ret;
if (!keys->authkeylen) {
@@ -842,8 +850,8 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx,
blocksize = crypto_shash_blocksize(hash_tfm);
if (keys->authkeylen > blocksize) {
ret = crypto_shash_digest(shash, keys->authkey,
keys->authkeylen, ctx->a_key);
ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey,
keys->authkeylen, ctx->a_key);
if (ret) {
pr_err("hisi_sec2: aead auth digest error!\n");
return -EINVAL;

383
drivers/crypto/hisilicon/sec2/sec_main.c
View File

@@ -80,6 +80,9 @@
#define SEC_VF_CNT_MASK 0xffffffc0
#define SEC_DBGFS_VAL_MAX_LEN 20
#define SEC_SQE_MASK_OFFSET 64
#define SEC_SQE_MASK_LEN 48
#define SEC_ADDR(qm, offset) ((qm)->io_base + (offset) + \
SEC_ENGINE_PF_CFG_OFF + SEC_ACC_COMMON_REG_OFF)
@@ -88,6 +91,11 @@ struct sec_hw_error {
const char *msg;
};
struct sec_dfx_item {
const char *name;
u32 offset;
};
static const char sec_name[] = "hisi_sec2";
static struct dentry *sec_debugfs_root;
static struct hisi_qm_list sec_devices;
@@ -110,7 +118,16 @@ static const char * const sec_dbg_file_name[] = {
[SEC_CLEAR_ENABLE] = "clear_enable",
};
static struct debugfs_reg32 sec_dfx_regs[] = {
static struct sec_dfx_item sec_dfx_labels[] = {
{"send_cnt", offsetof(struct sec_dfx, send_cnt)},
{"recv_cnt", offsetof(struct sec_dfx, recv_cnt)},
{"send_busy_cnt", offsetof(struct sec_dfx, send_busy_cnt)},
{"err_bd_cnt", offsetof(struct sec_dfx, err_bd_cnt)},
{"invalid_req_cnt", offsetof(struct sec_dfx, invalid_req_cnt)},
{"done_flag_cnt", offsetof(struct sec_dfx, done_flag_cnt)},
};
static const struct debugfs_reg32 sec_dfx_regs[] = {
{"SEC_PF_ABNORMAL_INT_SOURCE ", 0x301010},
{"SEC_SAA_EN ", 0x301270},
{"SEC_BD_LATENCY_MIN ", 0x301600},
@@ -136,45 +153,14 @@ static struct debugfs_reg32 sec_dfx_regs[] = {
static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp)
{
struct pci_dev *pdev;
u32 n, q_num;
u8 rev_id;
int ret;
if (!val)
return -EINVAL;
pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI,
SEC_PF_PCI_DEVICE_ID, NULL);
if (!pdev) {
q_num = min_t(u32, SEC_QUEUE_NUM_V1, SEC_QUEUE_NUM_V2);
pr_info("No device, suppose queue number is %d!\n", q_num);
} else {
rev_id = pdev->revision;
switch (rev_id) {
case QM_HW_V1:
q_num = SEC_QUEUE_NUM_V1;
break;
case QM_HW_V2:
q_num = SEC_QUEUE_NUM_V2;
break;
default:
return -EINVAL;
}
}
ret = kstrtou32(val, 10, &n);
if (ret || !n || n > q_num)
return -EINVAL;
return param_set_int(val, kp);
return q_num_set(val, kp, SEC_PF_PCI_DEVICE_ID);
}
static const struct kernel_param_ops sec_pf_q_num_ops = {
.set = sec_pf_q_num_set,
.get = param_get_int,
};
static u32 pf_q_num = SEC_PF_DEF_Q_NUM;
module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444);
MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 0-4096, v2 0-1024)");
@@ -207,6 +193,15 @@ static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF;
module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444);
MODULE_PARM_DESC(ctx_q_num, "Queue num in ctx (24 default, 2, 4, ..., 32)");
static const struct kernel_param_ops vfs_num_ops = {
.set = vfs_num_set,
.get = param_get_int,
};
static u32 vfs_num;
module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
void sec_destroy_qps(struct hisi_qp **qps, int qp_num)
{
hisi_qm_free_qps(qps, qp_num);
@@ -240,9 +235,8 @@ static const struct pci_device_id sec_dev_ids[] = {
};
MODULE_DEVICE_TABLE(pci, sec_dev_ids);
static u8 sec_get_endian(struct sec_dev *sec)
static u8 sec_get_endian(struct hisi_qm *qm)
{
struct hisi_qm *qm = &sec->qm;
u32 reg;
/*
@@ -270,9 +264,8 @@ static u8 sec_get_endian(struct sec_dev *sec)
return SEC_64BE;
}
static int sec_engine_init(struct sec_dev *sec)
static int sec_engine_init(struct hisi_qm *qm)
{
struct hisi_qm *qm = &sec->qm;
int ret;
u32 reg;
@@ -315,7 +308,7 @@ static int sec_engine_init(struct sec_dev *sec)
/* config endian */
reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
reg |= sec_get_endian(sec);
reg |= sec_get_endian(qm);
writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
/* Enable sm4 xts mode multiple iv */
@@ -325,10 +318,8 @@ static int sec_engine_init(struct sec_dev *sec)
return 0;
}
static int sec_set_user_domain_and_cache(struct sec_dev *sec)
static int sec_set_user_domain_and_cache(struct hisi_qm *qm)
{
struct hisi_qm *qm = &sec->qm;
/* qm user domain */
writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1);
writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE);
@@ -349,7 +340,7 @@ static int sec_set_user_domain_and_cache(struct sec_dev *sec)
CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |
FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL);
return sec_engine_init(sec);
return sec_engine_init(qm);
}
/* sec_debug_regs_clear() - clear the sec debug regs */
@@ -424,23 +415,22 @@ static u32 sec_current_qm_read(struct sec_debug_file *file)
static int sec_current_qm_write(struct sec_debug_file *file, u32 val)
{
struct hisi_qm *qm = file->qm;
struct sec_dev *sec = container_of(qm, struct sec_dev, qm);
u32 vfq_num;
u32 tmp;
if (val > sec->num_vfs)
if (val > qm->vfs_num)
return -EINVAL;
/* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
if (!val) {
qm->debug.curr_qm_qp_num = qm->qp_num;
} else {
vfq_num = (qm->ctrl_qp_num - qm->qp_num) / sec->num_vfs;
vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num;
if (val == sec->num_vfs)
if (val == qm->vfs_num)
qm->debug.curr_qm_qp_num =
qm->ctrl_qp_num - qm->qp_num -
(sec->num_vfs - 1) * vfq_num;
(qm->vfs_num - 1) * vfq_num;
else
qm->debug.curr_qm_qp_num = vfq_num;
}
@@ -570,10 +560,22 @@ static const struct file_operations sec_dbg_fops = {
static int sec_debugfs_atomic64_get(void *data, u64 *val)
{
*val = atomic64_read((atomic64_t *)data);
return 0;
}
static int sec_debugfs_atomic64_set(void *data, u64 val)
{
if (val)
return -EINVAL;
atomic64_set((atomic64_t *)data, 0);
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(sec_atomic64_ops, sec_debugfs_atomic64_get,
NULL, "%lld\n");
sec_debugfs_atomic64_set, "%lld\n");
static int sec_core_debug_init(struct sec_dev *sec)
{
@@ -582,6 +584,7 @@ static int sec_core_debug_init(struct sec_dev *sec)
struct sec_dfx *dfx = &sec->debug.dfx;
struct debugfs_regset32 *regset;
struct dentry *tmp_d;
int i;
tmp_d = debugfs_create_dir("sec_dfx", sec->qm.debug.debug_root);
@@ -593,13 +596,15 @@ static int sec_core_debug_init(struct sec_dev *sec)
regset->nregs = ARRAY_SIZE(sec_dfx_regs);
regset->base = qm->io_base;
debugfs_create_regset32("regs", 0444, tmp_d, regset);
if (qm->pdev->device == SEC_PF_PCI_DEVICE_ID)
debugfs_create_regset32("regs", 0444, tmp_d, regset);
debugfs_create_file("send_cnt", 0444, tmp_d,
&dfx->send_cnt, &sec_atomic64_ops);
debugfs_create_file("recv_cnt", 0444, tmp_d,
&dfx->recv_cnt, &sec_atomic64_ops);
for (i = 0; i < ARRAY_SIZE(sec_dfx_labels); i++) {
atomic64_t *data = (atomic64_t *)((uintptr_t)dfx +
sec_dfx_labels[i].offset);
debugfs_create_file(sec_dfx_labels[i].name, 0644,
tmp_d, data, &sec_atomic64_ops);
}
return 0;
}
@@ -630,6 +635,9 @@ static int sec_debugfs_init(struct sec_dev *sec)
qm->debug.debug_root = debugfs_create_dir(dev_name(dev),
sec_debugfs_root);
qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET;
qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN;
ret = hisi_qm_debug_init(qm);
if (ret)
goto failed_to_create;
@@ -675,8 +683,6 @@ static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts)
}
errs++;
}
writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE);
}
static u32 sec_get_hw_err_status(struct hisi_qm *qm)
@@ -684,17 +690,36 @@ static u32 sec_get_hw_err_status(struct hisi_qm *qm)
return readl(qm->io_base + SEC_CORE_INT_STATUS);
}
static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
{
writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE);
}
static void sec_open_axi_master_ooo(struct hisi_qm *qm)
{
u32 val;
val = readl(SEC_ADDR(qm, SEC_CONTROL_REG));
writel(val & SEC_AXI_SHUTDOWN_DISABLE, SEC_ADDR(qm, SEC_CONTROL_REG));
writel(val | SEC_AXI_SHUTDOWN_ENABLE, SEC_ADDR(qm, SEC_CONTROL_REG));
}
static const struct hisi_qm_err_ini sec_err_ini = {
.hw_init = sec_set_user_domain_and_cache,
.hw_err_enable = sec_hw_error_enable,
.hw_err_disable = sec_hw_error_disable,
.get_dev_hw_err_status = sec_get_hw_err_status,
.clear_dev_hw_err_status = sec_clear_hw_err_status,
.log_dev_hw_err = sec_log_hw_error,
.open_axi_master_ooo = sec_open_axi_master_ooo,
.err_info = {
.ce = QM_BASE_CE,
.nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT |
QM_ACC_WB_NOT_READY_TIMEOUT,
.fe = 0,
.msi = QM_DB_RANDOM_INVALID,
.ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC,
.msi_wr_port = BIT(0),
.acpi_rst = "SRST",
}
};
@@ -703,22 +728,14 @@ static int sec_pf_probe_init(struct sec_dev *sec)
struct hisi_qm *qm = &sec->qm;
int ret;
switch (qm->ver) {
case QM_HW_V1:
if (qm->ver == QM_HW_V1)
qm->ctrl_qp_num = SEC_QUEUE_NUM_V1;
break;
case QM_HW_V2:
else
qm->ctrl_qp_num = SEC_QUEUE_NUM_V2;
break;
default:
return -EINVAL;
}
qm->err_ini = &sec_err_ini;
ret = sec_set_user_domain_and_cache(sec);
ret = sec_set_user_domain_and_cache(qm);
if (ret)
return ret;
@@ -730,32 +747,30 @@ static int sec_pf_probe_init(struct sec_dev *sec)
static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
{
enum qm_hw_ver rev_id;
rev_id = hisi_qm_get_hw_version(pdev);
if (rev_id == QM_HW_UNKNOWN)
return -ENODEV;
int ret;
qm->pdev = pdev;
qm->ver = rev_id;
qm->ver = pdev->revision;
qm->sqe_size = SEC_SQE_SIZE;
qm->dev_name = sec_name;
qm->fun_type = (pdev->device == SEC_PF_PCI_DEVICE_ID) ?
QM_HW_PF : QM_HW_VF;
qm->use_dma_api = true;
return hisi_qm_init(qm);
}
static void sec_qm_uninit(struct hisi_qm *qm)
{
hisi_qm_uninit(qm);
}
static int sec_probe_init(struct hisi_qm *qm, struct sec_dev *sec)
{
int ret;
if (qm->fun_type == QM_HW_PF) {
qm->qp_base = SEC_PF_DEF_Q_BASE;
qm->qp_num = pf_q_num;
qm->debug.curr_qm_qp_num = pf_q_num;
qm->qm_list = &sec_devices;
} else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {
/*
* have no way to get qm configure in VM in v1 hardware,
* so currently force PF to uses SEC_PF_DEF_Q_NUM, and force
* to trigger only one VF in v1 hardware.
* v2 hardware has no such problem.
*/
qm->qp_base = SEC_PF_DEF_Q_NUM;
qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM;
}
/*
* WQ_HIGHPRI: SEC request must be low delayed,
@@ -763,47 +778,38 @@ static int sec_probe_init(struct hisi_qm *qm, struct sec_dev *sec)
* WQ_UNBOUND: SEC task is likely with long
* running CPU intensive workloads.
*/
qm->wq = alloc_workqueue("%s", WQ_HIGHPRI |
WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus(),
pci_name(qm->pdev));
qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM |
WQ_UNBOUND, num_online_cpus(),
pci_name(qm->pdev));
if (!qm->wq) {
pci_err(qm->pdev, "fail to alloc workqueue\n");
return -ENOMEM;
}
if (qm->fun_type == QM_HW_PF) {
qm->qp_base = SEC_PF_DEF_Q_BASE;
qm->qp_num = pf_q_num;
qm->debug.curr_qm_qp_num = pf_q_num;
ret = hisi_qm_init(qm);
if (ret)
destroy_workqueue(qm->wq);
return ret;
}
static void sec_qm_uninit(struct hisi_qm *qm)
{
hisi_qm_uninit(qm);
}
static int sec_probe_init(struct sec_dev *sec)
{
struct hisi_qm *qm = &sec->qm;
int ret;
if (qm->fun_type == QM_HW_PF) {
ret = sec_pf_probe_init(sec);
if (ret)
goto err_probe_uninit;
} else if (qm->fun_type == QM_HW_VF) {
/*
* have no way to get qm configure in VM in v1 hardware,
* so currently force PF to uses SEC_PF_DEF_Q_NUM, and force
* to trigger only one VF in v1 hardware.
* v2 hardware has no such problem.
*/
if (qm->ver == QM_HW_V1) {
qm->qp_base = SEC_PF_DEF_Q_NUM;
qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM;
} else if (qm->ver == QM_HW_V2) {
/* v2 starts to support get vft by mailbox */
ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
if (ret)
goto err_probe_uninit;
}
} else {
ret = -ENODEV;
goto err_probe_uninit;
return ret;
}
return 0;
err_probe_uninit:
destroy_workqueue(qm->wq);
return ret;
}
static void sec_probe_uninit(struct hisi_qm *qm)
@@ -840,20 +846,17 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (!sec)
return -ENOMEM;
pci_set_drvdata(pdev, sec);
qm = &sec->qm;
ret = sec_qm_init(qm, pdev);
if (ret) {
pci_err(pdev, "Failed to init SEC QM (%d)!\n", ret);
return ret;
}
sec->ctx_q_num = ctx_q_num;
sec_iommu_used_check(sec);
qm = &sec->qm;
ret = sec_qm_init(qm, pdev);
if (ret) {
pci_err(pdev, "Failed to pre init qm!\n");
return ret;
}
ret = sec_probe_init(qm, sec);
ret = sec_probe_init(sec);
if (ret) {
pci_err(pdev, "Failed to probe!\n");
goto err_qm_uninit;
@@ -877,8 +880,17 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id)
goto err_remove_from_list;
}
if (qm->fun_type == QM_HW_PF && vfs_num) {
ret = hisi_qm_sriov_enable(pdev, vfs_num);
if (ret < 0)
goto err_crypto_unregister;
}
return 0;
err_crypto_unregister:
sec_unregister_from_crypto();
err_remove_from_list:
hisi_qm_del_from_list(qm, &sec_devices);
sec_debugfs_exit(sec);
@@ -893,110 +905,6 @@ err_qm_uninit:
return ret;
}
/* now we only support equal assignment */
static int sec_vf_q_assign(struct sec_dev *sec, u32 num_vfs)
{
struct hisi_qm *qm = &sec->qm;
u32 qp_num = qm->qp_num;
u32 q_base = qp_num;
u32 q_num, remain_q_num;
int i, j, ret;
if (!num_vfs)
return -EINVAL;
remain_q_num = qm->ctrl_qp_num - qp_num;
q_num = remain_q_num / num_vfs;
for (i = 1; i <= num_vfs; i++) {
if (i == num_vfs)
q_num += remain_q_num % num_vfs;
ret = hisi_qm_set_vft(qm, i, q_base, q_num);
if (ret) {
for (j = i; j > 0; j--)
hisi_qm_set_vft(qm, j, 0, 0);
return ret;
}
q_base += q_num;
}
return 0;
}
static int sec_clear_vft_config(struct sec_dev *sec)
{
struct hisi_qm *qm = &sec->qm;
u32 num_vfs = sec->num_vfs;
int ret;
u32 i;
for (i = 1; i <= num_vfs; i++) {
ret = hisi_qm_set_vft(qm, i, 0, 0);
if (ret)
return ret;
}
sec->num_vfs = 0;
return 0;
}
static int sec_sriov_enable(struct pci_dev *pdev, int max_vfs)
{
struct sec_dev *sec = pci_get_drvdata(pdev);
int pre_existing_vfs, ret;
u32 num_vfs;
pre_existing_vfs = pci_num_vf(pdev);
if (pre_existing_vfs) {
pci_err(pdev, "Can't enable VF. Please disable at first!\n");
return 0;
}
num_vfs = min_t(u32, max_vfs, SEC_VF_NUM);
ret = sec_vf_q_assign(sec, num_vfs);
if (ret) {
pci_err(pdev, "Can't assign queues for VF!\n");
return ret;
}
sec->num_vfs = num_vfs;
ret = pci_enable_sriov(pdev, num_vfs);
if (ret) {
pci_err(pdev, "Can't enable VF!\n");
sec_clear_vft_config(sec);
return ret;
}
return num_vfs;
}
static int sec_sriov_disable(struct pci_dev *pdev)
{
struct sec_dev *sec = pci_get_drvdata(pdev);
if (pci_vfs_assigned(pdev)) {
pci_err(pdev, "Can't disable VFs while VFs are assigned!\n");
return -EPERM;
}
/* remove in sec_pci_driver will be called to free VF resources */
pci_disable_sriov(pdev);
return sec_clear_vft_config(sec);
}
static int sec_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
if (num_vfs)
return sec_sriov_enable(pdev, num_vfs);
else
return sec_sriov_disable(pdev);
}
static void sec_remove(struct pci_dev *pdev)
{
struct sec_dev *sec = pci_get_drvdata(pdev);
@@ -1006,8 +914,8 @@ static void sec_remove(struct pci_dev *pdev)
hisi_qm_del_from_list(qm, &sec_devices);
if (qm->fun_type == QM_HW_PF && sec->num_vfs)
(void)sec_sriov_disable(pdev);
if (qm->fun_type == QM_HW_PF && qm->vfs_num)
hisi_qm_sriov_disable(pdev);
sec_debugfs_exit(sec);
@@ -1023,6 +931,9 @@ static void sec_remove(struct pci_dev *pdev)
static const struct pci_error_handlers sec_err_handler = {
.error_detected = hisi_qm_dev_err_detected,
.slot_reset = hisi_qm_dev_slot_reset,
.reset_prepare = hisi_qm_reset_prepare,
.reset_done = hisi_qm_reset_done,
};
static struct pci_driver sec_pci_driver = {
@@ -1031,7 +942,7 @@ static struct pci_driver sec_pci_driver = {
.probe = sec_probe,
.remove = sec_remove,
.err_handler = &sec_err_handler,
.sriov_configure = sec_sriov_configure,
.sriov_configure = hisi_qm_sriov_configure,
};
static void sec_register_debugfs(void)

8
drivers/crypto/hisilicon/zip/zip.h
View File

@@ -28,12 +28,20 @@ enum hisi_zip_error_type {
HZIP_NC_ERR = 0x0d,
};
struct hisi_zip_dfx {
atomic64_t send_cnt;
atomic64_t recv_cnt;
atomic64_t send_busy_cnt;
atomic64_t err_bd_cnt;
};
struct hisi_zip_ctrl;
struct hisi_zip {
struct hisi_qm qm;
struct list_head list;
struct hisi_zip_ctrl *ctrl;
struct hisi_zip_dfx dfx;
};
struct hisi_zip_sqe {

20
drivers/crypto/hisilicon/zip/zip_crypto.c
View File

@@ -64,7 +64,6 @@ struct hisi_zip_req_q {
struct hisi_zip_qp_ctx {
struct hisi_qp *qp;
struct hisi_zip_sqe zip_sqe;
struct hisi_zip_req_q req_q;
struct hisi_acc_sgl_pool *sgl_pool;
struct hisi_zip *zip_dev;
@@ -333,6 +332,7 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
{
struct hisi_zip_sqe *sqe = data;
struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx;
struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
struct hisi_zip_req *req = req_q->q + sqe->tag;
struct acomp_req *acomp_req = req->req;
@@ -340,12 +340,14 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
u32 status, dlen, head_size;
int err = 0;
atomic64_inc(&dfx->recv_cnt);
status = sqe->dw3 & HZIP_BD_STATUS_M;
if (status != 0 && status != HZIP_NC_ERR) {
dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n",
(qp->alg_type == 0) ? "" : "de", qp->qp_id, status,
sqe->produced);
atomic64_inc(&dfx->err_bd_cnt);
err = -EIO;
}
dlen = sqe->produced;
@@ -484,11 +486,12 @@ static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req,
static int hisi_zip_do_work(struct hisi_zip_req *req,
struct hisi_zip_qp_ctx *qp_ctx)
{
struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe;
struct acomp_req *a_req = req->req;
struct hisi_qp *qp = qp_ctx->qp;
struct device *dev = &qp->qm->pdev->dev;
struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool;
struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
struct hisi_zip_sqe zip_sqe;
dma_addr_t input;
dma_addr_t output;
int ret;
@@ -511,15 +514,18 @@ static int hisi_zip_do_work(struct hisi_zip_req *req,
}
req->dma_dst = output;
hisi_zip_fill_sqe(zip_sqe, qp->req_type, input, output, a_req->slen,
hisi_zip_fill_sqe(&zip_sqe, qp->req_type, input, output, a_req->slen,
a_req->dlen, req->sskip, req->dskip);
hisi_zip_config_buf_type(zip_sqe, HZIP_SGL);
hisi_zip_config_tag(zip_sqe, req->req_id);
hisi_zip_config_buf_type(&zip_sqe, HZIP_SGL);
hisi_zip_config_tag(&zip_sqe, req->req_id);
/* send command to start a task */
ret = hisi_qp_send(qp, zip_sqe);
if (ret < 0)
atomic64_inc(&dfx->send_cnt);
ret = hisi_qp_send(qp, &zip_sqe);
if (ret < 0) {
atomic64_inc(&dfx->send_busy_cnt);
goto err_unmap_output;
}
return -EINPROGRESS;

396
drivers/crypto/hisilicon/zip/zip_main.c
View File

@@ -62,6 +62,7 @@
#define HZIP_CORE_INT_SOURCE 0x3010A0
#define HZIP_CORE_INT_MASK_REG 0x3010A4
#define HZIP_CORE_INT_SET 0x3010A8
#define HZIP_CORE_INT_STATUS 0x3010AC
#define HZIP_CORE_INT_STATUS_M_ECC BIT(1)
#define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148
@@ -83,8 +84,13 @@
#define HZIP_SOFT_CTRL_CNT_CLR_CE 0x301000
#define SOFT_CTRL_CNT_CLR_CE_BIT BIT(0)
#define HZIP_SOFT_CTRL_ZIP_CONTROL 0x30100C
#define HZIP_AXI_SHUTDOWN_ENABLE BIT(14)
#define HZIP_WR_PORT BIT(11)
#define HZIP_BUF_SIZE 22
#define HZIP_SQE_MASK_OFFSET 64
#define HZIP_SQE_MASK_LEN 48
static const char hisi_zip_name[] = "hisi_zip";
static struct dentry *hzip_debugfs_root;
@@ -95,6 +101,18 @@ struct hisi_zip_hw_error {
const char *msg;
};
struct zip_dfx_item {
const char *name;
u32 offset;
};
static struct zip_dfx_item zip_dfx_files[] = {
{"send_cnt", offsetof(struct hisi_zip_dfx, send_cnt)},
{"recv_cnt", offsetof(struct hisi_zip_dfx, recv_cnt)},
{"send_busy_cnt", offsetof(struct hisi_zip_dfx, send_busy_cnt)},
{"err_bd_cnt", offsetof(struct hisi_zip_dfx, err_bd_cnt)},
};
static const struct hisi_zip_hw_error zip_hw_error[] = {
{ .int_msk = BIT(0), .msg = "zip_ecc_1bitt_err" },
{ .int_msk = BIT(1), .msg = "zip_ecc_2bit_err" },
@@ -134,7 +152,6 @@ struct ctrl_debug_file {
* Just relevant for PF.
*/
struct hisi_zip_ctrl {
u32 num_vfs;
struct hisi_zip *hisi_zip;
struct dentry *debug_root;
struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM];
@@ -162,7 +179,7 @@ static const u64 core_offsets[] = {
[HZIP_DECOMP_CORE5] = 0x309000,
};
static struct debugfs_reg32 hzip_dfx_regs[] = {
static const struct debugfs_reg32 hzip_dfx_regs[] = {
{"HZIP_GET_BD_NUM ", 0x00ull},
{"HZIP_GET_RIGHT_BD ", 0x04ull},
{"HZIP_GET_ERROR_BD ", 0x08ull},
@@ -189,38 +206,7 @@ static struct debugfs_reg32 hzip_dfx_regs[] = {
static int pf_q_num_set(const char *val, const struct kernel_param *kp)
{
struct pci_dev *pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI,
PCI_DEVICE_ID_ZIP_PF, NULL);
u32 n, q_num;
u8 rev_id;
int ret;
if (!val)
return -EINVAL;
if (!pdev) {
q_num = min_t(u32, HZIP_QUEUE_NUM_V1, HZIP_QUEUE_NUM_V2);
pr_info("No device found currently, suppose queue number is %d\n",
q_num);
} else {
rev_id = pdev->revision;
switch (rev_id) {
case QM_HW_V1:
q_num = HZIP_QUEUE_NUM_V1;
break;
case QM_HW_V2:
q_num = HZIP_QUEUE_NUM_V2;
break;
default:
return -EINVAL;
}
}
ret = kstrtou32(val, 10, &n);
if (ret != 0 || n > q_num || n == 0)
return -EINVAL;
return param_set_int(val, kp);
return q_num_set(val, kp, PCI_DEVICE_ID_ZIP_PF);
}
static const struct kernel_param_ops pf_q_num_ops = {
@@ -232,9 +218,14 @@ static u32 pf_q_num = HZIP_PF_DEF_Q_NUM;
module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444);
MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 1-4096, v2 1-1024)");
static const struct kernel_param_ops vfs_num_ops = {
.set = vfs_num_set,
.get = param_get_int,
};
static u32 vfs_num;
module_param(vfs_num, uint, 0444);
MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63)");
module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
static const struct pci_device_id hisi_zip_dev_ids[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_ZIP_PF) },
@@ -250,9 +241,9 @@ int zip_create_qps(struct hisi_qp **qps, int qp_num)
return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps);
}
static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip)
static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm)
{
void __iomem *base = hisi_zip->qm.io_base;
void __iomem *base = qm->io_base;
/* qm user domain */
writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1);
@@ -279,7 +270,7 @@ static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip)
writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63);
if (hisi_zip->qm.use_sva) {
if (qm->use_sva) {
writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63);
writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63);
} else {
@@ -295,10 +286,14 @@ static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip)
writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |
CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |
FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL);
return 0;
}
static void hisi_zip_hw_error_enable(struct hisi_qm *qm)
{
u32 val;
if (qm->ver == QM_HW_V1) {
writel(HZIP_CORE_INT_MASK_ALL,
qm->io_base + HZIP_CORE_INT_MASK_REG);
@@ -317,12 +312,24 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm)
/* enable ZIP hw error interrupts */
writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG);
/* enable ZIP block master OOO when m-bit error occur */
val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
val = val | HZIP_AXI_SHUTDOWN_ENABLE;
writel(val, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
}
static void hisi_zip_hw_error_disable(struct hisi_qm *qm)
{
u32 val;
/* disable ZIP hw error interrupts */
writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG);
/* disable ZIP block master OOO when m-bit error occur */
val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
val = val & ~HZIP_AXI_SHUTDOWN_ENABLE;
writel(val, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
}
static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file)
@@ -342,21 +349,20 @@ static u32 current_qm_read(struct ctrl_debug_file *file)
static int current_qm_write(struct ctrl_debug_file *file, u32 val)
{
struct hisi_qm *qm = file_to_qm(file);
struct hisi_zip_ctrl *ctrl = file->ctrl;
u32 vfq_num;
u32 tmp;
if (val > ctrl->num_vfs)
if (val > qm->vfs_num)
return -EINVAL;
/* Calculate curr_qm_qp_num and store */
if (val == 0) {
qm->debug.curr_qm_qp_num = qm->qp_num;
} else {
vfq_num = (qm->ctrl_qp_num - qm->qp_num) / ctrl->num_vfs;
if (val == ctrl->num_vfs)
vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num;
if (val == qm->vfs_num)
qm->debug.curr_qm_qp_num = qm->ctrl_qp_num -
qm->qp_num - (ctrl->num_vfs - 1) * vfq_num;
qm->qp_num - (qm->vfs_num - 1) * vfq_num;
else
qm->debug.curr_qm_qp_num = vfq_num;
}
@@ -477,6 +483,27 @@ static const struct file_operations ctrl_debug_fops = {
.write = ctrl_debug_write,
};
static int zip_debugfs_atomic64_set(void *data, u64 val)
{
if (val)
return -EINVAL;
atomic64_set((atomic64_t *)data, 0);
return 0;
}
static int zip_debugfs_atomic64_get(void *data, u64 *val)
{
*val = atomic64_read((atomic64_t *)data);
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(zip_atomic64_ops, zip_debugfs_atomic64_get,
zip_debugfs_atomic64_set, "%llu\n");
static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl)
{
struct hisi_zip *hisi_zip = ctrl->hisi_zip;
@@ -508,6 +535,25 @@ static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl)
return 0;
}
static void hisi_zip_dfx_debug_init(struct hisi_qm *qm)
{
struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);
struct hisi_zip_dfx *dfx = &zip->dfx;
struct dentry *tmp_dir;
void *data;
int i;
tmp_dir = debugfs_create_dir("zip_dfx", qm->debug.debug_root);
for (i = 0; i < ARRAY_SIZE(zip_dfx_files); i++) {
data = (atomic64_t *)((uintptr_t)dfx + zip_dfx_files[i].offset);
debugfs_create_file(zip_dfx_files[i].name,
0644,
tmp_dir,
data,
&zip_atomic64_ops);
}
}
static int hisi_zip_ctrl_debug_init(struct hisi_zip_ctrl *ctrl)
{
int i;
@@ -534,6 +580,8 @@ static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip)
dev_d = debugfs_create_dir(dev_name(dev), hzip_debugfs_root);
qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET;
qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN;
qm->debug.debug_root = dev_d;
ret = hisi_qm_debug_init(qm);
if (ret)
@@ -546,6 +594,8 @@ static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip)
goto failed_to_create;
}
hisi_zip_dfx_debug_init(qm);
return 0;
failed_to_create:
@@ -598,8 +648,6 @@ static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts)
}
err++;
}
writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE);
}
static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm)
@@ -607,17 +655,55 @@ static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm)
return readl(qm->io_base + HZIP_CORE_INT_STATUS);
}
static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
{
writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE);
}
static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm)
{
u32 val;
val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
writel(val & ~HZIP_AXI_SHUTDOWN_ENABLE,
qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
writel(val | HZIP_AXI_SHUTDOWN_ENABLE,
qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
}
static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm)
{
u32 nfe_enb;
/* Disable ECC Mbit error report. */
nfe_enb = readl(qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
writel(nfe_enb & ~HZIP_CORE_INT_STATUS_M_ECC,
qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
/* Inject zip ECC Mbit error to block master ooo. */
writel(HZIP_CORE_INT_STATUS_M_ECC,
qm->io_base + HZIP_CORE_INT_SET);
}
static const struct hisi_qm_err_ini hisi_zip_err_ini = {
.hw_init = hisi_zip_set_user_domain_and_cache,
.hw_err_enable = hisi_zip_hw_error_enable,
.hw_err_disable = hisi_zip_hw_error_disable,
.get_dev_hw_err_status = hisi_zip_get_hw_err_status,
.clear_dev_hw_err_status = hisi_zip_clear_hw_err_status,
.log_dev_hw_err = hisi_zip_log_hw_error,
.open_axi_master_ooo = hisi_zip_open_axi_master_ooo,
.close_axi_master_ooo = hisi_zip_close_axi_master_ooo,
.err_info = {
.ce = QM_BASE_CE,
.nfe = QM_BASE_NFE |
QM_ACC_WB_NOT_READY_TIMEOUT,
.fe = 0,
.msi = QM_DB_RANDOM_INVALID,
.ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC,
.msi_wr_port = HZIP_WR_PORT,
.acpi_rst = "ZRST",
}
};
@@ -633,164 +719,35 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip)
hisi_zip->ctrl = ctrl;
ctrl->hisi_zip = hisi_zip;
switch (qm->ver) {
case QM_HW_V1:
if (qm->ver == QM_HW_V1)
qm->ctrl_qp_num = HZIP_QUEUE_NUM_V1;
break;
case QM_HW_V2:
else
qm->ctrl_qp_num = HZIP_QUEUE_NUM_V2;
break;
default:
return -EINVAL;
}
qm->err_ini = &hisi_zip_err_ini;
hisi_zip_set_user_domain_and_cache(hisi_zip);
hisi_zip_set_user_domain_and_cache(qm);
hisi_qm_dev_err_init(qm);
hisi_zip_debug_regs_clear(hisi_zip);
return 0;
}
/* Currently we only support equal assignment */
static int hisi_zip_vf_q_assign(struct hisi_zip *hisi_zip, int num_vfs)
static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
{
struct hisi_qm *qm = &hisi_zip->qm;
u32 qp_num = qm->qp_num;
u32 q_base = qp_num;
u32 q_num, remain_q_num, i;
int ret;
if (!num_vfs)
return -EINVAL;
remain_q_num = qm->ctrl_qp_num - qp_num;
if (remain_q_num < num_vfs)
return -EINVAL;
q_num = remain_q_num / num_vfs;
for (i = 1; i <= num_vfs; i++) {
if (i == num_vfs)
q_num += remain_q_num % num_vfs;
ret = hisi_qm_set_vft(qm, i, q_base, q_num);
if (ret)
return ret;
q_base += q_num;
}
return 0;
}
static int hisi_zip_clear_vft_config(struct hisi_zip *hisi_zip)
{
struct hisi_zip_ctrl *ctrl = hisi_zip->ctrl;
struct hisi_qm *qm = &hisi_zip->qm;
u32 i, num_vfs = ctrl->num_vfs;
int ret;
for (i = 1; i <= num_vfs; i++) {
ret = hisi_qm_set_vft(qm, i, 0, 0);
if (ret)
return ret;
}
ctrl->num_vfs = 0;
return 0;
}
static int hisi_zip_sriov_enable(struct pci_dev *pdev, int max_vfs)
{
struct hisi_zip *hisi_zip = pci_get_drvdata(pdev);
int pre_existing_vfs, num_vfs, ret;
pre_existing_vfs = pci_num_vf(pdev);
if (pre_existing_vfs) {
dev_err(&pdev->dev,
"Can't enable VF. Please disable pre-enabled VFs!\n");
return 0;
}
num_vfs = min_t(int, max_vfs, HZIP_VF_NUM);
ret = hisi_zip_vf_q_assign(hisi_zip, num_vfs);
if (ret) {
dev_err(&pdev->dev, "Can't assign queues for VF!\n");
return ret;
}
hisi_zip->ctrl->num_vfs = num_vfs;
ret = pci_enable_sriov(pdev, num_vfs);
if (ret) {
dev_err(&pdev->dev, "Can't enable VF!\n");
hisi_zip_clear_vft_config(hisi_zip);
return ret;
}
return num_vfs;
}
static int hisi_zip_sriov_disable(struct pci_dev *pdev)
{
struct hisi_zip *hisi_zip = pci_get_drvdata(pdev);
if (pci_vfs_assigned(pdev)) {
dev_err(&pdev->dev,
"Can't disable VFs while VFs are assigned!\n");
return -EPERM;
}
/* remove in hisi_zip_pci_driver will be called to free VF resources */
pci_disable_sriov(pdev);
return hisi_zip_clear_vft_config(hisi_zip);
}
static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct hisi_zip *hisi_zip;
enum qm_hw_ver rev_id;
struct hisi_qm *qm;
int ret;
rev_id = hisi_qm_get_hw_version(pdev);
if (rev_id == QM_HW_UNKNOWN)
return -EINVAL;
hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL);
if (!hisi_zip)
return -ENOMEM;
pci_set_drvdata(pdev, hisi_zip);
qm = &hisi_zip->qm;
qm->use_dma_api = true;
qm->pdev = pdev;
qm->ver = rev_id;
qm->ver = pdev->revision;
qm->algs = "zlib\ngzip";
qm->sqe_size = HZIP_SQE_SIZE;
qm->dev_name = hisi_zip_name;
qm->fun_type = (pdev->device == PCI_DEVICE_ID_ZIP_PF) ? QM_HW_PF :
QM_HW_VF;
ret = hisi_qm_init(qm);
if (ret) {
dev_err(&pdev->dev, "Failed to init qm!\n");
return ret;
}
qm->fun_type = (pdev->device == PCI_DEVICE_ID_ZIP_PF) ?
QM_HW_PF : QM_HW_VF;
if (qm->fun_type == QM_HW_PF) {
ret = hisi_zip_pf_probe_init(hisi_zip);
if (ret)
return ret;
qm->qp_base = HZIP_PF_DEF_Q_BASE;
qm->qp_num = pf_q_num;
} else if (qm->fun_type == QM_HW_VF) {
qm->qm_list = &zip_devices;
} else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {
/*
* have no way to get qm configure in VM in v1 hardware,
* so currently force PF to uses HZIP_PF_DEF_Q_NUM, and force
@@ -798,12 +755,49 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id)
*
* v2 hardware has no such problem.
*/
if (qm->ver == QM_HW_V1) {
qm->qp_base = HZIP_PF_DEF_Q_NUM;
qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM;
} else if (qm->ver == QM_HW_V2)
/* v2 starts to support get vft by mailbox */
hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
qm->qp_base = HZIP_PF_DEF_Q_NUM;
qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM;
}
return hisi_qm_init(qm);
}
static int hisi_zip_probe_init(struct hisi_zip *hisi_zip)
{
struct hisi_qm *qm = &hisi_zip->qm;
int ret;
if (qm->fun_type == QM_HW_PF) {
ret = hisi_zip_pf_probe_init(hisi_zip);
if (ret)
return ret;
}
return 0;
}
static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct hisi_zip *hisi_zip;
struct hisi_qm *qm;
int ret;
hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL);
if (!hisi_zip)
return -ENOMEM;
qm = &hisi_zip->qm;
ret = hisi_zip_qm_init(qm, pdev);
if (ret) {
pci_err(pdev, "Failed to init ZIP QM (%d)!\n", ret);
return ret;
}
ret = hisi_zip_probe_init(hisi_zip);
if (ret) {
pci_err(pdev, "Failed to probe (%d)!\n", ret);
goto err_qm_uninit;
}
ret = hisi_qm_start(qm);
@@ -823,7 +817,7 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id)
}
if (qm->fun_type == QM_HW_PF && vfs_num > 0) {
ret = hisi_zip_sriov_enable(pdev, vfs_num);
ret = hisi_qm_sriov_enable(pdev, vfs_num);
if (ret < 0)
goto err_remove_from_list;
}
@@ -836,15 +830,8 @@ err_remove_from_list:
hisi_qm_stop(qm);
err_qm_uninit:
hisi_qm_uninit(qm);
return ret;
}
static int hisi_zip_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
if (num_vfs == 0)
return hisi_zip_sriov_disable(pdev);
else
return hisi_zip_sriov_enable(pdev, num_vfs);
return ret;
}
static void hisi_zip_remove(struct pci_dev *pdev)
@@ -852,8 +839,8 @@ static void hisi_zip_remove(struct pci_dev *pdev)
struct hisi_zip *hisi_zip = pci_get_drvdata(pdev);
struct hisi_qm *qm = &hisi_zip->qm;
if (qm->fun_type == QM_HW_PF && hisi_zip->ctrl->num_vfs != 0)
hisi_zip_sriov_disable(pdev);
if (qm->fun_type == QM_HW_PF && qm->vfs_num)
hisi_qm_sriov_disable(pdev);
hisi_zip_debugfs_exit(hisi_zip);
hisi_qm_stop(qm);
@@ -865,6 +852,9 @@ static void hisi_zip_remove(struct pci_dev *pdev)
static const struct pci_error_handlers hisi_zip_err_handler = {
.error_detected = hisi_qm_dev_err_detected,
.slot_reset = hisi_qm_dev_slot_reset,
.reset_prepare = hisi_qm_reset_prepare,
.reset_done = hisi_qm_reset_done,
};
static struct pci_driver hisi_zip_pci_driver = {
@@ -873,7 +863,7 @@ static struct pci_driver hisi_zip_pci_driver = {
.probe = hisi_zip_probe,
.remove = hisi_zip_remove,
.sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ?
hisi_zip_sriov_configure : NULL,
hisi_qm_sriov_configure : NULL,
.err_handler = &hisi_zip_err_handler,
};

4
drivers/crypto/marvell/octeontx/otx_cptpf_main.c
View File

@@ -79,13 +79,13 @@ static int otx_cpt_device_init(struct otx_cpt_device *cpt)
/* Check BIST status */
bist = (u64)otx_cpt_check_bist_status(cpt);
if (bist) {
dev_err(dev, "RAM BIST failed with code 0x%llx", bist);
dev_err(dev, "RAM BIST failed with code 0x%llx\n", bist);
return -ENODEV;
}
bist = otx_cpt_check_exe_bist_status(cpt);
if (bist) {
dev_err(dev, "Engine BIST failed with code 0x%llx", bist);
dev_err(dev, "Engine BIST failed with code 0x%llx\n", bist);
return -ENODEV;
}

12
drivers/crypto/marvell/octeontx/otx_cptpf_mbox.c
View File

@@ -63,11 +63,11 @@ static void dump_mbox_msg(struct otx_cpt_mbox *mbox_msg, int vf_id)
hex_dump_to_buffer(mbox_msg, sizeof(struct otx_cpt_mbox), 16, 8,
raw_data_str, OTX_CPT_MAX_MBOX_DATA_STR_SIZE, false);
if (vf_id >= 0)
pr_debug("MBOX opcode %s received from VF%d raw_data %s",
pr_debug("MBOX opcode %s received from VF%d raw_data %s\n",
get_mbox_opcode_str(mbox_msg->msg), vf_id,
raw_data_str);
else
pr_debug("MBOX opcode %s received from PF raw_data %s",
pr_debug("MBOX opcode %s received from PF raw_data %s\n",
get_mbox_opcode_str(mbox_msg->msg), raw_data_str);
}
@@ -140,20 +140,20 @@ static int otx_cpt_bind_vq_to_grp(struct otx_cpt_device *cpt, u8 q, u8 grp)
struct otx_cpt_ucode *ucode;
if (q >= cpt->max_vfs) {
dev_err(dev, "Requested queue %d is > than maximum avail %d",
dev_err(dev, "Requested queue %d is > than maximum avail %d\n",
q, cpt->max_vfs);
return -EINVAL;
}
if (grp >= OTX_CPT_MAX_ENGINE_GROUPS) {
dev_err(dev, "Requested group %d is > than maximum avail %d",
dev_err(dev, "Requested group %d is > than maximum avail %d\n",
grp, OTX_CPT_MAX_ENGINE_GROUPS);
return -EINVAL;
}
eng_grp = &cpt->eng_grps.grp[grp];
if (!eng_grp->is_enabled) {
dev_err(dev, "Requested engine group %d is disabled", grp);
dev_err(dev, "Requested engine group %d is disabled\n", grp);
return -EINVAL;
}
@@ -212,7 +212,7 @@ static void otx_cpt_handle_mbox_intr(struct otx_cpt_device *cpt, int vf)
vftype = otx_cpt_bind_vq_to_grp(cpt, vf, (u8)mbx.data);
if ((vftype != OTX_CPT_AE_TYPES) &&
(vftype != OTX_CPT_SE_TYPES)) {
dev_err(dev, "VF%d binding to eng group %llu failed",
dev_err(dev, "VF%d binding to eng group %llu failed\n",
vf, mbx.data);
otx_cptpf_mbox_send_nack(cpt, vf, &mbx);
} else {

95
drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c
View File

@@ -62,7 +62,7 @@ static struct otx_cpt_bitmap get_cores_bmap(struct device *dev,
int i;
if (eng_grp->g->engs_num > OTX_CPT_MAX_ENGINES) {
dev_err(dev, "unsupported number of engines %d on octeontx",
dev_err(dev, "unsupported number of engines %d on octeontx\n",
eng_grp->g->engs_num);
return bmap;
}
@@ -78,7 +78,7 @@ static struct otx_cpt_bitmap get_cores_bmap(struct device *dev,
}
if (!found)
dev_err(dev, "No engines reserved for engine group %d",
dev_err(dev, "No engines reserved for engine group %d\n",
eng_grp->idx);
return bmap;
}
@@ -306,7 +306,7 @@ static int process_tar_file(struct device *dev,
ucode_size = ntohl(ucode_hdr->code_length) * 2;
if (!ucode_size || (size < round_up(ucode_size, 16) +
sizeof(struct otx_cpt_ucode_hdr) + OTX_CPT_UCODE_SIGN_LEN)) {
dev_err(dev, "Ucode %s invalid size", filename);
dev_err(dev, "Ucode %s invalid size\n", filename);
return -EINVAL;
}
@@ -379,18 +379,18 @@ static void print_tar_dbg_info(struct tar_arch_info_t *tar_arch,
{
struct tar_ucode_info_t *curr;
pr_debug("Tar archive filename %s", tar_filename);
pr_debug("Tar archive pointer %p, size %ld", tar_arch->fw->data,
pr_debug("Tar archive filename %s\n", tar_filename);
pr_debug("Tar archive pointer %p, size %ld\n", tar_arch->fw->data,
tar_arch->fw->size);
list_for_each_entry(curr, &tar_arch->ucodes, list) {
pr_debug("Ucode filename %s", curr->ucode.filename);
pr_debug("Ucode version string %s", curr->ucode.ver_str);
pr_debug("Ucode version %d.%d.%d.%d",
pr_debug("Ucode filename %s\n", curr->ucode.filename);
pr_debug("Ucode version string %s\n", curr->ucode.ver_str);
pr_debug("Ucode version %d.%d.%d.%d\n",
curr->ucode.ver_num.nn, curr->ucode.ver_num.xx,
curr->ucode.ver_num.yy, curr->ucode.ver_num.zz);
pr_debug("Ucode type (%d) %s", curr->ucode.type,
pr_debug("Ucode type (%d) %s\n", curr->ucode.type,
get_ucode_type_str(curr->ucode.type));
pr_debug("Ucode size %d", curr->ucode.size);
pr_debug("Ucode size %d\n", curr->ucode.size);
pr_debug("Ucode ptr %p\n", curr->ucode_ptr);
}
}
@@ -417,14 +417,14 @@ static struct tar_arch_info_t *load_tar_archive(struct device *dev,
goto release_tar_arch;
if (tar_arch->fw->size < TAR_BLOCK_LEN) {
dev_err(dev, "Invalid tar archive %s ", tar_filename);
dev_err(dev, "Invalid tar archive %s\n", tar_filename);
goto release_tar_arch;
}
tar_size = tar_arch->fw->size;
tar_blk = (struct tar_blk_t *) tar_arch->fw->data;
if (strncmp(tar_blk->hdr.magic, TAR_MAGIC, TAR_MAGIC_LEN - 1)) {
dev_err(dev, "Unsupported format of tar archive %s",
dev_err(dev, "Unsupported format of tar archive %s\n",
tar_filename);
goto release_tar_arch;
}
@@ -437,7 +437,7 @@ static struct tar_arch_info_t *load_tar_archive(struct device *dev,
if (tar_offs + cur_size > tar_size ||
tar_offs + 2*TAR_BLOCK_LEN > tar_size) {
dev_err(dev, "Invalid tar archive %s ", tar_filename);
dev_err(dev, "Invalid tar archive %s\n", tar_filename);
goto release_tar_arch;
}
@@ -458,7 +458,7 @@ static struct tar_arch_info_t *load_tar_archive(struct device *dev,
/* Check for the end of the archive */
if (tar_offs + 2*TAR_BLOCK_LEN > tar_size) {
dev_err(dev, "Invalid tar archive %s ", tar_filename);
dev_err(dev, "Invalid tar archive %s\n", tar_filename);
goto release_tar_arch;
}
@@ -563,13 +563,13 @@ static void print_engs_info(struct otx_cpt_eng_grp_info *eng_grp,
static void print_ucode_dbg_info(struct otx_cpt_ucode *ucode)
{
pr_debug("Ucode info");
pr_debug("Ucode version string %s", ucode->ver_str);
pr_debug("Ucode version %d.%d.%d.%d", ucode->ver_num.nn,
pr_debug("Ucode info\n");
pr_debug("Ucode version string %s\n", ucode->ver_str);
pr_debug("Ucode version %d.%d.%d.%d\n", ucode->ver_num.nn,
ucode->ver_num.xx, ucode->ver_num.yy, ucode->ver_num.zz);
pr_debug("Ucode type %s", get_ucode_type_str(ucode->type));
pr_debug("Ucode size %d", ucode->size);
pr_debug("Ucode virt address %16.16llx", (u64)ucode->align_va);
pr_debug("Ucode type %s\n", get_ucode_type_str(ucode->type));
pr_debug("Ucode size %d\n", ucode->size);
pr_debug("Ucode virt address %16.16llx\n", (u64)ucode->align_va);
pr_debug("Ucode phys address %16.16llx\n", ucode->align_dma);
}
@@ -600,19 +600,19 @@ static void print_dbg_info(struct device *dev,
u32 mask[4];
int i, j;
pr_debug("Engine groups global info");
pr_debug("max SE %d, max AE %d",
pr_debug("Engine groups global info\n");
pr_debug("max SE %d, max AE %d\n",
eng_grps->avail.max_se_cnt, eng_grps->avail.max_ae_cnt);
pr_debug("free SE %d", eng_grps->avail.se_cnt);
pr_debug("free AE %d", eng_grps->avail.ae_cnt);
pr_debug("free SE %d\n", eng_grps->avail.se_cnt);
pr_debug("free AE %d\n", eng_grps->avail.ae_cnt);
for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) {
grp = &eng_grps->grp[i];
pr_debug("engine_group%d, state %s", i, grp->is_enabled ?
pr_debug("engine_group%d, state %s\n", i, grp->is_enabled ?
"enabled" : "disabled");
if (grp->is_enabled) {
mirrored_grp = &eng_grps->grp[grp->mirror.idx];
pr_debug("Ucode0 filename %s, version %s",
pr_debug("Ucode0 filename %s, version %s\n",
grp->mirror.is_ena ?
mirrored_grp->ucode[0].filename :
grp->ucode[0].filename,
@@ -626,18 +626,18 @@ static void print_dbg_info(struct device *dev,
if (engs->type) {
print_engs_info(grp, engs_info,
2*OTX_CPT_UCODE_NAME_LENGTH, j);
pr_debug("Slot%d: %s", j, engs_info);
pr_debug("Slot%d: %s\n", j, engs_info);
bitmap_to_arr32(mask, engs->bmap,
eng_grps->engs_num);
pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x",
pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x\n",
mask[3], mask[2], mask[1], mask[0]);
} else
pr_debug("Slot%d not used", j);
pr_debug("Slot%d not used\n", j);
}
if (grp->is_enabled) {
cpt_print_engines_mask(grp, dev, engs_mask,
OTX_CPT_UCODE_NAME_LENGTH);
pr_debug("Cmask: %s", engs_mask);
pr_debug("Cmask: %s\n", engs_mask);
}
}
}
@@ -766,7 +766,7 @@ static int check_engines_availability(struct device *dev,
if (avail_cnt < req_eng->count) {
dev_err(dev,
"Error available %s engines %d < than requested %d",
"Error available %s engines %d < than requested %d\n",
get_eng_type_str(req_eng->type),
avail_cnt, req_eng->count);
return -EBUSY;
@@ -867,7 +867,7 @@ static int copy_ucode_to_dma_mem(struct device *dev,
OTX_CPT_UCODE_ALIGNMENT,
&ucode->dma, GFP_KERNEL);
if (!ucode->va) {
dev_err(dev, "Unable to allocate space for microcode");
dev_err(dev, "Unable to allocate space for microcode\n");
return -ENOMEM;
}
ucode->align_va = PTR_ALIGN(ucode->va, OTX_CPT_UCODE_ALIGNMENT);
@@ -905,15 +905,15 @@ static int ucode_load(struct device *dev, struct otx_cpt_ucode *ucode,
ucode->size = ntohl(ucode_hdr->code_length) * 2;
if (!ucode->size || (fw->size < round_up(ucode->size, 16)
+ sizeof(struct otx_cpt_ucode_hdr) + OTX_CPT_UCODE_SIGN_LEN)) {
dev_err(dev, "Ucode %s invalid size", ucode_filename);
dev_err(dev, "Ucode %s invalid size\n", ucode_filename);
ret = -EINVAL;
goto release_fw;
}
ret = get_ucode_type(ucode_hdr, &ucode->type);
if (ret) {
dev_err(dev, "Microcode %s unknown type 0x%x", ucode->filename,
ucode->type);
dev_err(dev, "Microcode %s unknown type 0x%x\n",
ucode->filename, ucode->type);
goto release_fw;
}
@@ -1083,7 +1083,7 @@ static int eng_grp_update_masks(struct device *dev,
break;
default:
dev_err(dev, "Invalid engine type %d", engs->type);
dev_err(dev, "Invalid engine type %d\n", engs->type);
return -EINVAL;
}
@@ -1142,13 +1142,14 @@ static int delete_engine_group(struct device *dev,
return -EINVAL;
if (eng_grp->mirror.ref_count) {
dev_err(dev, "Can't delete engine_group%d as it is used by:",
dev_err(dev, "Can't delete engine_group%d as it is used by engine_group(s):",
eng_grp->idx);
for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) {
if (eng_grp->g->grp[i].mirror.is_ena &&
eng_grp->g->grp[i].mirror.idx == eng_grp->idx)
dev_err(dev, "engine_group%d", i);
pr_cont(" %d", i);
}
pr_cont("\n");
return -EINVAL;
}
@@ -1182,7 +1183,7 @@ static int validate_1_ucode_scenario(struct device *dev,
if (!otx_cpt_uc_supports_eng_type(&eng_grp->ucode[0],
engs[i].type)) {
dev_err(dev,
"Microcode %s does not support %s engines",
"Microcode %s does not support %s engines\n",
eng_grp->ucode[0].filename,
get_eng_type_str(engs[i].type));
return -EINVAL;
@@ -1220,7 +1221,7 @@ static int create_engine_group(struct device *dev,
/* Validate if requested engine types are supported by this device */
for (i = 0; i < engs_cnt; i++)
if (!dev_supports_eng_type(eng_grps, engs[i].type)) {
dev_err(dev, "Device does not support %s engines",
dev_err(dev, "Device does not support %s engines\n",
get_eng_type_str(engs[i].type));
return -EPERM;
}
@@ -1228,7 +1229,7 @@ static int create_engine_group(struct device *dev,
/* Find engine group which is not used */
eng_grp = find_unused_eng_grp(eng_grps);
if (!eng_grp) {
dev_err(dev, "Error all engine groups are being used");
dev_err(dev, "Error all engine groups are being used\n");
return -ENOSPC;
}
@@ -1298,11 +1299,11 @@ static int create_engine_group(struct device *dev,
eng_grp->is_enabled = true;
if (eng_grp->mirror.is_ena)
dev_info(dev,
"Engine_group%d: reuse microcode %s from group %d",
"Engine_group%d: reuse microcode %s from group %d\n",
eng_grp->idx, mirrored_eng_grp->ucode[0].ver_str,
mirrored_eng_grp->idx);
else
dev_info(dev, "Engine_group%d: microcode loaded %s",
dev_info(dev, "Engine_group%d: microcode loaded %s\n",
eng_grp->idx, eng_grp->ucode[0].ver_str);
return 0;
@@ -1412,14 +1413,14 @@ static ssize_t ucode_load_store(struct device *dev,
} else {
if (del_grp_idx < 0 ||
del_grp_idx >= OTX_CPT_MAX_ENGINE_GROUPS) {
dev_err(dev, "Invalid engine group index %d",
dev_err(dev, "Invalid engine group index %d\n",
del_grp_idx);
ret = -EINVAL;
return ret;
}
if (!eng_grps->grp[del_grp_idx].is_enabled) {
dev_err(dev, "Error engine_group%d is not configured",
dev_err(dev, "Error engine_group%d is not configured\n",
del_grp_idx);
ret = -EINVAL;
return ret;
@@ -1568,7 +1569,7 @@ void otx_cpt_disable_all_cores(struct otx_cpt_device *cpt)
udelay(CSR_DELAY);
reg = readq(cpt->reg_base + OTX_CPT_PF_EXEC_BUSY);
if (timeout--) {
dev_warn(&cpt->pdev->dev, "Cores still busy");
dev_warn(&cpt->pdev->dev, "Cores still busy\n");
break;
}
}
@@ -1626,7 +1627,7 @@ int otx_cpt_init_eng_grps(struct pci_dev *pdev,
eng_grps->avail.max_ae_cnt;
if (eng_grps->engs_num > OTX_CPT_MAX_ENGINES) {
dev_err(&pdev->dev,
"Number of engines %d > than max supported %d",
"Number of engines %d > than max supported %d\n",
eng_grps->engs_num, OTX_CPT_MAX_ENGINES);
ret = -EINVAL;
goto err;

6
drivers/crypto/marvell/octeontx/otx_cptvf_algs.c
View File

@@ -1660,7 +1660,7 @@ int otx_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
case OTX_CPT_SE_TYPES:
count = atomic_read(&se_devices.count);
if (count >= CPT_MAX_VF_NUM) {
dev_err(&pdev->dev, "No space to add a new device");
dev_err(&pdev->dev, "No space to add a new device\n");
ret = -ENOSPC;
goto err;
}
@@ -1687,7 +1687,7 @@ int otx_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
case OTX_CPT_AE_TYPES:
count = atomic_read(&ae_devices.count);
if (count >= CPT_MAX_VF_NUM) {
dev_err(&pdev->dev, "No space to a add new device");
dev_err(&pdev->dev, "No space to a add new device\n");
ret = -ENOSPC;
goto err;
}
@@ -1728,7 +1728,7 @@ void otx_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod,
}
if (!dev_found) {
dev_err(&pdev->dev, "%s device not found", __func__);
dev_err(&pdev->dev, "%s device not found\n", __func__);
goto exit;
}

12
drivers/crypto/marvell/octeontx/otx_cptvf_main.c
View File

@@ -584,7 +584,7 @@ static irqreturn_t cptvf_done_intr_handler(int __always_unused irq,
cptvf_write_vq_done_ack(cptvf, intr);
wqe = get_cptvf_vq_wqe(cptvf, 0);
if (unlikely(!wqe)) {
dev_err(&pdev->dev, "No work to schedule for VF (%d)",
dev_err(&pdev->dev, "No work to schedule for VF (%d)\n",
cptvf->vfid);
return IRQ_NONE;
}
@@ -602,7 +602,7 @@ static void cptvf_set_irq_affinity(struct otx_cptvf *cptvf, int vec)
if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
GFP_KERNEL)) {
dev_err(&pdev->dev,
"Allocation failed for affinity_mask for VF %d",
"Allocation failed for affinity_mask for VF %d\n",
cptvf->vfid);
return;
}
@@ -691,7 +691,7 @@ static ssize_t vf_engine_group_store(struct device *dev,
return -EINVAL;
if (val >= OTX_CPT_MAX_ENGINE_GROUPS) {
dev_err(dev, "Engine group >= than max available groups %d",
dev_err(dev, "Engine group >= than max available groups %d\n",
OTX_CPT_MAX_ENGINE_GROUPS);
return -EINVAL;
}
@@ -837,7 +837,7 @@ static int otx_cptvf_probe(struct pci_dev *pdev,
cptvf_misc_intr_handler, 0, "CPT VF misc intr",
cptvf);
if (err) {
dev_err(dev, "Failed to request misc irq");
dev_err(dev, "Failed to request misc irq\n");
goto free_vectors;
}
@@ -854,7 +854,7 @@ static int otx_cptvf_probe(struct pci_dev *pdev,
cptvf->cqinfo.qchunksize = OTX_CPT_CMD_QCHUNK_SIZE;
err = cptvf_sw_init(cptvf, OTX_CPT_CMD_QLEN, OTX_CPT_NUM_QS_PER_VF);
if (err) {
dev_err(dev, "cptvf_sw_init() failed");
dev_err(dev, "cptvf_sw_init() failed\n");
goto free_misc_irq;
}
/* Convey VQ LEN to PF */
@@ -946,7 +946,7 @@ static void otx_cptvf_remove(struct pci_dev *pdev)
/* Convey DOWN to PF */
if (otx_cptvf_send_vf_down(cptvf)) {
dev_err(&pdev->dev, "PF not responding to DOWN msg");
dev_err(&pdev->dev, "PF not responding to DOWN msg\n");
} else {
sysfs_remove_group(&pdev->dev.kobj, &otx_cptvf_sysfs_group);
otx_cpt_crypto_exit(pdev, THIS_MODULE, cptvf->vftype);

10
drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c
View File

@@ -314,7 +314,7 @@ static int process_request(struct pci_dev *pdev, struct otx_cpt_req_info *req,
GFP_ATOMIC;
ret = setup_sgio_list(pdev, &info, req, gfp);
if (unlikely(ret)) {
dev_err(&pdev->dev, "Setting up SG list failed");
dev_err(&pdev->dev, "Setting up SG list failed\n");
goto request_cleanup;
}
cpt_req->dlen = info->dlen;
@@ -410,17 +410,17 @@ int otx_cpt_do_request(struct pci_dev *pdev, struct otx_cpt_req_info *req,
struct otx_cptvf *cptvf = pci_get_drvdata(pdev);
if (!otx_cpt_device_ready(cptvf)) {
dev_err(&pdev->dev, "CPT Device is not ready");
dev_err(&pdev->dev, "CPT Device is not ready\n");
return -ENODEV;
}
if ((cptvf->vftype == OTX_CPT_SE_TYPES) && (!req->ctrl.s.se_req)) {
dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request\n",
cptvf->vfid);
return -EINVAL;
} else if ((cptvf->vftype == OTX_CPT_AE_TYPES) &&
(req->ctrl.s.se_req)) {
dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request\n",
cptvf->vfid);
return -EINVAL;
}
@@ -461,7 +461,7 @@ static int cpt_process_ccode(struct pci_dev *pdev,
/* check for timeout */
if (time_after_eq(jiffies, cpt_info->time_in +
OTX_CPT_COMMAND_TIMEOUT * HZ))
dev_warn(&pdev->dev, "Request timed out 0x%p", req);
dev_warn(&pdev->dev, "Request timed out 0x%p\n", req);
else if (cpt_info->extra_time < OTX_CPT_TIME_IN_RESET_COUNT) {
cpt_info->time_in = jiffies;
cpt_info->extra_time++;

7
drivers/crypto/mediatek/mtk-sha.c
View File

@@ -805,12 +805,9 @@ static int mtk_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
size_t ds = crypto_shash_digestsize(bctx->shash);
int err, i;
SHASH_DESC_ON_STACK(shash, bctx->shash);
shash->tfm = bctx->shash;
if (keylen > bs) {
err = crypto_shash_digest(shash, key, keylen, bctx->ipad);
err = crypto_shash_tfm_digest(bctx->shash, key, keylen,
bctx->ipad);
if (err)
return err;
keylen = ds;

7
drivers/crypto/n2_core.c
View File

@@ -462,7 +462,6 @@ static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
struct crypto_shash *child_shash = ctx->child_shash;
struct crypto_ahash *fallback_tfm;
SHASH_DESC_ON_STACK(shash, child_shash);
int err, bs, ds;
fallback_tfm = ctx->base.fallback_tfm;
@@ -470,14 +469,12 @@ static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
if (err)
return err;
shash->tfm = child_shash;
bs = crypto_shash_blocksize(child_shash);
ds = crypto_shash_digestsize(child_shash);
BUG_ON(ds > N2_HASH_KEY_MAX);
if (keylen > bs) {
err = crypto_shash_digest(shash, key, keylen,
ctx->hash_key);
err = crypto_shash_tfm_digest(child_shash, key, keylen,
ctx->hash_key);
if (err)
return err;
keylen = ds;

21
drivers/crypto/omap-sham.c
View File

@@ -33,7 +33,6 @@
#include <linux/of_irq.h>
#include <linux/delay.h>
#include <linux/crypto.h>
#include <linux/cryptohash.h>
#include <crypto/scatterwalk.h>
#include <crypto/algapi.h>
#include <crypto/sha.h>
@@ -1245,16 +1244,6 @@ static int omap_sham_update(struct ahash_request *req)
return omap_sham_enqueue(req, OP_UPDATE);
}
static int omap_sham_shash_digest(struct crypto_shash *tfm, u32 flags,
const u8 *data, unsigned int len, u8 *out)
{
SHASH_DESC_ON_STACK(shash, tfm);
shash->tfm = tfm;
return crypto_shash_digest(shash, data, len, out);
}
static int omap_sham_final_shash(struct ahash_request *req)
{
struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
@@ -1270,9 +1259,8 @@ static int omap_sham_final_shash(struct ahash_request *req)
!test_bit(FLAGS_AUTO_XOR, &ctx->dd->flags))
offset = get_block_size(ctx);
return omap_sham_shash_digest(tctx->fallback, req->base.flags,
ctx->buffer + offset,
ctx->bufcnt - offset, req->result);
return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer + offset,
ctx->bufcnt - offset, req->result);
}
static int omap_sham_final(struct ahash_request *req)
@@ -1351,9 +1339,8 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
return err;
if (keylen > bs) {
err = omap_sham_shash_digest(bctx->shash,
crypto_shash_get_flags(bctx->shash),
key, keylen, bctx->ipad);
err = crypto_shash_tfm_digest(bctx->shash, key, keylen,
bctx->ipad);
if (err)
return err;
keylen = ds;

39
drivers/crypto/s5p-sss.c
View File

@@ -1520,37 +1520,6 @@ static int s5p_hash_update(struct ahash_request *req)
return s5p_hash_enqueue(req, true); /* HASH_OP_UPDATE */
}
/**
* s5p_hash_shash_digest() - calculate shash digest
* @tfm: crypto transformation
* @flags: tfm flags
* @data: input data
* @len: length of data
* @out: output buffer
*/
static int s5p_hash_shash_digest(struct crypto_shash *tfm, u32 flags,
const u8 *data, unsigned int len, u8 *out)
{
SHASH_DESC_ON_STACK(shash, tfm);
shash->tfm = tfm;
return crypto_shash_digest(shash, data, len, out);
}
/**
* s5p_hash_final_shash() - calculate shash digest
* @req: AHASH request
*/
static int s5p_hash_final_shash(struct ahash_request *req)
{
struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
return s5p_hash_shash_digest(tctx->fallback, req->base.flags,
ctx->buffer, ctx->bufcnt, req->result);
}
/**
* s5p_hash_final() - close up hash and calculate digest
* @req: AHASH request
@@ -1582,8 +1551,12 @@ static int s5p_hash_final(struct ahash_request *req)
if (ctx->error)
return -EINVAL; /* uncompleted hash is not needed */
if (!ctx->digcnt && ctx->bufcnt < BUFLEN)
return s5p_hash_final_shash(req);
if (!ctx->digcnt && ctx->bufcnt < BUFLEN) {
struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer,
ctx->bufcnt, req->result);
}
return s5p_hash_enqueue(req, false); /* HASH_OP_FINAL */
}

268
drivers/crypto/stm32/stm32-crc32.c
View File

@@ -28,18 +28,23 @@
/* Registers values */
#define CRC_CR_RESET BIT(0)
#define CRC_CR_REVERSE (BIT(7) | BIT(6) | BIT(5))
#define CRC_INIT_DEFAULT 0xFFFFFFFF
#define CRC_CR_REV_IN_WORD (BIT(6) | BIT(5))
#define CRC_CR_REV_IN_BYTE BIT(5)
#define CRC_CR_REV_OUT BIT(7)
#define CRC32C_INIT_DEFAULT 0xFFFFFFFF
#define CRC_AUTOSUSPEND_DELAY 50
static unsigned int burst_size;
module_param(burst_size, uint, 0644);
MODULE_PARM_DESC(burst_size, "Select burst byte size (0 unlimited)");
struct stm32_crc {
struct list_head list;
struct device *dev;
void __iomem *regs;
struct clk *clk;
u8 pending_data[sizeof(u32)];
size_t nb_pending_bytes;
spinlock_t lock;
};
struct stm32_crc_list {
@@ -59,14 +64,13 @@ struct stm32_crc_ctx {
struct stm32_crc_desc_ctx {
u32 partial; /* crc32c: partial in first 4 bytes of that struct */
struct stm32_crc *crc;
};
static int stm32_crc32_cra_init(struct crypto_tfm *tfm)
{
struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm);
mctx->key = CRC_INIT_DEFAULT;
mctx->key = 0;
mctx->poly = CRC32_POLY_LE;
return 0;
}
@@ -75,7 +79,7 @@ static int stm32_crc32c_cra_init(struct crypto_tfm *tfm)
{
struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm);
mctx->key = CRC_INIT_DEFAULT;
mctx->key = CRC32C_INIT_DEFAULT;
mctx->poly = CRC32C_POLY_LE;
return 0;
}
@@ -92,32 +96,109 @@ static int stm32_crc_setkey(struct crypto_shash *tfm, const u8 *key,
return 0;
}
static struct stm32_crc *stm32_crc_get_next_crc(void)
{
struct stm32_crc *crc;
spin_lock_bh(&crc_list.lock);
crc = list_first_entry(&crc_list.dev_list, struct stm32_crc, list);
if (crc)
list_move_tail(&crc->list, &crc_list.dev_list);
spin_unlock_bh(&crc_list.lock);
return crc;
}
static int stm32_crc_init(struct shash_desc *desc)
{
struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc);
struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
struct stm32_crc *crc;
unsigned long flags;
spin_lock_bh(&crc_list.lock);
list_for_each_entry(crc, &crc_list.dev_list, list) {
ctx->crc = crc;
break;
}
spin_unlock_bh(&crc_list.lock);
crc = stm32_crc_get_next_crc();
if (!crc)
return -ENODEV;
pm_runtime_get_sync(ctx->crc->dev);
pm_runtime_get_sync(crc->dev);
spin_lock_irqsave(&crc->lock, flags);
/* Reset, set key, poly and configure in bit reverse mode */
writel_relaxed(bitrev32(mctx->key), ctx->crc->regs + CRC_INIT);
writel_relaxed(bitrev32(mctx->poly), ctx->crc->regs + CRC_POL);
writel_relaxed(CRC_CR_RESET | CRC_CR_REVERSE, ctx->crc->regs + CRC_CR);
writel_relaxed(bitrev32(mctx->key), crc->regs + CRC_INIT);
writel_relaxed(bitrev32(mctx->poly), crc->regs + CRC_POL);
writel_relaxed(CRC_CR_RESET | CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT,
crc->regs + CRC_CR);
/* Store partial result */
ctx->partial = readl_relaxed(ctx->crc->regs + CRC_DR);
ctx->crc->nb_pending_bytes = 0;
ctx->partial = readl_relaxed(crc->regs + CRC_DR);
pm_runtime_mark_last_busy(ctx->crc->dev);
pm_runtime_put_autosuspend(ctx->crc->dev);
spin_unlock_irqrestore(&crc->lock, flags);
pm_runtime_mark_last_busy(crc->dev);
pm_runtime_put_autosuspend(crc->dev);
return 0;
}
static int burst_update(struct shash_desc *desc, const u8 *d8,
size_t length)
{
struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc);
struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
struct stm32_crc *crc;
unsigned long flags;
crc = stm32_crc_get_next_crc();
if (!crc)
return -ENODEV;
pm_runtime_get_sync(crc->dev);
spin_lock_irqsave(&crc->lock, flags);
/*
* Restore previously calculated CRC for this context as init value
* Restore polynomial configuration
* Configure in register for word input data,
* Configure out register in reversed bit mode data.
*/
writel_relaxed(bitrev32(ctx->partial), crc->regs + CRC_INIT);
writel_relaxed(bitrev32(mctx->poly), crc->regs + CRC_POL);
writel_relaxed(CRC_CR_RESET | CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT,
crc->regs + CRC_CR);
if (d8 != PTR_ALIGN(d8, sizeof(u32))) {
/* Configure for byte data */
writel_relaxed(CRC_CR_REV_IN_BYTE | CRC_CR_REV_OUT,
crc->regs + CRC_CR);
while (d8 != PTR_ALIGN(d8, sizeof(u32)) && length) {
writeb_relaxed(*d8++, crc->regs + CRC_DR);
length--;
}
/* Configure for word data */
writel_relaxed(CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT,
crc->regs + CRC_CR);
}
for (; length >= sizeof(u32); d8 += sizeof(u32), length -= sizeof(u32))
writel_relaxed(*((u32 *)d8), crc->regs + CRC_DR);
if (length) {
/* Configure for byte data */
writel_relaxed(CRC_CR_REV_IN_BYTE | CRC_CR_REV_OUT,
crc->regs + CRC_CR);
while (length--)
writeb_relaxed(*d8++, crc->regs + CRC_DR);
}
/* Store partial result */
ctx->partial = readl_relaxed(crc->regs + CRC_DR);
spin_unlock_irqrestore(&crc->lock, flags);
pm_runtime_mark_last_busy(crc->dev);
pm_runtime_put_autosuspend(crc->dev);
return 0;
}
@@ -125,55 +206,26 @@ static int stm32_crc_init(struct shash_desc *desc)
static int stm32_crc_update(struct shash_desc *desc, const u8 *d8,
unsigned int length)
{
struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc);
struct stm32_crc *crc = ctx->crc;
u32 *d32;
unsigned int i;
const unsigned int burst_sz = burst_size;
unsigned int rem_sz;
const u8 *cur;
size_t size;
int ret;
pm_runtime_get_sync(crc->dev);
if (!burst_sz)
return burst_update(desc, d8, length);
if (unlikely(crc->nb_pending_bytes)) {
while (crc->nb_pending_bytes != sizeof(u32) && length) {
/* Fill in pending data */
crc->pending_data[crc->nb_pending_bytes++] = *(d8++);
length--;
}
if (crc->nb_pending_bytes == sizeof(u32)) {
/* Process completed pending data */
writel_relaxed(*(u32 *)crc->pending_data,
crc->regs + CRC_DR);
crc->nb_pending_bytes = 0;
}
/* Digest first bytes not 32bit aligned at first pass in the loop */
size = min(length,
burst_sz + (unsigned int)d8 - ALIGN_DOWN((unsigned int)d8,
sizeof(u32)));
for (rem_sz = length, cur = d8; rem_sz;
rem_sz -= size, cur += size, size = min(rem_sz, burst_sz)) {
ret = burst_update(desc, cur, size);
if (ret)
return ret;
}
d32 = (u32 *)d8;
for (i = 0; i < length >> 2; i++)
/* Process 32 bits data */
writel_relaxed(*(d32++), crc->regs + CRC_DR);
/* Store partial result */
ctx->partial = readl_relaxed(crc->regs + CRC_DR);
pm_runtime_mark_last_busy(crc->dev);
pm_runtime_put_autosuspend(crc->dev);
/* Check for pending data (non 32 bits) */
length &= 3;
if (likely(!length))
return 0;
if ((crc->nb_pending_bytes + length) >= sizeof(u32)) {
/* Shall not happen */
dev_err(crc->dev, "Pending data overflow\n");
return -EINVAL;
}
d8 = (const u8 *)d32;
for (i = 0; i < length; i++)
/* Store pending data */
crc->pending_data[crc->nb_pending_bytes++] = *(d8++);
return 0;
}
@@ -202,6 +254,8 @@ static int stm32_crc_digest(struct shash_desc *desc, const u8 *data,
return stm32_crc_init(desc) ?: stm32_crc_finup(desc, data, length, out);
}
static unsigned int refcnt;
static DEFINE_MUTEX(refcnt_lock);
static struct shash_alg algs[] = {
/* CRC-32 */
{
@@ -284,20 +338,29 @@ static int stm32_crc_probe(struct platform_device *pdev)
pm_runtime_get_noresume(dev);
pm_runtime_set_active(dev);
pm_runtime_irq_safe(dev);
pm_runtime_enable(dev);
spin_lock_init(&crc->lock);
platform_set_drvdata(pdev, crc);
spin_lock(&crc_list.lock);
list_add(&crc->list, &crc_list.dev_list);
spin_unlock(&crc_list.lock);
ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
if (ret) {
dev_err(dev, "Failed to register\n");
clk_disable_unprepare(crc->clk);
return ret;
mutex_lock(&refcnt_lock);
if (!refcnt) {
ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
if (ret) {
mutex_unlock(&refcnt_lock);
dev_err(dev, "Failed to register\n");
clk_disable_unprepare(crc->clk);
return ret;
}
}
refcnt++;
mutex_unlock(&refcnt_lock);
dev_info(dev, "Initialized\n");
@@ -318,7 +381,10 @@ static int stm32_crc_remove(struct platform_device *pdev)
list_del(&crc->list);
spin_unlock(&crc_list.lock);
crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
mutex_lock(&refcnt_lock);
if (!--refcnt)
crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
mutex_unlock(&refcnt_lock);
pm_runtime_disable(crc->dev);
pm_runtime_put_noidle(crc->dev);
@@ -328,34 +394,60 @@ static int stm32_crc_remove(struct platform_device *pdev)
return 0;
}
#ifdef CONFIG_PM
static int stm32_crc_runtime_suspend(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
clk_disable_unprepare(crc->clk);
return 0;
}
static int stm32_crc_runtime_resume(struct device *dev)
static int __maybe_unused stm32_crc_suspend(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
int ret;
ret = clk_prepare_enable(crc->clk);
ret = pm_runtime_force_suspend(dev);
if (ret)
return ret;
clk_unprepare(crc->clk);
return 0;
}
static int __maybe_unused stm32_crc_resume(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
int ret;
ret = clk_prepare(crc->clk);
if (ret) {
dev_err(crc->dev, "Failed to prepare_enable clock\n");
dev_err(crc->dev, "Failed to prepare clock\n");
return ret;
}
return pm_runtime_force_resume(dev);
}
static int __maybe_unused stm32_crc_runtime_suspend(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
clk_disable(crc->clk);
return 0;
}
static int __maybe_unused stm32_crc_runtime_resume(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
int ret;
ret = clk_enable(crc->clk);
if (ret) {
dev_err(crc->dev, "Failed to enable clock\n");
return ret;
}
return 0;
}
#endif
static const struct dev_pm_ops stm32_crc_pm_ops = {
SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
pm_runtime_force_resume)
SET_SYSTEM_SLEEP_PM_OPS(stm32_crc_suspend,
stm32_crc_resume)
SET_RUNTIME_PM_OPS(stm32_crc_runtime_suspend,
stm32_crc_runtime_resume, NULL)
};

38
drivers/crypto/stm32/stm32-hash.c
View File

@@ -507,6 +507,7 @@ static int stm32_hash_hmac_dma_send(struct stm32_hash_dev *hdev)
static int stm32_hash_dma_init(struct stm32_hash_dev *hdev)
{
struct dma_slave_config dma_conf;
struct dma_chan *chan;
int err;
memset(&dma_conf, 0, sizeof(dma_conf));
@@ -518,11 +519,11 @@ static int stm32_hash_dma_init(struct stm32_hash_dev *hdev)
dma_conf.dst_maxburst = hdev->dma_maxburst;
dma_conf.device_fc = false;
hdev->dma_lch = dma_request_chan(hdev->dev, "in");
if (IS_ERR(hdev->dma_lch)) {
dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");
return PTR_ERR(hdev->dma_lch);
}
chan = dma_request_chan(hdev->dev, "in");
if (IS_ERR(chan))
return PTR_ERR(chan);
hdev->dma_lch = chan;
err = dmaengine_slave_config(hdev->dma_lch, &dma_conf);
if (err) {
@@ -1463,8 +1464,11 @@ static int stm32_hash_probe(struct platform_device *pdev)
hdev->clk = devm_clk_get(&pdev->dev, NULL);
if (IS_ERR(hdev->clk)) {
dev_err(dev, "failed to get clock for hash (%lu)\n",
PTR_ERR(hdev->clk));
if (PTR_ERR(hdev->clk) != -EPROBE_DEFER) {
dev_err(dev, "failed to get clock for hash (%lu)\n",
PTR_ERR(hdev->clk));
}
return PTR_ERR(hdev->clk);
}
@@ -1482,7 +1486,12 @@ static int stm32_hash_probe(struct platform_device *pdev)
pm_runtime_enable(dev);
hdev->rst = devm_reset_control_get(&pdev->dev, NULL);
if (!IS_ERR(hdev->rst)) {
if (IS_ERR(hdev->rst)) {
if (PTR_ERR(hdev->rst) == -EPROBE_DEFER) {
ret = -EPROBE_DEFER;
goto err_reset;
}
} else {
reset_control_assert(hdev->rst);
udelay(2);
reset_control_deassert(hdev->rst);
@@ -1493,8 +1502,15 @@ static int stm32_hash_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, hdev);
ret = stm32_hash_dma_init(hdev);
if (ret)
switch (ret) {
case 0:
break;
case -ENOENT:
dev_dbg(dev, "DMA mode not available\n");
break;
default:
goto err_dma;
}
spin_lock(&stm32_hash.lock);
list_add_tail(&hdev->list, &stm32_hash.dev_list);
@@ -1532,10 +1548,10 @@ err_engine:
spin_lock(&stm32_hash.lock);
list_del(&hdev->list);
spin_unlock(&stm32_hash.lock);
err_dma:
if (hdev->dma_lch)
dma_release_channel(hdev->dma_lch);
err_reset:
pm_runtime_disable(dev);
pm_runtime_put_noidle(dev);