xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Browse Source 
Pull crypto updates from Herbert Xu:
 "Here is the crypto update for 4.3:

  API:

   - the AEAD interface transition is now complete.
   - add top-level skcipher interface.

  Drivers:

   - x86-64 acceleration for chacha20/poly1305.
   - add sunxi-ss Allwinner Security System crypto accelerator.
   - add RSA algorithm to qat driver.
   - add SRIOV support to qat driver.
   - add LS1021A support to caam.
   - add i.MX6 support to caam"

* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (163 commits)
  crypto: algif_aead - fix for multiple operations on AF_ALG sockets
  crypto: qat - enable legacy VFs
  MPI: Fix mpi_read_buffer
  crypto: qat - silence a static checker warning
  crypto: vmx - Fixing opcode issue
  crypto: caam - Use the preferred style for memory allocations
  crypto: caam - Propagate the real error code in caam_probe
  crypto: caam - Fix the error handling in caam_probe
  crypto: caam - fix writing to JQCR_MS when using service interface
  crypto: hash - Add AHASH_REQUEST_ON_STACK
  crypto: testmgr - Use new skcipher interface
  crypto: skcipher - Add top-level skcipher interface
  crypto: cmac - allow usage in FIPS mode
  crypto: sahara - Use dmam_alloc_coherent
  crypto: caam - Add support for LS1021A
  crypto: qat - Don't move data inside output buffer
  crypto: vmx - Fixing GHASH Key issue on little endian
  crypto: vmx - Fixing AES-CTR counter bug
  crypto: null - Add missing Kconfig tristate for NULL2
  crypto: nx - Add forward declaration for struct crypto_aead
  ...
This commit is contained in:
Linus Torvalds
2015-08-31 17:38:39 -07:00
parent f36fc04e4c bf433416e6
commit d4c90396ed
154 changed files with 15742 additions and 7446 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
drivers/crypto/nx/Kconfig
View File

@@ -14,11 +14,14 @@ config CRYPTO_DEV_NX_ENCRYPT
config CRYPTO_DEV_NX_COMPRESS
tristate "Compression acceleration support"
default y
select CRYPTO_ALGAPI
select 842_DECOMPRESS
help
Support for PowerPC Nest (NX) compression acceleration. This
module supports acceleration for compressing memory with the 842
algorithm. One of the platform drivers must be selected also.
If you choose 'M' here, this module will be called nx_compress.
algorithm using the cryptographic API. One of the platform
drivers must be selected also. If you choose 'M' here, this
module will be called nx_compress.
if CRYPTO_DEV_NX_COMPRESS
@@ -42,14 +45,4 @@ config CRYPTO_DEV_NX_COMPRESS_POWERNV
algorithm. This supports NX hardware on the PowerNV platform.
If you choose 'M' here, this module will be called nx_compress_powernv.
config CRYPTO_DEV_NX_COMPRESS_CRYPTO
tristate "Compression acceleration cryptographic interface"
select CRYPTO_ALGAPI
select 842_DECOMPRESS
default y
help
Support for PowerPC Nest (NX) accelerators using the cryptographic
API. If you choose 'M' here, this module will be called
nx_compress_crypto.
endif

8
drivers/crypto/nx/Makefile
View File

@@ -10,12 +10,8 @@ nx-crypto-objs := nx.o \
nx-sha256.o \
nx-sha512.o
obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS) += nx-compress.o nx-compress-platform.o
obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o
obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o
obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_CRYPTO) += nx-compress-crypto.o
obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o nx-compress.o
obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o nx-compress.o
nx-compress-objs := nx-842.o
nx-compress-platform-objs := nx-842-platform.o
nx-compress-pseries-objs := nx-842-pseries.o
nx-compress-powernv-objs := nx-842-powernv.o
nx-compress-crypto-objs := nx-842-crypto.o

580
drivers/crypto/nx/nx-842-crypto.c
View File

@@ -1,580 +0,0 @@
/*
* Cryptographic API for the NX-842 hardware compression.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* Copyright (C) IBM Corporation, 2011-2015
*
* Original Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
* Seth Jennings <sjenning@linux.vnet.ibm.com>
*
* Rewrite: Dan Streetman <ddstreet@ieee.org>
*
* This is an interface to the NX-842 compression hardware in PowerPC
* processors. Most of the complexity of this drvier is due to the fact that
* the NX-842 compression hardware requires the input and output data buffers
* to be specifically aligned, to be a specific multiple in length, and within
* specific minimum and maximum lengths. Those restrictions, provided by the
* nx-842 driver via nx842_constraints, mean this driver must use bounce
* buffers and headers to correct misaligned in or out buffers, and to split
* input buffers that are too large.
*
* This driver will fall back to software decompression if the hardware
* decompression fails, so this driver's decompression should never fail as
* long as the provided compressed buffer is valid. Any compressed buffer
* created by this driver will have a header (except ones where the input
* perfectly matches the constraints); so users of this driver cannot simply
* pass a compressed buffer created by this driver over to the 842 software
* decompression library. Instead, users must use this driver to decompress;
* if the hardware fails or is unavailable, the compressed buffer will be
* parsed and the header removed, and the raw 842 buffer(s) passed to the 842
* software decompression library.
*
* This does not fall back to software compression, however, since the caller
* of this function is specifically requesting hardware compression; if the
* hardware compression fails, the caller can fall back to software
* compression, and the raw 842 compressed buffer that the software compressor
* creates can be passed to this driver for hardware decompression; any
* buffer without our specific header magic is assumed to be a raw 842 buffer
* and passed directly to the hardware. Note that the software compression
* library will produce a compressed buffer that is incompatible with the
* hardware decompressor if the original input buffer length is not a multiple
* of 8; if such a compressed buffer is passed to this driver for
* decompression, the hardware will reject it and this driver will then pass
* it over to the software library for decompression.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/init.h>
#include <linux/module.h>
#include <linux/crypto.h>
#include <linux/vmalloc.h>
#include <linux/sw842.h>
#include <linux/ratelimit.h>
#include "nx-842.h"
/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit
* template (see lib/842/842.h), so this magic number will never appear at
* the start of a raw 842 compressed buffer. That is important, as any buffer
* passed to us without this magic is assumed to be a raw 842 compressed
* buffer, and passed directly to the hardware to decompress.
*/
#define NX842_CRYPTO_MAGIC (0xf842)
#define NX842_CRYPTO_GROUP_MAX (0x20)
#define NX842_CRYPTO_HEADER_SIZE(g) \
(sizeof(struct nx842_crypto_header) + \
sizeof(struct nx842_crypto_header_group) * (g))
#define NX842_CRYPTO_HEADER_MAX_SIZE \
NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX)
/* bounce buffer size */
#define BOUNCE_BUFFER_ORDER (2)
#define BOUNCE_BUFFER_SIZE \
((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER))
/* try longer on comp because we can fallback to sw decomp if hw is busy */
#define COMP_BUSY_TIMEOUT (250) /* ms */
#define DECOMP_BUSY_TIMEOUT (50) /* ms */
struct nx842_crypto_header_group {
__be16 padding; /* unused bytes at start of group */
__be32 compressed_length; /* compressed bytes in group */
__be32 uncompressed_length; /* bytes after decompression */
} __packed;
struct nx842_crypto_header {
__be16 magic; /* NX842_CRYPTO_MAGIC */
__be16 ignore; /* decompressed end bytes to ignore */
u8 groups; /* total groups in this header */
struct nx842_crypto_header_group group[];
} __packed;
struct nx842_crypto_param {
u8 *in;
unsigned int iremain;
u8 *out;
unsigned int oremain;
unsigned int ototal;
};
static int update_param(struct nx842_crypto_param *p,
unsigned int slen, unsigned int dlen)
{
if (p->iremain < slen)
return -EOVERFLOW;
if (p->oremain < dlen)
return -ENOSPC;
p->in += slen;
p->iremain -= slen;
p->out += dlen;
p->oremain -= dlen;
p->ototal += dlen;
return 0;
}
struct nx842_crypto_ctx {
u8 *wmem;
u8 *sbounce, *dbounce;
struct nx842_crypto_header header;
struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX];
};
static int nx842_crypto_init(struct crypto_tfm *tfm)
{
struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
ctx->wmem = kmalloc(nx842_workmem_size(), GFP_KERNEL);
ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) {
kfree(ctx->wmem);
free_page((unsigned long)ctx->sbounce);
free_page((unsigned long)ctx->dbounce);
return -ENOMEM;
}
return 0;
}
static void nx842_crypto_exit(struct crypto_tfm *tfm)
{
struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
kfree(ctx->wmem);
free_page((unsigned long)ctx->sbounce);
free_page((unsigned long)ctx->dbounce);
}
static int read_constraints(struct nx842_constraints *c)
{
int ret;
ret = nx842_constraints(c);
if (ret) {
pr_err_ratelimited("could not get nx842 constraints : %d\n",
ret);
return ret;
}
/* limit maximum, to always have enough bounce buffer to decompress */
if (c->maximum > BOUNCE_BUFFER_SIZE) {
c->maximum = BOUNCE_BUFFER_SIZE;
pr_info_once("limiting nx842 maximum to %x\n", c->maximum);
}
return 0;
}
static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf)
{
int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
/* compress should have added space for header */
if (s > be16_to_cpu(hdr->group[0].padding)) {
pr_err("Internal error: no space for header\n");
return -EINVAL;
}
memcpy(buf, hdr, s);
print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0);
return 0;
}
static int compress(struct nx842_crypto_ctx *ctx,
struct nx842_crypto_param *p,
struct nx842_crypto_header_group *g,
struct nx842_constraints *c,
u16 *ignore,
unsigned int hdrsize)
{
unsigned int slen = p->iremain, dlen = p->oremain, tmplen;
unsigned int adj_slen = slen;
u8 *src = p->in, *dst = p->out;
int ret, dskip = 0;
ktime_t timeout;
if (p->iremain == 0)
return -EOVERFLOW;
if (p->oremain == 0 || hdrsize + c->minimum > dlen)
return -ENOSPC;
if (slen % c->multiple)
adj_slen = round_up(slen, c->multiple);
if (slen < c->minimum)
adj_slen = c->minimum;
if (slen > c->maximum)
adj_slen = slen = c->maximum;
if (adj_slen > slen || (u64)src % c->alignment) {
adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE);
slen = min(slen, BOUNCE_BUFFER_SIZE);
if (adj_slen > slen)
memset(ctx->sbounce + slen, 0, adj_slen - slen);
memcpy(ctx->sbounce, src, slen);
src = ctx->sbounce;
slen = adj_slen;
pr_debug("using comp sbounce buffer, len %x\n", slen);
}
dst += hdrsize;
dlen -= hdrsize;
if ((u64)dst % c->alignment) {
dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst);
dst += dskip;
dlen -= dskip;
}
if (dlen % c->multiple)
dlen = round_down(dlen, c->multiple);
if (dlen < c->minimum) {
nospc:
dst = ctx->dbounce;
dlen = min(p->oremain, BOUNCE_BUFFER_SIZE);
dlen = round_down(dlen, c->multiple);
dskip = 0;
pr_debug("using comp dbounce buffer, len %x\n", dlen);
}
if (dlen > c->maximum)
dlen = c->maximum;
tmplen = dlen;
timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT);
do {
dlen = tmplen; /* reset dlen, if we're retrying */
ret = nx842_compress(src, slen, dst, &dlen, ctx->wmem);
/* possibly we should reduce the slen here, instead of
* retrying with the dbounce buffer?
*/
if (ret == -ENOSPC && dst != ctx->dbounce)
goto nospc;
} while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
if (ret)
return ret;
dskip += hdrsize;
if (dst == ctx->dbounce)
memcpy(p->out + dskip, dst, dlen);
g->padding = cpu_to_be16(dskip);
g->compressed_length = cpu_to_be32(dlen);
g->uncompressed_length = cpu_to_be32(slen);
if (p->iremain < slen) {
*ignore = slen - p->iremain;
slen = p->iremain;
}
pr_debug("compress slen %x ignore %x dlen %x padding %x\n",
slen, *ignore, dlen, dskip);
return update_param(p, slen, dskip + dlen);
}
static int nx842_crypto_compress(struct crypto_tfm *tfm,
const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen)
{
struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
struct nx842_crypto_header *hdr = &ctx->header;
struct nx842_crypto_param p;
struct nx842_constraints c;
unsigned int groups, hdrsize, h;
int ret, n;
bool add_header;
u16 ignore = 0;
p.in = (u8 *)src;
p.iremain = slen;
p.out = dst;
p.oremain = *dlen;
p.ototal = 0;
*dlen = 0;
ret = read_constraints(&c);
if (ret)
return ret;
groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX,
DIV_ROUND_UP(p.iremain, c.maximum));
hdrsize = NX842_CRYPTO_HEADER_SIZE(groups);
/* skip adding header if the buffers meet all constraints */
add_header = (p.iremain % c.multiple ||
p.iremain < c.minimum ||
p.iremain > c.maximum ||
(u64)p.in % c.alignment ||
p.oremain % c.multiple ||
p.oremain < c.minimum ||
p.oremain > c.maximum ||
(u64)p.out % c.alignment);
hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC);
hdr->groups = 0;
hdr->ignore = 0;
while (p.iremain > 0) {
n = hdr->groups++;
if (hdr->groups > NX842_CRYPTO_GROUP_MAX)
return -ENOSPC;
/* header goes before first group */
h = !n && add_header ? hdrsize : 0;
if (ignore)
pr_warn("interal error, ignore is set %x\n", ignore);
ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h);
if (ret)
return ret;
}
if (!add_header && hdr->groups > 1) {
pr_err("Internal error: No header but multiple groups\n");
return -EINVAL;
}
/* ignore indicates the input stream needed to be padded */
hdr->ignore = cpu_to_be16(ignore);
if (ignore)
pr_debug("marked %d bytes as ignore\n", ignore);
if (add_header)
ret = nx842_crypto_add_header(hdr, dst);
if (ret)
return ret;
*dlen = p.ototal;
pr_debug("compress total slen %x dlen %x\n", slen, *dlen);
return 0;
}
static int decompress(struct nx842_crypto_ctx *ctx,
struct nx842_crypto_param *p,
struct nx842_crypto_header_group *g,
struct nx842_constraints *c,
u16 ignore,
bool usehw)
{
unsigned int slen = be32_to_cpu(g->compressed_length);
unsigned int required_len = be32_to_cpu(g->uncompressed_length);
unsigned int dlen = p->oremain, tmplen;
unsigned int adj_slen = slen;
u8 *src = p->in, *dst = p->out;
u16 padding = be16_to_cpu(g->padding);
int ret, spadding = 0, dpadding = 0;
ktime_t timeout;
if (!slen || !required_len)
return -EINVAL;
if (p->iremain <= 0 || padding + slen > p->iremain)
return -EOVERFLOW;
if (p->oremain <= 0 || required_len - ignore > p->oremain)
return -ENOSPC;
src += padding;
if (!usehw)
goto usesw;
if (slen % c->multiple)
adj_slen = round_up(slen, c->multiple);
if (slen < c->minimum)
adj_slen = c->minimum;
if (slen > c->maximum)
goto usesw;
if (slen < adj_slen || (u64)src % c->alignment) {
/* we can append padding bytes because the 842 format defines
* an "end" template (see lib/842/842_decompress.c) and will
* ignore any bytes following it.
*/
if (slen < adj_slen)
memset(ctx->sbounce + slen, 0, adj_slen - slen);
memcpy(ctx->sbounce, src, slen);
src = ctx->sbounce;
spadding = adj_slen - slen;
slen = adj_slen;
pr_debug("using decomp sbounce buffer, len %x\n", slen);
}
if (dlen % c->multiple)
dlen = round_down(dlen, c->multiple);
if (dlen < required_len || (u64)dst % c->alignment) {
dst = ctx->dbounce;
dlen = min(required_len, BOUNCE_BUFFER_SIZE);
pr_debug("using decomp dbounce buffer, len %x\n", dlen);
}
if (dlen < c->minimum)
goto usesw;
if (dlen > c->maximum)
dlen = c->maximum;
tmplen = dlen;
timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT);
do {
dlen = tmplen; /* reset dlen, if we're retrying */
ret = nx842_decompress(src, slen, dst, &dlen, ctx->wmem);
} while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
if (ret) {
usesw:
/* reset everything, sw doesn't have constraints */
src = p->in + padding;
slen = be32_to_cpu(g->compressed_length);
spadding = 0;
dst = p->out;
dlen = p->oremain;
dpadding = 0;
if (dlen < required_len) { /* have ignore bytes */
dst = ctx->dbounce;
dlen = BOUNCE_BUFFER_SIZE;
}
pr_info_ratelimited("using software 842 decompression\n");
ret = sw842_decompress(src, slen, dst, &dlen);
}
if (ret)
return ret;
slen -= spadding;
dlen -= ignore;
if (ignore)
pr_debug("ignoring last %x bytes\n", ignore);
if (dst == ctx->dbounce)
memcpy(p->out, dst, dlen);
pr_debug("decompress slen %x padding %x dlen %x ignore %x\n",
slen, padding, dlen, ignore);
return update_param(p, slen + padding, dlen);
}
static int nx842_crypto_decompress(struct crypto_tfm *tfm,
const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen)
{
struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
struct nx842_crypto_header *hdr;
struct nx842_crypto_param p;
struct nx842_constraints c;
int n, ret, hdr_len;
u16 ignore = 0;
bool usehw = true;
p.in = (u8 *)src;
p.iremain = slen;
p.out = dst;
p.oremain = *dlen;
p.ototal = 0;
*dlen = 0;
if (read_constraints(&c))
usehw = false;
hdr = (struct nx842_crypto_header *)src;
/* If it doesn't start with our header magic number, assume it's a raw
* 842 compressed buffer and pass it directly to the hardware driver
*/
if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) {
struct nx842_crypto_header_group g = {
.padding = 0,
.compressed_length = cpu_to_be32(p.iremain),
.uncompressed_length = cpu_to_be32(p.oremain),
};
ret = decompress(ctx, &p, &g, &c, 0, usehw);
if (ret)
return ret;
*dlen = p.ototal;
return 0;
}
if (!hdr->groups) {
pr_err("header has no groups\n");
return -EINVAL;
}
if (hdr->groups > NX842_CRYPTO_GROUP_MAX) {
pr_err("header has too many groups %x, max %x\n",
hdr->groups, NX842_CRYPTO_GROUP_MAX);
return -EINVAL;
}
hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
if (hdr_len > slen)
return -EOVERFLOW;
memcpy(&ctx->header, src, hdr_len);
hdr = &ctx->header;
for (n = 0; n < hdr->groups; n++) {
/* ignore applies to last group */
if (n + 1 == hdr->groups)
ignore = be16_to_cpu(hdr->ignore);
ret = decompress(ctx, &p, &hdr->group[n], &c, ignore, usehw);
if (ret)
return ret;
}
*dlen = p.ototal;
pr_debug("decompress total slen %x dlen %x\n", slen, *dlen);
return 0;
}
static struct crypto_alg alg = {
.cra_name = "842",
.cra_driver_name = "842-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
.cra_ctxsize = sizeof(struct nx842_crypto_ctx),
.cra_module = THIS_MODULE,
.cra_init = nx842_crypto_init,
.cra_exit = nx842_crypto_exit,
.cra_u = { .compress = {
.coa_compress = nx842_crypto_compress,
.coa_decompress = nx842_crypto_decompress } }
};
static int __init nx842_crypto_mod_init(void)
{
return crypto_register_alg(&alg);
}
module_init(nx842_crypto_mod_init);
static void __exit nx842_crypto_mod_exit(void)
{
crypto_unregister_alg(&alg);
}
module_exit(nx842_crypto_mod_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Interface");
MODULE_ALIAS_CRYPTO("842");
MODULE_ALIAS_CRYPTO("842-nx");
MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");

84
drivers/crypto/nx/nx-842-platform.c
View File

@@ -1,84 +0,0 @@
#include "nx-842.h"
/* this is needed, separate from the main nx-842.c driver, because that main
* driver loads the platform drivers during its init(), and it expects one
* (or none) of the platform drivers to set this pointer to its driver.
* That means this pointer can't be in the main nx-842 driver, because it
* wouldn't be accessible until after the main driver loaded, which wouldn't
* be possible as it's waiting for the platform driver to load. So place it
* here.
*/
static struct nx842_driver *driver;
static DEFINE_SPINLOCK(driver_lock);
struct nx842_driver *nx842_platform_driver(void)
{
return driver;
}
EXPORT_SYMBOL_GPL(nx842_platform_driver);
bool nx842_platform_driver_set(struct nx842_driver *_driver)
{
bool ret = false;
spin_lock(&driver_lock);
if (!driver) {
driver = _driver;
ret = true;
} else
WARN(1, "can't set platform driver, already set to %s\n",
driver->name);
spin_unlock(&driver_lock);
return ret;
}
EXPORT_SYMBOL_GPL(nx842_platform_driver_set);
/* only call this from the platform driver exit function */
void nx842_platform_driver_unset(struct nx842_driver *_driver)
{
spin_lock(&driver_lock);
if (driver == _driver)
driver = NULL;
else if (driver)
WARN(1, "can't unset platform driver %s, currently set to %s\n",
_driver->name, driver->name);
else
WARN(1, "can't unset platform driver, already unset\n");
spin_unlock(&driver_lock);
}
EXPORT_SYMBOL_GPL(nx842_platform_driver_unset);
bool nx842_platform_driver_get(void)
{
bool ret = false;
spin_lock(&driver_lock);
if (driver)
ret = try_module_get(driver->owner);
spin_unlock(&driver_lock);
return ret;
}
EXPORT_SYMBOL_GPL(nx842_platform_driver_get);
void nx842_platform_driver_put(void)
{
spin_lock(&driver_lock);
if (driver)
module_put(driver->owner);
spin_unlock(&driver_lock);
}
EXPORT_SYMBOL_GPL(nx842_platform_driver_put);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
MODULE_DESCRIPTION("842 H/W Compression platform driver");

42
drivers/crypto/nx/nx-842-powernv.c
View File

@@ -26,6 +26,8 @@
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors");
MODULE_ALIAS_CRYPTO("842");
MODULE_ALIAS_CRYPTO("842-nx");
#define WORKMEM_ALIGN (CRB_ALIGN)
#define CSB_WAIT_MAX (5000) /* ms */
@@ -344,7 +346,8 @@ static int wait_for_csb(struct nx842_workmem *wmem,
}
/* successful completion */
pr_debug_ratelimited("Processed %u bytes in %lu us\n", csb->count,
pr_debug_ratelimited("Processed %u bytes in %lu us\n",
be32_to_cpu(csb->count),
(unsigned long)ktime_us_delta(now, start));
return 0;
@@ -581,9 +584,29 @@ static struct nx842_driver nx842_powernv_driver = {
.decompress = nx842_powernv_decompress,
};
static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)
{
return nx842_crypto_init(tfm, &nx842_powernv_driver);
}
static struct crypto_alg nx842_powernv_alg = {
.cra_name = "842",
.cra_driver_name = "842-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
.cra_ctxsize = sizeof(struct nx842_crypto_ctx),
.cra_module = THIS_MODULE,
.cra_init = nx842_powernv_crypto_init,
.cra_exit = nx842_crypto_exit,
.cra_u = { .compress = {
.coa_compress = nx842_crypto_compress,
.coa_decompress = nx842_crypto_decompress } }
};
static __init int nx842_powernv_init(void)
{
struct device_node *dn;
int ret;
/* verify workmem size/align restrictions */
BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN);
@@ -594,17 +617,14 @@ static __init int nx842_powernv_init(void)
BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
pr_info("loading\n");
for_each_compatible_node(dn, NULL, "ibm,power-nx")
nx842_powernv_probe(dn);
if (!nx842_ct) {
pr_err("no coprocessors found\n");
if (!nx842_ct)
return -ENODEV;
}
if (!nx842_platform_driver_set(&nx842_powernv_driver)) {
ret = crypto_register_alg(&nx842_powernv_alg);
if (ret) {
struct nx842_coproc *coproc, *n;
list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
@@ -612,11 +632,9 @@ static __init int nx842_powernv_init(void)
kfree(coproc);
}
return -EEXIST;
return ret;
}
pr_info("loaded\n");
return 0;
}
module_init(nx842_powernv_init);
@@ -625,13 +643,11 @@ static void __exit nx842_powernv_exit(void)
{
struct nx842_coproc *coproc, *n;
nx842_platform_driver_unset(&nx842_powernv_driver);
crypto_unregister_alg(&nx842_powernv_alg);
list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
list_del(&coproc->list);
kfree(coproc);
}
pr_info("unloaded\n");
}
module_exit(nx842_powernv_exit);

139
drivers/crypto/nx/nx-842-pseries.c
View File

@@ -29,6 +29,8 @@
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
MODULE_ALIAS_CRYPTO("842");
MODULE_ALIAS_CRYPTO("842-nx");
static struct nx842_constraints nx842_pseries_constraints = {
.alignment = DDE_BUFFER_ALIGN,
@@ -99,11 +101,6 @@ struct nx842_workmem {
#define NX842_HW_PAGE_SIZE (4096)
#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))
enum nx842_status {
UNAVAILABLE,
AVAILABLE
};
struct ibm_nx842_counters {
atomic64_t comp_complete;
atomic64_t comp_failed;
@@ -121,7 +118,6 @@ static struct nx842_devdata {
unsigned int max_sg_len;
unsigned int max_sync_size;
unsigned int max_sync_sg;
enum nx842_status status;
} __rcu *devdata;
static DEFINE_SPINLOCK(devdata_mutex);
@@ -230,9 +226,12 @@ static int nx842_validate_result(struct device *dev,
switch (csb->completion_code) {
case 0: /* Completed without error */
break;
case 64: /* Target bytes > Source bytes during compression */
case 64: /* Compression ok, but output larger than input */
dev_dbg(dev, "%s: output size larger than input size\n",
__func__);
break;
case 13: /* Output buffer too small */
dev_dbg(dev, "%s: Compression output larger than input\n",
dev_dbg(dev, "%s: Out of space in output buffer\n",
__func__);
return -ENOSPC;
case 66: /* Input data contains an illegal template field */
@@ -537,41 +536,36 @@ static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
devdata->max_sync_size = 0;
devdata->max_sync_sg = 0;
devdata->max_sg_len = 0;
devdata->status = UNAVAILABLE;
return 0;
} else
return -ENOENT;
}
/**
* nx842_OF_upd_status -- Update the device info from OF status prop
* nx842_OF_upd_status -- Check the device info from OF status prop
*
* The status property indicates if the accelerator is enabled. If the
* device is in the OF tree it indicates that the hardware is present.
* The status field indicates if the device is enabled when the status
* is 'okay'. Otherwise the device driver will be disabled.
*
* @devdata - struct nx842_devdata to update
* @prop - struct property point containing the maxsyncop for the update
*
* Returns:
* 0 - Device is available
* -EINVAL - Device is not available
* -ENODEV - Device is not available
*/
static int nx842_OF_upd_status(struct nx842_devdata *devdata,
struct property *prop) {
int ret = 0;
static int nx842_OF_upd_status(struct property *prop)
{
const char *status = (const char *)prop->value;
if (!strncmp(status, "okay", (size_t)prop->length)) {
devdata->status = AVAILABLE;
} else {
dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n",
__func__, status);
devdata->status = UNAVAILABLE;
}
if (!strncmp(status, "okay", (size_t)prop->length))
return 0;
if (!strncmp(status, "disabled", (size_t)prop->length))
return -ENODEV;
dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
return ret;
return -EINVAL;
}
/**
@@ -735,6 +729,10 @@ static int nx842_OF_upd(struct property *new_prop)
int ret = 0;
unsigned long flags;
new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
if (!new_devdata)
return -ENOMEM;
spin_lock_irqsave(&devdata_mutex, flags);
old_devdata = rcu_dereference_check(devdata,
lockdep_is_held(&devdata_mutex));
@@ -744,16 +742,10 @@ static int nx842_OF_upd(struct property *new_prop)
if (!old_devdata || !of_node) {
pr_err("%s: device is not available\n", __func__);
spin_unlock_irqrestore(&devdata_mutex, flags);
kfree(new_devdata);
return -ENODEV;
}
new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
if (!new_devdata) {
dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__);
ret = -ENOMEM;
goto error_out;
}
memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
new_devdata->counters = old_devdata->counters;
@@ -777,7 +769,7 @@ static int nx842_OF_upd(struct property *new_prop)
goto out;
/* Perform property updates */
ret = nx842_OF_upd_status(new_devdata, status);
ret = nx842_OF_upd_status(status);
if (ret)
goto error_out;
@@ -970,13 +962,43 @@ static struct nx842_driver nx842_pseries_driver = {
.decompress = nx842_pseries_decompress,
};
static int __init nx842_probe(struct vio_dev *viodev,
const struct vio_device_id *id)
static int nx842_pseries_crypto_init(struct crypto_tfm *tfm)
{
return nx842_crypto_init(tfm, &nx842_pseries_driver);
}
static struct crypto_alg nx842_pseries_alg = {
.cra_name = "842",
.cra_driver_name = "842-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
.cra_ctxsize = sizeof(struct nx842_crypto_ctx),
.cra_module = THIS_MODULE,
.cra_init = nx842_pseries_crypto_init,
.cra_exit = nx842_crypto_exit,
.cra_u = { .compress = {
.coa_compress = nx842_crypto_compress,
.coa_decompress = nx842_crypto_decompress } }
};
static int nx842_probe(struct vio_dev *viodev,
const struct vio_device_id *id)
{
struct nx842_devdata *old_devdata, *new_devdata = NULL;
unsigned long flags;
int ret = 0;
new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
if (!new_devdata)
return -ENOMEM;
new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
GFP_NOFS);
if (!new_devdata->counters) {
kfree(new_devdata);
return -ENOMEM;
}
spin_lock_irqsave(&devdata_mutex, flags);
old_devdata = rcu_dereference_check(devdata,
lockdep_is_held(&devdata_mutex));
@@ -989,21 +1011,6 @@ static int __init nx842_probe(struct vio_dev *viodev,
dev_set_drvdata(&viodev->dev, NULL);
new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
if (!new_devdata) {
dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__);
ret = -ENOMEM;
goto error_unlock;
}
new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
GFP_NOFS);
if (!new_devdata->counters) {
dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__);
ret = -ENOMEM;
goto error_unlock;
}
new_devdata->vdev = viodev;
new_devdata->dev = &viodev->dev;
nx842_OF_set_defaults(new_devdata);
@@ -1016,9 +1023,12 @@ static int __init nx842_probe(struct vio_dev *viodev,
of_reconfig_notifier_register(&nx842_of_nb);
ret = nx842_OF_upd(NULL);
if (ret && ret != -ENODEV) {
dev_err(&viodev->dev, "could not parse device tree. %d\n", ret);
ret = -1;
if (ret)
goto error;
ret = crypto_register_alg(&nx842_pseries_alg);
if (ret) {
dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);
goto error;
}
@@ -1043,7 +1053,7 @@ error:
return ret;
}
static int __exit nx842_remove(struct vio_dev *viodev)
static int nx842_remove(struct vio_dev *viodev)
{
struct nx842_devdata *old_devdata;
unsigned long flags;
@@ -1051,6 +1061,8 @@ static int __exit nx842_remove(struct vio_dev *viodev)
pr_info("Removing IBM Power 842 compression device\n");
sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
crypto_unregister_alg(&nx842_pseries_alg);
spin_lock_irqsave(&devdata_mutex, flags);
old_devdata = rcu_dereference_check(devdata,
lockdep_is_held(&devdata_mutex));
@@ -1074,18 +1086,16 @@ static struct vio_device_id nx842_vio_driver_ids[] = {
static struct vio_driver nx842_vio_driver = {
.name = KBUILD_MODNAME,
.probe = nx842_probe,
.remove = __exit_p(nx842_remove),
.remove = nx842_remove,
.get_desired_dma = nx842_get_desired_dma,
.id_table = nx842_vio_driver_ids,
};
static int __init nx842_init(void)
static int __init nx842_pseries_init(void)
{
struct nx842_devdata *new_devdata;
int ret;
pr_info("Registering IBM Power 842 compression driver\n");
if (!of_find_compatible_node(NULL, NULL, "ibm,compression"))
return -ENODEV;
@@ -1095,7 +1105,6 @@ static int __init nx842_init(void)
pr_err("Could not allocate memory for device data\n");
return -ENOMEM;
}
new_devdata->status = UNAVAILABLE;
RCU_INIT_POINTER(devdata, new_devdata);
ret = vio_register_driver(&nx842_vio_driver);
@@ -1106,24 +1115,18 @@ static int __init nx842_init(void)
return ret;
}
if (!nx842_platform_driver_set(&nx842_pseries_driver)) {
vio_unregister_driver(&nx842_vio_driver);
kfree(new_devdata);
return -EEXIST;
}
return 0;
}
module_init(nx842_init);
module_init(nx842_pseries_init);
static void __exit nx842_exit(void)
static void __exit nx842_pseries_exit(void)
{
struct nx842_devdata *old_devdata;
unsigned long flags;
pr_info("Exiting IBM Power 842 compression driver\n");
nx842_platform_driver_unset(&nx842_pseries_driver);
crypto_unregister_alg(&nx842_pseries_alg);
spin_lock_irqsave(&devdata_mutex, flags);
old_devdata = rcu_dereference_check(devdata,
lockdep_is_held(&devdata_mutex));
@@ -1136,5 +1139,5 @@ static void __exit nx842_exit(void)
vio_unregister_driver(&nx842_vio_driver);
}
module_exit(nx842_exit);
module_exit(nx842_pseries_exit);

572
drivers/crypto/nx/nx-842.c
View File

@@ -1,10 +1,5 @@
/*
* Driver frontend for IBM Power 842 compression accelerator
*
* Copyright (C) 2015 Dan Streetman, IBM Corp
*
* Designer of the Power data compression engine:
* Bulent Abali <abali@us.ibm.com>
* Cryptographic API for the NX-842 hardware compression.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -15,89 +10,522 @@
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* Copyright (C) IBM Corporation, 2011-2015
*
* Designer of the Power data compression engine:
* Bulent Abali <abali@us.ibm.com>
*
* Original Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
* Seth Jennings <sjenning@linux.vnet.ibm.com>
*
* Rewrite: Dan Streetman <ddstreet@ieee.org>
*
* This is an interface to the NX-842 compression hardware in PowerPC
* processors. Most of the complexity of this drvier is due to the fact that
* the NX-842 compression hardware requires the input and output data buffers
* to be specifically aligned, to be a specific multiple in length, and within
* specific minimum and maximum lengths. Those restrictions, provided by the
* nx-842 driver via nx842_constraints, mean this driver must use bounce
* buffers and headers to correct misaligned in or out buffers, and to split
* input buffers that are too large.
*
* This driver will fall back to software decompression if the hardware
* decompression fails, so this driver's decompression should never fail as
* long as the provided compressed buffer is valid. Any compressed buffer
* created by this driver will have a header (except ones where the input
* perfectly matches the constraints); so users of this driver cannot simply
* pass a compressed buffer created by this driver over to the 842 software
* decompression library. Instead, users must use this driver to decompress;
* if the hardware fails or is unavailable, the compressed buffer will be
* parsed and the header removed, and the raw 842 buffer(s) passed to the 842
* software decompression library.
*
* This does not fall back to software compression, however, since the caller
* of this function is specifically requesting hardware compression; if the
* hardware compression fails, the caller can fall back to software
* compression, and the raw 842 compressed buffer that the software compressor
* creates can be passed to this driver for hardware decompression; any
* buffer without our specific header magic is assumed to be a raw 842 buffer
* and passed directly to the hardware. Note that the software compression
* library will produce a compressed buffer that is incompatible with the
* hardware decompressor if the original input buffer length is not a multiple
* of 8; if such a compressed buffer is passed to this driver for
* decompression, the hardware will reject it and this driver will then pass
* it over to the software library for decompression.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/vmalloc.h>
#include <linux/sw842.h>
#include <linux/spinlock.h>
#include "nx-842.h"
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
/**
* nx842_constraints
*
* This provides the driver's constraints. Different nx842 implementations
* may have varying requirements. The constraints are:
* @alignment: All buffers should be aligned to this
* @multiple: All buffer lengths should be a multiple of this
* @minimum: Buffer lengths must not be less than this amount
* @maximum: Buffer lengths must not be more than this amount
*
* The constraints apply to all buffers and lengths, both input and output,
* for both compression and decompression, except for the minimum which
* only applies to compression input and decompression output; the
* compressed data can be less than the minimum constraint. It can be
* assumed that compressed data will always adhere to the multiple
* constraint.
*
* The driver may succeed even if these constraints are violated;
* however the driver can return failure or suffer reduced performance
* if any constraint is not met.
/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit
* template (see lib/842/842.h), so this magic number will never appear at
* the start of a raw 842 compressed buffer. That is important, as any buffer
* passed to us without this magic is assumed to be a raw 842 compressed
* buffer, and passed directly to the hardware to decompress.
*/
int nx842_constraints(struct nx842_constraints *c)
#define NX842_CRYPTO_MAGIC (0xf842)
#define NX842_CRYPTO_HEADER_SIZE(g) \
(sizeof(struct nx842_crypto_header) + \
sizeof(struct nx842_crypto_header_group) * (g))
#define NX842_CRYPTO_HEADER_MAX_SIZE \
NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX)
/* bounce buffer size */
#define BOUNCE_BUFFER_ORDER (2)
#define BOUNCE_BUFFER_SIZE \
((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER))
/* try longer on comp because we can fallback to sw decomp if hw is busy */
#define COMP_BUSY_TIMEOUT (250) /* ms */
#define DECOMP_BUSY_TIMEOUT (50) /* ms */
struct nx842_crypto_param {
u8 *in;
unsigned int iremain;
u8 *out;
unsigned int oremain;
unsigned int ototal;
};
static int update_param(struct nx842_crypto_param *p,
unsigned int slen, unsigned int dlen)
{
memcpy(c, nx842_platform_driver()->constraints, sizeof(*c));
if (p->iremain < slen)
return -EOVERFLOW;
if (p->oremain < dlen)
return -ENOSPC;
p->in += slen;
p->iremain -= slen;
p->out += dlen;
p->oremain -= dlen;
p->ototal += dlen;
return 0;
}
EXPORT_SYMBOL_GPL(nx842_constraints);
/**
* nx842_workmem_size
*
* Get the amount of working memory the driver requires.
*/
size_t nx842_workmem_size(void)
int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver)
{
return nx842_platform_driver()->workmem_size;
}
EXPORT_SYMBOL_GPL(nx842_workmem_size);
struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
int nx842_compress(const unsigned char *in, unsigned int ilen,
unsigned char *out, unsigned int *olen, void *wmem)
{
return nx842_platform_driver()->compress(in, ilen, out, olen, wmem);
}
EXPORT_SYMBOL_GPL(nx842_compress);
int nx842_decompress(const unsigned char *in, unsigned int ilen,
unsigned char *out, unsigned int *olen, void *wmem)
{
return nx842_platform_driver()->decompress(in, ilen, out, olen, wmem);
}
EXPORT_SYMBOL_GPL(nx842_decompress);
static __init int nx842_init(void)
{
request_module("nx-compress-powernv");
request_module("nx-compress-pseries");
/* we prevent loading if there's no platform driver, and we get the
* module that set it so it won't unload, so we don't need to check
* if it's set in any of the above functions
*/
if (!nx842_platform_driver_get()) {
pr_err("no nx842 driver found.\n");
return -ENODEV;
spin_lock_init(&ctx->lock);
ctx->driver = driver;
ctx->wmem = kmalloc(driver->workmem_size, GFP_KERNEL);
ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) {
kfree(ctx->wmem);
free_page((unsigned long)ctx->sbounce);
free_page((unsigned long)ctx->dbounce);
return -ENOMEM;
}
return 0;
}
module_init(nx842_init);
EXPORT_SYMBOL_GPL(nx842_crypto_init);
static void __exit nx842_exit(void)
void nx842_crypto_exit(struct crypto_tfm *tfm)
{
nx842_platform_driver_put();
struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
kfree(ctx->wmem);
free_page((unsigned long)ctx->sbounce);
free_page((unsigned long)ctx->dbounce);
}
module_exit(nx842_exit);
EXPORT_SYMBOL_GPL(nx842_crypto_exit);
static void check_constraints(struct nx842_constraints *c)
{
/* limit maximum, to always have enough bounce buffer to decompress */
if (c->maximum > BOUNCE_BUFFER_SIZE)
c->maximum = BOUNCE_BUFFER_SIZE;
}
static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf)
{
int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
/* compress should have added space for header */
if (s > be16_to_cpu(hdr->group[0].padding)) {
pr_err("Internal error: no space for header\n");
return -EINVAL;
}
memcpy(buf, hdr, s);
print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0);
return 0;
}
static int compress(struct nx842_crypto_ctx *ctx,
struct nx842_crypto_param *p,
struct nx842_crypto_header_group *g,
struct nx842_constraints *c,
u16 *ignore,
unsigned int hdrsize)
{
unsigned int slen = p->iremain, dlen = p->oremain, tmplen;
unsigned int adj_slen = slen;
u8 *src = p->in, *dst = p->out;
int ret, dskip = 0;
ktime_t timeout;
if (p->iremain == 0)
return -EOVERFLOW;
if (p->oremain == 0 || hdrsize + c->minimum > dlen)
return -ENOSPC;
if (slen % c->multiple)
adj_slen = round_up(slen, c->multiple);
if (slen < c->minimum)
adj_slen = c->minimum;
if (slen > c->maximum)
adj_slen = slen = c->maximum;
if (adj_slen > slen || (u64)src % c->alignment) {
adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE);
slen = min(slen, BOUNCE_BUFFER_SIZE);
if (adj_slen > slen)
memset(ctx->sbounce + slen, 0, adj_slen - slen);
memcpy(ctx->sbounce, src, slen);
src = ctx->sbounce;
slen = adj_slen;
pr_debug("using comp sbounce buffer, len %x\n", slen);
}
dst += hdrsize;
dlen -= hdrsize;
if ((u64)dst % c->alignment) {
dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst);
dst += dskip;
dlen -= dskip;
}
if (dlen % c->multiple)
dlen = round_down(dlen, c->multiple);
if (dlen < c->minimum) {
nospc:
dst = ctx->dbounce;
dlen = min(p->oremain, BOUNCE_BUFFER_SIZE);
dlen = round_down(dlen, c->multiple);
dskip = 0;
pr_debug("using comp dbounce buffer, len %x\n", dlen);
}
if (dlen > c->maximum)
dlen = c->maximum;
tmplen = dlen;
timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT);
do {
dlen = tmplen; /* reset dlen, if we're retrying */
ret = ctx->driver->compress(src, slen, dst, &dlen, ctx->wmem);
/* possibly we should reduce the slen here, instead of
* retrying with the dbounce buffer?
*/
if (ret == -ENOSPC && dst != ctx->dbounce)
goto nospc;
} while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
if (ret)
return ret;
dskip += hdrsize;
if (dst == ctx->dbounce)
memcpy(p->out + dskip, dst, dlen);
g->padding = cpu_to_be16(dskip);
g->compressed_length = cpu_to_be32(dlen);
g->uncompressed_length = cpu_to_be32(slen);
if (p->iremain < slen) {
*ignore = slen - p->iremain;
slen = p->iremain;
}
pr_debug("compress slen %x ignore %x dlen %x padding %x\n",
slen, *ignore, dlen, dskip);
return update_param(p, slen, dskip + dlen);
}
int nx842_crypto_compress(struct crypto_tfm *tfm,
const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen)
{
struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
struct nx842_crypto_header *hdr = &ctx->header;
struct nx842_crypto_param p;
struct nx842_constraints c = *ctx->driver->constraints;
unsigned int groups, hdrsize, h;
int ret, n;
bool add_header;
u16 ignore = 0;
check_constraints(&c);
p.in = (u8 *)src;
p.iremain = slen;
p.out = dst;
p.oremain = *dlen;
p.ototal = 0;
*dlen = 0;
groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX,
DIV_ROUND_UP(p.iremain, c.maximum));
hdrsize = NX842_CRYPTO_HEADER_SIZE(groups);
spin_lock_bh(&ctx->lock);
/* skip adding header if the buffers meet all constraints */
add_header = (p.iremain % c.multiple ||
p.iremain < c.minimum ||
p.iremain > c.maximum ||
(u64)p.in % c.alignment ||
p.oremain % c.multiple ||
p.oremain < c.minimum ||
p.oremain > c.maximum ||
(u64)p.out % c.alignment);
hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC);
hdr->groups = 0;
hdr->ignore = 0;
while (p.iremain > 0) {
n = hdr->groups++;
ret = -ENOSPC;
if (hdr->groups > NX842_CRYPTO_GROUP_MAX)
goto unlock;
/* header goes before first group */
h = !n && add_header ? hdrsize : 0;
if (ignore)
pr_warn("interal error, ignore is set %x\n", ignore);
ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h);
if (ret)
goto unlock;
}
if (!add_header && hdr->groups > 1) {
pr_err("Internal error: No header but multiple groups\n");
ret = -EINVAL;
goto unlock;
}
/* ignore indicates the input stream needed to be padded */
hdr->ignore = cpu_to_be16(ignore);
if (ignore)
pr_debug("marked %d bytes as ignore\n", ignore);
if (add_header)
ret = nx842_crypto_add_header(hdr, dst);
if (ret)
goto unlock;
*dlen = p.ototal;
pr_debug("compress total slen %x dlen %x\n", slen, *dlen);
unlock:
spin_unlock_bh(&ctx->lock);
return ret;
}
EXPORT_SYMBOL_GPL(nx842_crypto_compress);
static int decompress(struct nx842_crypto_ctx *ctx,
struct nx842_crypto_param *p,
struct nx842_crypto_header_group *g,
struct nx842_constraints *c,
u16 ignore)
{
unsigned int slen = be32_to_cpu(g->compressed_length);
unsigned int required_len = be32_to_cpu(g->uncompressed_length);
unsigned int dlen = p->oremain, tmplen;
unsigned int adj_slen = slen;
u8 *src = p->in, *dst = p->out;
u16 padding = be16_to_cpu(g->padding);
int ret, spadding = 0, dpadding = 0;
ktime_t timeout;
if (!slen || !required_len)
return -EINVAL;
if (p->iremain <= 0 || padding + slen > p->iremain)
return -EOVERFLOW;
if (p->oremain <= 0 || required_len - ignore > p->oremain)
return -ENOSPC;
src += padding;
if (slen % c->multiple)
adj_slen = round_up(slen, c->multiple);
if (slen < c->minimum)
adj_slen = c->minimum;
if (slen > c->maximum)
goto usesw;
if (slen < adj_slen || (u64)src % c->alignment) {
/* we can append padding bytes because the 842 format defines
* an "end" template (see lib/842/842_decompress.c) and will
* ignore any bytes following it.
*/
if (slen < adj_slen)
memset(ctx->sbounce + slen, 0, adj_slen - slen);
memcpy(ctx->sbounce, src, slen);
src = ctx->sbounce;
spadding = adj_slen - slen;
slen = adj_slen;
pr_debug("using decomp sbounce buffer, len %x\n", slen);
}
if (dlen % c->multiple)
dlen = round_down(dlen, c->multiple);
if (dlen < required_len || (u64)dst % c->alignment) {
dst = ctx->dbounce;
dlen = min(required_len, BOUNCE_BUFFER_SIZE);
pr_debug("using decomp dbounce buffer, len %x\n", dlen);
}
if (dlen < c->minimum)
goto usesw;
if (dlen > c->maximum)
dlen = c->maximum;
tmplen = dlen;
timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT);
do {
dlen = tmplen; /* reset dlen, if we're retrying */
ret = ctx->driver->decompress(src, slen, dst, &dlen, ctx->wmem);
} while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
if (ret) {
usesw:
/* reset everything, sw doesn't have constraints */
src = p->in + padding;
slen = be32_to_cpu(g->compressed_length);
spadding = 0;
dst = p->out;
dlen = p->oremain;
dpadding = 0;
if (dlen < required_len) { /* have ignore bytes */
dst = ctx->dbounce;
dlen = BOUNCE_BUFFER_SIZE;
}
pr_info_ratelimited("using software 842 decompression\n");
ret = sw842_decompress(src, slen, dst, &dlen);
}
if (ret)
return ret;
slen -= spadding;
dlen -= ignore;
if (ignore)
pr_debug("ignoring last %x bytes\n", ignore);
if (dst == ctx->dbounce)
memcpy(p->out, dst, dlen);
pr_debug("decompress slen %x padding %x dlen %x ignore %x\n",
slen, padding, dlen, ignore);
return update_param(p, slen + padding, dlen);
}
int nx842_crypto_decompress(struct crypto_tfm *tfm,
const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen)
{
struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
struct nx842_crypto_header *hdr;
struct nx842_crypto_param p;
struct nx842_constraints c = *ctx->driver->constraints;
int n, ret, hdr_len;
u16 ignore = 0;
check_constraints(&c);
p.in = (u8 *)src;
p.iremain = slen;
p.out = dst;
p.oremain = *dlen;
p.ototal = 0;
*dlen = 0;
hdr = (struct nx842_crypto_header *)src;
spin_lock_bh(&ctx->lock);
/* If it doesn't start with our header magic number, assume it's a raw
* 842 compressed buffer and pass it directly to the hardware driver
*/
if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) {
struct nx842_crypto_header_group g = {
.padding = 0,
.compressed_length = cpu_to_be32(p.iremain),
.uncompressed_length = cpu_to_be32(p.oremain),
};
ret = decompress(ctx, &p, &g, &c, 0);
if (ret)
goto unlock;
goto success;
}
if (!hdr->groups) {
pr_err("header has no groups\n");
ret = -EINVAL;
goto unlock;
}
if (hdr->groups > NX842_CRYPTO_GROUP_MAX) {
pr_err("header has too many groups %x, max %x\n",
hdr->groups, NX842_CRYPTO_GROUP_MAX);
ret = -EINVAL;
goto unlock;
}
hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
if (hdr_len > slen) {
ret = -EOVERFLOW;
goto unlock;
}
memcpy(&ctx->header, src, hdr_len);
hdr = &ctx->header;
for (n = 0; n < hdr->groups; n++) {
/* ignore applies to last group */
if (n + 1 == hdr->groups)
ignore = be16_to_cpu(hdr->ignore);
ret = decompress(ctx, &p, &hdr->group[n], &c, ignore);
if (ret)
goto unlock;
}
success:
*dlen = p.ototal;
pr_debug("decompress total slen %x dlen %x\n", slen, *dlen);
ret = 0;
unlock:
spin_unlock_bh(&ctx->lock);
return ret;
}
EXPORT_SYMBOL_GPL(nx842_crypto_decompress);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Driver");
MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");

65
drivers/crypto/nx/nx-842.h
View File

@@ -3,8 +3,9 @@
#define __NX_842_H__
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/sw842.h>
#include <linux/crypto.h>
#include <linux/of.h>
#include <linux/slab.h>
#include <linux/io.h>
@@ -104,6 +105,25 @@ static inline unsigned long nx842_get_pa(void *addr)
#define GET_FIELD(v, m) (((v) & (m)) >> MASK_LSH(m))
#define SET_FIELD(v, m, val) (((v) & ~(m)) | (((val) << MASK_LSH(m)) & (m)))
/**
* This provides the driver's constraints. Different nx842 implementations
* may have varying requirements. The constraints are:
* @alignment: All buffers should be aligned to this
* @multiple: All buffer lengths should be a multiple of this
* @minimum: Buffer lengths must not be less than this amount
* @maximum: Buffer lengths must not be more than this amount
*
* The constraints apply to all buffers and lengths, both input and output,
* for both compression and decompression, except for the minimum which
* only applies to compression input and decompression output; the
* compressed data can be less than the minimum constraint. It can be
* assumed that compressed data will always adhere to the multiple
* constraint.
*
* The driver may succeed even if these constraints are violated;
* however the driver can return failure or suffer reduced performance
* if any constraint is not met.
*/
struct nx842_constraints {
int alignment;
int multiple;
@@ -126,19 +146,40 @@ struct nx842_driver {
void *wrkmem);
};
struct nx842_driver *nx842_platform_driver(void);
bool nx842_platform_driver_set(struct nx842_driver *driver);
void nx842_platform_driver_unset(struct nx842_driver *driver);
bool nx842_platform_driver_get(void);
void nx842_platform_driver_put(void);
struct nx842_crypto_header_group {
__be16 padding; /* unused bytes at start of group */
__be32 compressed_length; /* compressed bytes in group */
__be32 uncompressed_length; /* bytes after decompression */
} __packed;
size_t nx842_workmem_size(void);
struct nx842_crypto_header {
__be16 magic; /* NX842_CRYPTO_MAGIC */
__be16 ignore; /* decompressed end bytes to ignore */
u8 groups; /* total groups in this header */
struct nx842_crypto_header_group group[];
} __packed;
int nx842_constraints(struct nx842_constraints *constraints);
#define NX842_CRYPTO_GROUP_MAX (0x20)
int nx842_compress(const unsigned char *in, unsigned int in_len,
unsigned char *out, unsigned int *out_len, void *wrkmem);
int nx842_decompress(const unsigned char *in, unsigned int in_len,
unsigned char *out, unsigned int *out_len, void *wrkmem);
struct nx842_crypto_ctx {
spinlock_t lock;
u8 *wmem;
u8 *sbounce, *dbounce;
struct nx842_crypto_header header;
struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX];
struct nx842_driver *driver;
};
int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver);
void nx842_crypto_exit(struct crypto_tfm *tfm);
int nx842_crypto_compress(struct crypto_tfm *tfm,
const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen);
int nx842_crypto_decompress(struct crypto_tfm *tfm,
const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen);
#endif /* __NX_842_H__ */

151
drivers/crypto/nx/nx-aes-ccm.c
View File

@@ -94,8 +94,6 @@ static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm,
return -EINVAL;
}
crypto_aead_crt(tfm)->authsize = authsize;
return 0;
}
@@ -111,8 +109,6 @@ static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm,
return -EINVAL;
}
crypto_aead_crt(tfm)->authsize = authsize;
return 0;
}
@@ -174,6 +170,7 @@ static int generate_pat(u8 *iv,
struct nx_crypto_ctx *nx_ctx,
unsigned int authsize,
unsigned int nbytes,
unsigned int assoclen,
u8 *out)
{
struct nx_sg *nx_insg = nx_ctx->in_sg;
@@ -200,16 +197,16 @@ static int generate_pat(u8 *iv,
* greater than 2^32.
*/
if (!req->assoclen) {
if (!assoclen) {
b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
} else if (req->assoclen <= 14) {
} else if (assoclen <= 14) {
/* if associated data is 14 bytes or less, we do 1 GCM
* operation on 2 AES blocks, B0 (stored in the csbcpb) and B1,
* which is fed in through the source buffers here */
b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
b1 = nx_ctx->priv.ccm.iauth_tag;
iauth_len = req->assoclen;
} else if (req->assoclen <= 65280) {
iauth_len = assoclen;
} else if (assoclen <= 65280) {
/* if associated data is less than (2^16 - 2^8), we construct
* B1 differently and feed in the associated data to a CCA
* operation */
@@ -223,7 +220,7 @@ static int generate_pat(u8 *iv,
}
/* generate B0 */
rc = generate_b0(iv, req->assoclen, authsize, nbytes, b0);
rc = generate_b0(iv, assoclen, authsize, nbytes, b0);
if (rc)
return rc;
@@ -233,22 +230,22 @@ static int generate_pat(u8 *iv,
*/
if (b1) {
memset(b1, 0, 16);
if (req->assoclen <= 65280) {
*(u16 *)b1 = (u16)req->assoclen;
scatterwalk_map_and_copy(b1 + 2, req->assoc, 0,
if (assoclen <= 65280) {
*(u16 *)b1 = assoclen;
scatterwalk_map_and_copy(b1 + 2, req->src, 0,
iauth_len, SCATTERWALK_FROM_SG);
} else {
*(u16 *)b1 = (u16)(0xfffe);
*(u32 *)&b1[2] = (u32)req->assoclen;
scatterwalk_map_and_copy(b1 + 6, req->assoc, 0,
*(u32 *)&b1[2] = assoclen;
scatterwalk_map_and_copy(b1 + 6, req->src, 0,
iauth_len, SCATTERWALK_FROM_SG);
}
}
/* now copy any remaining AAD to scatterlist and call nx... */
if (!req->assoclen) {
if (!assoclen) {
return rc;
} else if (req->assoclen <= 14) {
} else if (assoclen <= 14) {
unsigned int len = 16;
nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen);
@@ -280,7 +277,7 @@ static int generate_pat(u8 *iv,
return rc;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
atomic64_add(assoclen, &nx_ctx->stats->aes_bytes);
} else {
unsigned int processed = 0, to_process;
@@ -294,15 +291,15 @@ static int generate_pat(u8 *iv,
nx_ctx->ap->databytelen/NX_PAGE_SIZE);
do {
to_process = min_t(u32, req->assoclen - processed,
to_process = min_t(u32, assoclen - processed,
nx_ctx->ap->databytelen);
nx_insg = nx_walk_and_build(nx_ctx->in_sg,
nx_ctx->ap->sglen,
req->assoc, processed,
req->src, processed,
&to_process);
if ((to_process + processed) < req->assoclen) {
if ((to_process + processed) < assoclen) {
NX_CPB_FDM(nx_ctx->csbcpb_aead) |=
NX_FDM_INTERMEDIATE;
} else {
@@ -328,11 +325,10 @@ static int generate_pat(u8 *iv,
NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(req->assoclen,
&(nx_ctx->stats->aes_bytes));
atomic64_add(assoclen, &nx_ctx->stats->aes_bytes);
processed += to_process;
} while (processed < req->assoclen);
} while (processed < assoclen);
result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
}
@@ -343,7 +339,8 @@ static int generate_pat(u8 *iv,
}
static int ccm_nx_decrypt(struct aead_request *req,
struct blkcipher_desc *desc)
struct blkcipher_desc *desc,
unsigned int assoclen)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
@@ -360,10 +357,10 @@ static int ccm_nx_decrypt(struct aead_request *req,
/* copy out the auth tag to compare with later */
scatterwalk_map_and_copy(priv->oauth_tag,
req->src, nbytes, authsize,
req->src, nbytes + req->assoclen, authsize,
SCATTERWALK_FROM_SG);
rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes,
rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes, assoclen,
csbcpb->cpb.aes_ccm.in_pat_or_b0);
if (rc)
goto out;
@@ -383,8 +380,8 @@ static int ccm_nx_decrypt(struct aead_request *req,
NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src,
&to_process, processed,
csbcpb->cpb.aes_ccm.iv_or_ctr);
&to_process, processed + req->assoclen,
csbcpb->cpb.aes_ccm.iv_or_ctr);
if (rc)
goto out;
@@ -420,7 +417,8 @@ out:
}
static int ccm_nx_encrypt(struct aead_request *req,
struct blkcipher_desc *desc)
struct blkcipher_desc *desc,
unsigned int assoclen)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
@@ -432,7 +430,7 @@ static int ccm_nx_encrypt(struct aead_request *req,
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes,
rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes, assoclen,
csbcpb->cpb.aes_ccm.in_pat_or_b0);
if (rc)
goto out;
@@ -451,7 +449,7 @@ static int ccm_nx_encrypt(struct aead_request *req,
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src,
&to_process, processed,
&to_process, processed + req->assoclen,
csbcpb->cpb.aes_ccm.iv_or_ctr);
if (rc)
goto out;
@@ -483,7 +481,7 @@ static int ccm_nx_encrypt(struct aead_request *req,
/* copy out the auth tag */
scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac,
req->dst, nbytes, authsize,
req->dst, nbytes + req->assoclen, authsize,
SCATTERWALK_TO_SG);
out:
@@ -503,9 +501,8 @@ static int ccm4309_aes_nx_encrypt(struct aead_request *req)
memcpy(iv + 4, req->iv, 8);
desc.info = iv;
desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
return ccm_nx_encrypt(req, &desc);
return ccm_nx_encrypt(req, &desc, req->assoclen - 8);
}
static int ccm_aes_nx_encrypt(struct aead_request *req)
@@ -514,13 +511,12 @@ static int ccm_aes_nx_encrypt(struct aead_request *req)
int rc;
desc.info = req->iv;
desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
rc = crypto_ccm_check_iv(desc.info);
if (rc)
return rc;
return ccm_nx_encrypt(req, &desc);
return ccm_nx_encrypt(req, &desc, req->assoclen);
}
static int ccm4309_aes_nx_decrypt(struct aead_request *req)
@@ -535,9 +531,8 @@ static int ccm4309_aes_nx_decrypt(struct aead_request *req)
memcpy(iv + 4, req->iv, 8);
desc.info = iv;
desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
return ccm_nx_decrypt(req, &desc);
return ccm_nx_decrypt(req, &desc, req->assoclen - 8);
}
static int ccm_aes_nx_decrypt(struct aead_request *req)
@@ -546,13 +541,12 @@ static int ccm_aes_nx_decrypt(struct aead_request *req)
int rc;
desc.info = req->iv;
desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
rc = crypto_ccm_check_iv(desc.info);
if (rc)
return rc;
return ccm_nx_decrypt(req, &desc);
return ccm_nx_decrypt(req, &desc, req->assoclen);
}
/* tell the block cipher walk routines that this is a stream cipher by
@@ -560,47 +554,42 @@ static int ccm_aes_nx_decrypt(struct aead_request *req)
* during encrypt/decrypt doesn't solve this problem, because it calls
* blkcipher_walk_done under the covers, which doesn't use walk->blocksize,
* but instead uses this tfm->blocksize. */
struct crypto_alg nx_ccm_aes_alg = {
.cra_name = "ccm(aes)",
.cra_driver_name = "ccm-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AEAD |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_aead_type,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_ccm_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_aead = {
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = AES_BLOCK_SIZE,
.setkey = ccm_aes_nx_set_key,
.setauthsize = ccm_aes_nx_setauthsize,
.encrypt = ccm_aes_nx_encrypt,
.decrypt = ccm_aes_nx_decrypt,
}
struct aead_alg nx_ccm_aes_alg = {
.base = {
.cra_name = "ccm(aes)",
.cra_driver_name = "ccm-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_module = THIS_MODULE,
},
.init = nx_crypto_ctx_aes_ccm_init,
.exit = nx_crypto_ctx_aead_exit,
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = AES_BLOCK_SIZE,
.setkey = ccm_aes_nx_set_key,
.setauthsize = ccm_aes_nx_setauthsize,
.encrypt = ccm_aes_nx_encrypt,
.decrypt = ccm_aes_nx_decrypt,
};
struct crypto_alg nx_ccm4309_aes_alg = {
.cra_name = "rfc4309(ccm(aes))",
.cra_driver_name = "rfc4309-ccm-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AEAD |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_nivaead_type,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_ccm_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_aead = {
.ivsize = 8,
.maxauthsize = AES_BLOCK_SIZE,
.setkey = ccm4309_aes_nx_set_key,
.setauthsize = ccm4309_aes_nx_setauthsize,
.encrypt = ccm4309_aes_nx_encrypt,
.decrypt = ccm4309_aes_nx_decrypt,
.geniv = "seqiv",
}
struct aead_alg nx_ccm4309_aes_alg = {
.base = {
.cra_name = "rfc4309(ccm(aes))",
.cra_driver_name = "rfc4309-ccm-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_module = THIS_MODULE,
},
.init = nx_crypto_ctx_aes_ccm_init,
.exit = nx_crypto_ctx_aead_exit,
.ivsize = 8,
.maxauthsize = AES_BLOCK_SIZE,
.setkey = ccm4309_aes_nx_set_key,
.setauthsize = ccm4309_aes_nx_setauthsize,
.encrypt = ccm4309_aes_nx_encrypt,
.decrypt = ccm4309_aes_nx_decrypt,
};

21
drivers/crypto/nx/nx-aes-ctr.c
View File

@@ -144,27 +144,6 @@ static int ctr3686_aes_nx_crypt(struct blkcipher_desc *desc,
return ctr_aes_nx_crypt(desc, dst, src, nbytes);
}
struct crypto_alg nx_ctr_aes_alg = {
.cra_name = "ctr(aes)",
.cra_driver_name = "ctr-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_ctr_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_blkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = ctr_aes_nx_set_key,
.encrypt = ctr_aes_nx_crypt,
.decrypt = ctr_aes_nx_crypt,
}
};
struct crypto_alg nx_ctr3686_aes_alg = {
.cra_name = "rfc3686(ctr(aes))",
.cra_driver_name = "rfc3686-ctr-aes-nx",

64
drivers/crypto/nx/nx-aes-gcm.c
View File

@@ -21,11 +21,9 @@
#include <crypto/internal/aead.h>
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
@@ -36,7 +34,7 @@ static int gcm_aes_nx_set_key(struct crypto_aead *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
@@ -75,7 +73,7 @@ static int gcm4106_aes_nx_set_key(struct crypto_aead *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm);
char *nonce = nx_ctx->priv.gcm.nonce;
int rc;
@@ -110,13 +108,14 @@ static int gcm4106_aes_nx_setauthsize(struct crypto_aead *tfm,
static int nx_gca(struct nx_crypto_ctx *nx_ctx,
struct aead_request *req,
u8 *out)
u8 *out,
unsigned int assoclen)
{
int rc;
struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
struct scatter_walk walk;
struct nx_sg *nx_sg = nx_ctx->in_sg;
unsigned int nbytes = req->assoclen;
unsigned int nbytes = assoclen;
unsigned int processed = 0, to_process;
unsigned int max_sg_len;
@@ -167,7 +166,7 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx,
NX_CPB_FDM(csbcpb_aead) |= NX_FDM_CONTINUATION;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
atomic64_add(assoclen, &(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < nbytes);
@@ -177,13 +176,15 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx,
return rc;
}
static int gmac(struct aead_request *req, struct blkcipher_desc *desc)
static int gmac(struct aead_request *req, struct blkcipher_desc *desc,
unsigned int assoclen)
{
int rc;
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_crypto_ctx *nx_ctx =
crypto_aead_ctx(crypto_aead_reqtfm(req));
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_sg *nx_sg;
unsigned int nbytes = req->assoclen;
unsigned int nbytes = assoclen;
unsigned int processed = 0, to_process;
unsigned int max_sg_len;
@@ -238,7 +239,7 @@ static int gmac(struct aead_request *req, struct blkcipher_desc *desc)
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
atomic64_add(assoclen, &(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < nbytes);
@@ -253,7 +254,8 @@ static int gcm_empty(struct aead_request *req, struct blkcipher_desc *desc,
int enc)
{
int rc;
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_crypto_ctx *nx_ctx =
crypto_aead_ctx(crypto_aead_reqtfm(req));
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
char out[AES_BLOCK_SIZE];
struct nx_sg *in_sg, *out_sg;
@@ -314,9 +316,11 @@ out:
return rc;
}
static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
static int gcm_aes_nx_crypt(struct aead_request *req, int enc,
unsigned int assoclen)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_crypto_ctx *nx_ctx =
crypto_aead_ctx(crypto_aead_reqtfm(req));
struct nx_gcm_rctx *rctx = aead_request_ctx(req);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct blkcipher_desc desc;
@@ -332,10 +336,10 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
*(u32 *)(desc.info + NX_GCM_CTR_OFFSET) = 1;
if (nbytes == 0) {
if (req->assoclen == 0)
if (assoclen == 0)
rc = gcm_empty(req, &desc, enc);
else
rc = gmac(req, &desc);
rc = gmac(req, &desc, assoclen);
if (rc)
goto out;
else
@@ -343,9 +347,10 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
}
/* Process associated data */
csbcpb->cpb.aes_gcm.bit_length_aad = req->assoclen * 8;
if (req->assoclen) {
rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad);
csbcpb->cpb.aes_gcm.bit_length_aad = assoclen * 8;
if (assoclen) {
rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad,
assoclen);
if (rc)
goto out;
}
@@ -363,7 +368,6 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
to_process = nbytes - processed;
csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8;
desc.tfm = (struct crypto_blkcipher *) req->base.tfm;
rc = nx_build_sg_lists(nx_ctx, &desc, req->dst,
req->src, &to_process,
processed + req->assoclen,
@@ -430,7 +434,7 @@ static int gcm_aes_nx_encrypt(struct aead_request *req)
memcpy(iv, req->iv, 12);
return gcm_aes_nx_crypt(req, 1);
return gcm_aes_nx_crypt(req, 1, req->assoclen);
}
static int gcm_aes_nx_decrypt(struct aead_request *req)
@@ -440,12 +444,13 @@ static int gcm_aes_nx_decrypt(struct aead_request *req)
memcpy(iv, req->iv, 12);
return gcm_aes_nx_crypt(req, 0);
return gcm_aes_nx_crypt(req, 0, req->assoclen);
}
static int gcm4106_aes_nx_encrypt(struct aead_request *req)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_crypto_ctx *nx_ctx =
crypto_aead_ctx(crypto_aead_reqtfm(req));
struct nx_gcm_rctx *rctx = aead_request_ctx(req);
char *iv = rctx->iv;
char *nonce = nx_ctx->priv.gcm.nonce;
@@ -453,12 +458,16 @@ static int gcm4106_aes_nx_encrypt(struct aead_request *req)
memcpy(iv, nonce, NX_GCM4106_NONCE_LEN);
memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8);
return gcm_aes_nx_crypt(req, 1);
if (req->assoclen < 8)
return -EINVAL;
return gcm_aes_nx_crypt(req, 1, req->assoclen - 8);
}
static int gcm4106_aes_nx_decrypt(struct aead_request *req)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_crypto_ctx *nx_ctx =
crypto_aead_ctx(crypto_aead_reqtfm(req));
struct nx_gcm_rctx *rctx = aead_request_ctx(req);
char *iv = rctx->iv;
char *nonce = nx_ctx->priv.gcm.nonce;
@@ -466,7 +475,10 @@ static int gcm4106_aes_nx_decrypt(struct aead_request *req)
memcpy(iv, nonce, NX_GCM4106_NONCE_LEN);
memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8);
return gcm_aes_nx_crypt(req, 0);
if (req->assoclen < 8)
return -EINVAL;
return gcm_aes_nx_crypt(req, 0, req->assoclen - 8);
}
/* tell the block cipher walk routines that this is a stream cipher by

30
drivers/crypto/nx/nx.c
View File

@@ -596,13 +596,9 @@ static int nx_register_algs(void)
if (rc)
goto out_unreg_ecb;
rc = nx_register_alg(&nx_ctr_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
if (rc)
goto out_unreg_cbc;
rc = nx_register_alg(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
if (rc)
goto out_unreg_ctr;
goto out_unreg_cbc;
rc = nx_register_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
if (rc)
@@ -612,11 +608,11 @@ static int nx_register_algs(void)
if (rc)
goto out_unreg_gcm;
rc = nx_register_alg(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
rc = nx_register_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
if (rc)
goto out_unreg_gcm4106;
rc = nx_register_alg(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
rc = nx_register_aead(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
if (rc)
goto out_unreg_ccm;
@@ -644,17 +640,15 @@ out_unreg_s256:
nx_unregister_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA,
NX_PROPS_SHA256);
out_unreg_ccm4309:
nx_unregister_alg(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
nx_unregister_aead(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
out_unreg_ccm:
nx_unregister_alg(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
nx_unregister_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
out_unreg_gcm4106:
nx_unregister_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
out_unreg_gcm:
nx_unregister_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
out_unreg_ctr3686:
nx_unregister_alg(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
out_unreg_ctr:
nx_unregister_alg(&nx_ctr_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
out_unreg_cbc:
nx_unregister_alg(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC);
out_unreg_ecb:
@@ -711,11 +705,10 @@ static int nx_crypto_ctx_init(struct nx_crypto_ctx *nx_ctx, u32 fc, u32 mode)
}
/* entry points from the crypto tfm initializers */
int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm)
int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm)
{
crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
sizeof(struct nx_ccm_rctx));
return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
crypto_aead_set_reqsize(tfm, sizeof(struct nx_ccm_rctx));
return nx_crypto_ctx_init(crypto_aead_ctx(tfm), NX_FC_AES,
NX_MODE_AES_CCM);
}
@@ -813,16 +806,15 @@ static int nx_remove(struct vio_dev *viodev)
NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA256);
nx_unregister_shash(&nx_shash_sha256_alg,
NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA512);
nx_unregister_alg(&nx_ccm4309_aes_alg,
NX_FC_AES, NX_MODE_AES_CCM);
nx_unregister_alg(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
nx_unregister_aead(&nx_ccm4309_aes_alg,
NX_FC_AES, NX_MODE_AES_CCM);
nx_unregister_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
nx_unregister_aead(&nx_gcm4106_aes_alg,
NX_FC_AES, NX_MODE_AES_GCM);
nx_unregister_aead(&nx_gcm_aes_alg,
NX_FC_AES, NX_MODE_AES_GCM);
nx_unregister_alg(&nx_ctr3686_aes_alg,
NX_FC_AES, NX_MODE_AES_CTR);
nx_unregister_alg(&nx_ctr_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
nx_unregister_alg(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC);
nx_unregister_alg(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB);
}

9
drivers/crypto/nx/nx.h
View File

@@ -149,8 +149,10 @@ struct nx_crypto_ctx {
} priv;
};
struct crypto_aead;
/* prototypes */
int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm);
int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm);
int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_aes_ctr_init(struct crypto_tfm *tfm);
@@ -187,10 +189,9 @@ extern struct crypto_alg nx_cbc_aes_alg;
extern struct crypto_alg nx_ecb_aes_alg;
extern struct aead_alg nx_gcm_aes_alg;
extern struct aead_alg nx_gcm4106_aes_alg;
extern struct crypto_alg nx_ctr_aes_alg;
extern struct crypto_alg nx_ctr3686_aes_alg;
extern struct crypto_alg nx_ccm_aes_alg;
extern struct crypto_alg nx_ccm4309_aes_alg;
extern struct aead_alg nx_ccm_aes_alg;
extern struct aead_alg nx_ccm4309_aes_alg;
extern struct shash_alg nx_shash_aes_xcbc_alg;
extern struct shash_alg nx_shash_sha512_alg;
extern struct shash_alg nx_shash_sha256_alg;