From ca81a1a1b8d79dd6706c9463a81e9491e940ca2b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 26 Feb 2013 17:52:15 +0800 Subject: crypto: crc32c - Kill pointless CRYPTO_CRC32C_X86_64 option This bool option can never be set to anything other than y. So let's just kill it. Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 63947a8..ca96761 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -52,5 +52,5 @@ aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o crc32c-intel-y := crc32c-intel_glue.o -crc32c-intel-$(CONFIG_CRYPTO_CRC32C_X86_64) += crc32c-pcl-intel-asm_64.o +crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o diff --git a/crypto/Kconfig b/crypto/Kconfig index 05c0ce5..aed52b2 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -322,19 +322,9 @@ config CRYPTO_CRC32C by iSCSI for header and data digests and by others. See Castagnoli93. Module will be crc32c. -config CRYPTO_CRC32C_X86_64 - bool - depends on X86 && 64BIT - select CRYPTO_HASH - help - In Intel processor with SSE4.2 supported, the processor will - support CRC32C calculation using hardware accelerated CRC32 - instruction optimized with PCLMULQDQ instruction when available. - config CRYPTO_CRC32C_INTEL tristate "CRC32c INTEL hardware acceleration" depends on X86 - select CRYPTO_CRC32C_X86_64 if 64BIT select CRYPTO_HASH help In Intel processor with SSE4.2 supported, the processor will -- cgit v0.10.2 From 6150f3bc0b4f94f0eea3e32b4e7462025e4bd972 Mon Sep 17 00:00:00 2001 From: Nicolas Royer Date: Wed, 20 Feb 2013 17:10:23 +0100 Subject: ARM: AT91SAM9G45: same platform data structure for all crypto peripherals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only AES use DMA in AT91SAM9G45 (TDES and SHA use PDC). However latest Atmel TDES and SHA IP releases use DMA instead of PDC. --> Atmel TDES and SHA drivers need DMA platform data for those IP releases. Goal of this patch is to use the same platform data structure for all Atmel crypto peripherals. This structure contains information about DMA interface. Signed-off-by: Nicolas Royer Acked-by: Nicolas Ferre Acked-by: Eric Bénard Tested-by: Eric Bénard Signed-off-by: Herbert Xu diff --git a/arch/arm/mach-at91/at91sam9g45_devices.c b/arch/arm/mach-at91/at91sam9g45_devices.c index 827c9f2..f0bf682 100644 --- a/arch/arm/mach-at91/at91sam9g45_devices.c +++ b/arch/arm/mach-at91/at91sam9g45_devices.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include @@ -1900,7 +1900,8 @@ static void __init at91_add_device_tdes(void) {} * -------------------------------------------------------------------- */ #if defined(CONFIG_CRYPTO_DEV_ATMEL_AES) || defined(CONFIG_CRYPTO_DEV_ATMEL_AES_MODULE) -static struct aes_platform_data aes_data; +static struct crypto_platform_data aes_data; +static struct crypto_dma_data alt_atslave; static u64 aes_dmamask = DMA_BIT_MASK(32); static struct resource aes_resources[] = { @@ -1931,23 +1932,20 @@ static struct platform_device at91sam9g45_aes_device = { static void __init at91_add_device_aes(void) { struct at_dma_slave *atslave; - struct aes_dma_data *alt_atslave; - - alt_atslave = kzalloc(sizeof(struct aes_dma_data), GFP_KERNEL); /* DMA TX slave channel configuration */ - atslave = &alt_atslave->txdata; + atslave = &alt_atslave.txdata; atslave->dma_dev = &at_hdmac_device.dev; atslave->cfg = ATC_FIFOCFG_ENOUGHSPACE | ATC_SRC_H2SEL_HW | ATC_SRC_PER(AT_DMA_ID_AES_RX); /* DMA RX slave channel configuration */ - atslave = &alt_atslave->rxdata; + atslave = &alt_atslave.rxdata; atslave->dma_dev = &at_hdmac_device.dev; atslave->cfg = ATC_FIFOCFG_ENOUGHSPACE | ATC_DST_H2SEL_HW | ATC_DST_PER(AT_DMA_ID_AES_TX); - aes_data.dma_slave = alt_atslave; + aes_data.dma_slave = &alt_atslave; platform_device_register(&at91sam9g45_aes_device); } #else diff --git a/include/linux/platform_data/atmel-aes.h b/include/linux/platform_data/atmel-aes.h deleted file mode 100644 index ab68082..0000000 --- a/include/linux/platform_data/atmel-aes.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef __LINUX_ATMEL_AES_H -#define __LINUX_ATMEL_AES_H - -#include - -/** - * struct aes_dma_data - DMA data for AES - */ -struct aes_dma_data { - struct at_dma_slave txdata; - struct at_dma_slave rxdata; -}; - -/** - * struct aes_platform_data - board-specific AES configuration - * @dma_slave: DMA slave interface to use in data transfers. - */ -struct aes_platform_data { - struct aes_dma_data *dma_slave; -}; - -#endif /* __LINUX_ATMEL_AES_H */ diff --git a/include/linux/platform_data/crypto-atmel.h b/include/linux/platform_data/crypto-atmel.h new file mode 100644 index 0000000..b46e0d9 --- /dev/null +++ b/include/linux/platform_data/crypto-atmel.h @@ -0,0 +1,22 @@ +#ifndef __LINUX_CRYPTO_ATMEL_H +#define __LINUX_CRYPTO_ATMEL_H + +#include + +/** + * struct crypto_dma_data - DMA data for AES/TDES/SHA + */ +struct crypto_dma_data { + struct at_dma_slave txdata; + struct at_dma_slave rxdata; +}; + +/** + * struct crypto_platform_data - board-specific AES/TDES/SHA configuration + * @dma_slave: DMA slave interface to use in data transfers. + */ +struct crypto_platform_data { + struct crypto_dma_data *dma_slave; +}; + +#endif /* __LINUX_CRYPTO_ATMEL_H */ -- cgit v0.10.2 From cadc4ab8f6f73719ef0e124320cdd210d1c9ff3e Mon Sep 17 00:00:00 2001 From: Nicolas Royer Date: Wed, 20 Feb 2013 17:10:24 +0100 Subject: crypto: atmel-aes - add support for latest release of the IP (0x130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates from previous IP release (0x120): - add cfb64 support - add DMA double input buffer support Signed-off-by: Nicolas Royer Acked-by: Nicolas Ferre Acked-by: Eric Bénard Tested-by: Eric Bénard Signed-off-by: Herbert Xu diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 6f22ba5..c1efd91 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -38,7 +38,7 @@ #include #include #include -#include +#include #include "atmel-aes-regs.h" #define CFB8_BLOCK_SIZE 1 @@ -47,7 +47,7 @@ #define CFB64_BLOCK_SIZE 8 /* AES flags */ -#define AES_FLAGS_MODE_MASK 0x01ff +#define AES_FLAGS_MODE_MASK 0x03ff #define AES_FLAGS_ENCRYPT BIT(0) #define AES_FLAGS_CBC BIT(1) #define AES_FLAGS_CFB BIT(2) @@ -55,21 +55,26 @@ #define AES_FLAGS_CFB16 BIT(4) #define AES_FLAGS_CFB32 BIT(5) #define AES_FLAGS_CFB64 BIT(6) -#define AES_FLAGS_OFB BIT(7) -#define AES_FLAGS_CTR BIT(8) +#define AES_FLAGS_CFB128 BIT(7) +#define AES_FLAGS_OFB BIT(8) +#define AES_FLAGS_CTR BIT(9) #define AES_FLAGS_INIT BIT(16) #define AES_FLAGS_DMA BIT(17) #define AES_FLAGS_BUSY BIT(18) +#define AES_FLAGS_FAST BIT(19) -#define AES_FLAGS_DUALBUFF BIT(24) - -#define ATMEL_AES_QUEUE_LENGTH 1 -#define ATMEL_AES_CACHE_SIZE 0 +#define ATMEL_AES_QUEUE_LENGTH 50 #define ATMEL_AES_DMA_THRESHOLD 16 +struct atmel_aes_caps { + bool has_dualbuff; + bool has_cfb64; + u32 max_burst_size; +}; + struct atmel_aes_dev; struct atmel_aes_ctx { @@ -77,6 +82,8 @@ struct atmel_aes_ctx { int keylen; u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + + u16 block_size; }; struct atmel_aes_reqctx { @@ -112,20 +119,27 @@ struct atmel_aes_dev { struct scatterlist *in_sg; unsigned int nb_in_sg; - + size_t in_offset; struct scatterlist *out_sg; unsigned int nb_out_sg; + size_t out_offset; size_t bufcnt; + size_t buflen; + size_t dma_size; - u8 buf_in[ATMEL_AES_DMA_THRESHOLD] __aligned(sizeof(u32)); - int dma_in; + void *buf_in; + int dma_in; + dma_addr_t dma_addr_in; struct atmel_aes_dma dma_lch_in; - u8 buf_out[ATMEL_AES_DMA_THRESHOLD] __aligned(sizeof(u32)); - int dma_out; + void *buf_out; + int dma_out; + dma_addr_t dma_addr_out; struct atmel_aes_dma dma_lch_out; + struct atmel_aes_caps caps; + u32 hw_version; }; @@ -165,6 +179,37 @@ static int atmel_aes_sg_length(struct ablkcipher_request *req, return sg_nb; } +static int atmel_aes_sg_copy(struct scatterlist **sg, size_t *offset, + void *buf, size_t buflen, size_t total, int out) +{ + unsigned int count, off = 0; + + while (buflen && total) { + count = min((*sg)->length - *offset, total); + count = min(count, buflen); + + if (!count) + return off; + + scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out); + + off += count; + buflen -= count; + *offset += count; + total -= count; + + if (*offset == (*sg)->length) { + *sg = sg_next(*sg); + if (*sg) + *offset = 0; + else + total = 0; + } + } + + return off; +} + static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset) { return readl_relaxed(dd->io_base + offset); @@ -190,14 +235,6 @@ static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset, atmel_aes_write(dd, offset, *value); } -static void atmel_aes_dualbuff_test(struct atmel_aes_dev *dd) -{ - atmel_aes_write(dd, AES_MR, AES_MR_DUALBUFF); - - if (atmel_aes_read(dd, AES_MR) & AES_MR_DUALBUFF) - dd->flags |= AES_FLAGS_DUALBUFF; -} - static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_ctx *ctx) { struct atmel_aes_dev *aes_dd = NULL; @@ -225,7 +262,7 @@ static int atmel_aes_hw_init(struct atmel_aes_dev *dd) if (!(dd->flags & AES_FLAGS_INIT)) { atmel_aes_write(dd, AES_CR, AES_CR_SWRST); - atmel_aes_dualbuff_test(dd); + atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); dd->flags |= AES_FLAGS_INIT; dd->err = 0; } @@ -233,11 +270,19 @@ static int atmel_aes_hw_init(struct atmel_aes_dev *dd) return 0; } +static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd) +{ + return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff; +} + static void atmel_aes_hw_version_init(struct atmel_aes_dev *dd) { atmel_aes_hw_init(dd); - dd->hw_version = atmel_aes_read(dd, AES_HW_VERSION); + dd->hw_version = atmel_aes_get_version(dd); + + dev_info(dd->dev, + "version: 0x%x\n", dd->hw_version); clk_disable_unprepare(dd->iclk); } @@ -260,50 +305,77 @@ static void atmel_aes_dma_callback(void *data) tasklet_schedule(&dd->done_task); } -static int atmel_aes_crypt_dma(struct atmel_aes_dev *dd) +static int atmel_aes_crypt_dma(struct atmel_aes_dev *dd, + dma_addr_t dma_addr_in, dma_addr_t dma_addr_out, int length) { + struct scatterlist sg[2]; struct dma_async_tx_descriptor *in_desc, *out_desc; - int nb_dma_sg_in, nb_dma_sg_out; - dd->nb_in_sg = atmel_aes_sg_length(dd->req, dd->in_sg); - if (!dd->nb_in_sg) - goto exit_err; + dd->dma_size = length; - nb_dma_sg_in = dma_map_sg(dd->dev, dd->in_sg, dd->nb_in_sg, - DMA_TO_DEVICE); - if (!nb_dma_sg_in) - goto exit_err; + if (!(dd->flags & AES_FLAGS_FAST)) { + dma_sync_single_for_device(dd->dev, dma_addr_in, length, + DMA_TO_DEVICE); + } - in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, dd->in_sg, - nb_dma_sg_in, DMA_MEM_TO_DEV, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (dd->flags & AES_FLAGS_CFB8) { + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_1_BYTE; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_1_BYTE; + } else if (dd->flags & AES_FLAGS_CFB16) { + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_2_BYTES; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_2_BYTES; + } else { + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + } - if (!in_desc) - goto unmap_in; + if (dd->flags & (AES_FLAGS_CFB8 | AES_FLAGS_CFB16 | + AES_FLAGS_CFB32 | AES_FLAGS_CFB64)) { + dd->dma_lch_in.dma_conf.src_maxburst = 1; + dd->dma_lch_in.dma_conf.dst_maxburst = 1; + dd->dma_lch_out.dma_conf.src_maxburst = 1; + dd->dma_lch_out.dma_conf.dst_maxburst = 1; + } else { + dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size; + dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size; + dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size; + dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size; + } - /* callback not needed */ + dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); + dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf); - dd->nb_out_sg = atmel_aes_sg_length(dd->req, dd->out_sg); - if (!dd->nb_out_sg) - goto unmap_in; + dd->flags |= AES_FLAGS_DMA; - nb_dma_sg_out = dma_map_sg(dd->dev, dd->out_sg, dd->nb_out_sg, - DMA_FROM_DEVICE); - if (!nb_dma_sg_out) - goto unmap_out; + sg_init_table(&sg[0], 1); + sg_dma_address(&sg[0]) = dma_addr_in; + sg_dma_len(&sg[0]) = length; - out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, dd->out_sg, - nb_dma_sg_out, DMA_DEV_TO_MEM, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + sg_init_table(&sg[1], 1); + sg_dma_address(&sg[1]) = dma_addr_out; + sg_dma_len(&sg[1]) = length; + + in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0], + 1, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!in_desc) + return -EINVAL; + out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1], + 1, DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!out_desc) - goto unmap_out; + return -EINVAL; out_desc->callback = atmel_aes_dma_callback; out_desc->callback_param = dd; - dd->total -= dd->req->nbytes; - dmaengine_submit(out_desc); dma_async_issue_pending(dd->dma_lch_out.chan); @@ -311,15 +383,6 @@ static int atmel_aes_crypt_dma(struct atmel_aes_dev *dd) dma_async_issue_pending(dd->dma_lch_in.chan); return 0; - -unmap_out: - dma_unmap_sg(dd->dev, dd->out_sg, dd->nb_out_sg, - DMA_FROM_DEVICE); -unmap_in: - dma_unmap_sg(dd->dev, dd->in_sg, dd->nb_in_sg, - DMA_TO_DEVICE); -exit_err: - return -EINVAL; } static int atmel_aes_crypt_cpu_start(struct atmel_aes_dev *dd) @@ -352,30 +415,66 @@ static int atmel_aes_crypt_cpu_start(struct atmel_aes_dev *dd) static int atmel_aes_crypt_dma_start(struct atmel_aes_dev *dd) { - int err; + int err, fast = 0, in, out; + size_t count; + dma_addr_t addr_in, addr_out; + + if ((!dd->in_offset) && (!dd->out_offset)) { + /* check for alignment */ + in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) && + IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size); + out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) && + IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size); + fast = in && out; + + if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg)) + fast = 0; + } + + + if (fast) { + count = min(dd->total, sg_dma_len(dd->in_sg)); + count = min(count, sg_dma_len(dd->out_sg)); + + err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + + err = dma_map_sg(dd->dev, dd->out_sg, 1, + DMA_FROM_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + dma_unmap_sg(dd->dev, dd->in_sg, 1, + DMA_TO_DEVICE); + return -EINVAL; + } + + addr_in = sg_dma_address(dd->in_sg); + addr_out = sg_dma_address(dd->out_sg); + + dd->flags |= AES_FLAGS_FAST; - if (dd->flags & AES_FLAGS_CFB8) { - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_1_BYTE; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_1_BYTE; - } else if (dd->flags & AES_FLAGS_CFB16) { - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_2_BYTES; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_2_BYTES; } else { - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; + /* use cache buffers */ + count = atmel_aes_sg_copy(&dd->in_sg, &dd->in_offset, + dd->buf_in, dd->buflen, dd->total, 0); + + addr_in = dd->dma_addr_in; + addr_out = dd->dma_addr_out; + + dd->flags &= ~AES_FLAGS_FAST; } - dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); - dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf); + dd->total -= count; - dd->flags |= AES_FLAGS_DMA; - err = atmel_aes_crypt_dma(dd); + err = atmel_aes_crypt_dma(dd, addr_in, addr_out, count); + + if (err && (dd->flags & AES_FLAGS_FAST)) { + dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); + } return err; } @@ -410,6 +509,8 @@ static int atmel_aes_write_ctrl(struct atmel_aes_dev *dd) valmr |= AES_MR_CFBS_32b; else if (dd->flags & AES_FLAGS_CFB64) valmr |= AES_MR_CFBS_64b; + else if (dd->flags & AES_FLAGS_CFB128) + valmr |= AES_MR_CFBS_128b; } else if (dd->flags & AES_FLAGS_OFB) { valmr |= AES_MR_OPMOD_OFB; } else if (dd->flags & AES_FLAGS_CTR) { @@ -423,7 +524,7 @@ static int atmel_aes_write_ctrl(struct atmel_aes_dev *dd) if (dd->total > ATMEL_AES_DMA_THRESHOLD) { valmr |= AES_MR_SMOD_IDATAR0; - if (dd->flags & AES_FLAGS_DUALBUFF) + if (dd->caps.has_dualbuff) valmr |= AES_MR_DUALBUFF; } else { valmr |= AES_MR_SMOD_AUTO; @@ -477,7 +578,9 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, /* assign new request to device */ dd->req = req; dd->total = req->nbytes; + dd->in_offset = 0; dd->in_sg = req->src; + dd->out_offset = 0; dd->out_sg = req->dst; rctx = ablkcipher_request_ctx(req); @@ -506,18 +609,86 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, static int atmel_aes_crypt_dma_stop(struct atmel_aes_dev *dd) { int err = -EINVAL; + size_t count; if (dd->flags & AES_FLAGS_DMA) { - dma_unmap_sg(dd->dev, dd->out_sg, - dd->nb_out_sg, DMA_FROM_DEVICE); - dma_unmap_sg(dd->dev, dd->in_sg, - dd->nb_in_sg, DMA_TO_DEVICE); err = 0; + if (dd->flags & AES_FLAGS_FAST) { + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); + } else { + dma_sync_single_for_device(dd->dev, dd->dma_addr_out, + dd->dma_size, DMA_FROM_DEVICE); + + /* copy data */ + count = atmel_aes_sg_copy(&dd->out_sg, &dd->out_offset, + dd->buf_out, dd->buflen, dd->dma_size, 1); + if (count != dd->dma_size) { + err = -EINVAL; + pr_err("not all data converted: %u\n", count); + } + } } return err; } + +static int atmel_aes_buff_init(struct atmel_aes_dev *dd) +{ + int err = -ENOMEM; + + dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0); + dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0); + dd->buflen = PAGE_SIZE; + dd->buflen &= ~(AES_BLOCK_SIZE - 1); + + if (!dd->buf_in || !dd->buf_out) { + dev_err(dd->dev, "unable to alloc pages.\n"); + goto err_alloc; + } + + /* MAP here */ + dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, + dd->buflen, DMA_TO_DEVICE); + if (dma_mapping_error(dd->dev, dd->dma_addr_in)) { + dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); + err = -EINVAL; + goto err_map_in; + } + + dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, + dd->buflen, DMA_FROM_DEVICE); + if (dma_mapping_error(dd->dev, dd->dma_addr_out)) { + dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); + err = -EINVAL; + goto err_map_out; + } + + return 0; + +err_map_out: + dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, + DMA_TO_DEVICE); +err_map_in: + free_page((unsigned long)dd->buf_out); + free_page((unsigned long)dd->buf_in); +err_alloc: + if (err) + pr_err("error: %d\n", err); + return err; +} + +static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) +{ + dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, + DMA_FROM_DEVICE); + dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, + DMA_TO_DEVICE); + free_page((unsigned long)dd->buf_out); + free_page((unsigned long)dd->buf_in); +} + static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) { struct atmel_aes_ctx *ctx = crypto_ablkcipher_ctx( @@ -525,9 +696,30 @@ static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); struct atmel_aes_dev *dd; - if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { - pr_err("request size is not exact amount of AES blocks\n"); - return -EINVAL; + if (mode & AES_FLAGS_CFB8) { + if (!IS_ALIGNED(req->nbytes, CFB8_BLOCK_SIZE)) { + pr_err("request size is not exact amount of CFB8 blocks\n"); + return -EINVAL; + } + ctx->block_size = CFB8_BLOCK_SIZE; + } else if (mode & AES_FLAGS_CFB16) { + if (!IS_ALIGNED(req->nbytes, CFB16_BLOCK_SIZE)) { + pr_err("request size is not exact amount of CFB16 blocks\n"); + return -EINVAL; + } + ctx->block_size = CFB16_BLOCK_SIZE; + } else if (mode & AES_FLAGS_CFB32) { + if (!IS_ALIGNED(req->nbytes, CFB32_BLOCK_SIZE)) { + pr_err("request size is not exact amount of CFB32 blocks\n"); + return -EINVAL; + } + ctx->block_size = CFB32_BLOCK_SIZE; + } else { + if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { + pr_err("request size is not exact amount of AES blocks\n"); + return -EINVAL; + } + ctx->block_size = AES_BLOCK_SIZE; } dd = atmel_aes_find_dev(ctx); @@ -551,14 +743,12 @@ static bool atmel_aes_filter(struct dma_chan *chan, void *slave) } } -static int atmel_aes_dma_init(struct atmel_aes_dev *dd) +static int atmel_aes_dma_init(struct atmel_aes_dev *dd, + struct crypto_platform_data *pdata) { int err = -ENOMEM; - struct aes_platform_data *pdata; dma_cap_mask_t mask_in, mask_out; - pdata = dd->dev->platform_data; - if (pdata && pdata->dma_slave->txdata.dma_dev && pdata->dma_slave->rxdata.dma_dev) { @@ -568,28 +758,38 @@ static int atmel_aes_dma_init(struct atmel_aes_dev *dd) dd->dma_lch_in.chan = dma_request_channel(mask_in, atmel_aes_filter, &pdata->dma_slave->rxdata); + if (!dd->dma_lch_in.chan) goto err_dma_in; dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + AES_IDATAR(0); - dd->dma_lch_in.dma_conf.src_maxburst = 1; - dd->dma_lch_in.dma_conf.dst_maxburst = 1; + dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; dd->dma_lch_in.dma_conf.device_fc = false; dma_cap_zero(mask_out); dma_cap_set(DMA_SLAVE, mask_out); dd->dma_lch_out.chan = dma_request_channel(mask_out, atmel_aes_filter, &pdata->dma_slave->txdata); + if (!dd->dma_lch_out.chan) goto err_dma_out; dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + AES_ODATAR(0); - dd->dma_lch_out.dma_conf.src_maxburst = 1; - dd->dma_lch_out.dma_conf.dst_maxburst = 1; + dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size; + dd->dma_lch_out.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; dd->dma_lch_out.dma_conf.device_fc = false; return 0; @@ -665,13 +865,13 @@ static int atmel_aes_ofb_decrypt(struct ablkcipher_request *req) static int atmel_aes_cfb_encrypt(struct ablkcipher_request *req) { return atmel_aes_crypt(req, - AES_FLAGS_ENCRYPT | AES_FLAGS_CFB); + AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB128); } static int atmel_aes_cfb_decrypt(struct ablkcipher_request *req) { return atmel_aes_crypt(req, - AES_FLAGS_CFB); + AES_FLAGS_CFB | AES_FLAGS_CFB128); } static int atmel_aes_cfb64_encrypt(struct ablkcipher_request *req) @@ -753,7 +953,7 @@ static struct crypto_alg aes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), - .cra_alignmask = 0x0, + .cra_alignmask = 0xf, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, @@ -773,7 +973,7 @@ static struct crypto_alg aes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), - .cra_alignmask = 0x0, + .cra_alignmask = 0xf, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, @@ -794,7 +994,7 @@ static struct crypto_alg aes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), - .cra_alignmask = 0x0, + .cra_alignmask = 0xf, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, @@ -815,7 +1015,7 @@ static struct crypto_alg aes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), - .cra_alignmask = 0x0, + .cra_alignmask = 0xf, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, @@ -836,7 +1036,7 @@ static struct crypto_alg aes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB32_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), - .cra_alignmask = 0x0, + .cra_alignmask = 0x3, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, @@ -857,7 +1057,7 @@ static struct crypto_alg aes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB16_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), - .cra_alignmask = 0x0, + .cra_alignmask = 0x1, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, @@ -899,7 +1099,7 @@ static struct crypto_alg aes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), - .cra_alignmask = 0x0, + .cra_alignmask = 0xf, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, @@ -915,15 +1115,14 @@ static struct crypto_alg aes_algs[] = { }, }; -static struct crypto_alg aes_cfb64_alg[] = { -{ +static struct crypto_alg aes_cfb64_alg = { .cra_name = "cfb64(aes)", .cra_driver_name = "atmel-cfb64-aes", .cra_priority = 100, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB64_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), - .cra_alignmask = 0x0, + .cra_alignmask = 0x7, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, @@ -936,7 +1135,6 @@ static struct crypto_alg aes_cfb64_alg[] = { .encrypt = atmel_aes_cfb64_encrypt, .decrypt = atmel_aes_cfb64_decrypt, } -}, }; static void atmel_aes_queue_task(unsigned long data) @@ -969,7 +1167,14 @@ static void atmel_aes_done_task(unsigned long data) err = dd->err ? : err; if (dd->total && !err) { - err = atmel_aes_crypt_dma_start(dd); + if (dd->flags & AES_FLAGS_FAST) { + dd->in_sg = sg_next(dd->in_sg); + dd->out_sg = sg_next(dd->out_sg); + if (!dd->in_sg || !dd->out_sg) + err = -EINVAL; + } + if (!err) + err = atmel_aes_crypt_dma_start(dd); if (!err) return; /* DMA started. Not fininishing. */ } @@ -1003,8 +1208,8 @@ static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) for (i = 0; i < ARRAY_SIZE(aes_algs); i++) crypto_unregister_alg(&aes_algs[i]); - if (dd->hw_version >= 0x130) - crypto_unregister_alg(&aes_cfb64_alg[0]); + if (dd->caps.has_cfb64) + crypto_unregister_alg(&aes_cfb64_alg); } static int atmel_aes_register_algs(struct atmel_aes_dev *dd) @@ -1017,10 +1222,8 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd) goto err_aes_algs; } - atmel_aes_hw_version_init(dd); - - if (dd->hw_version >= 0x130) { - err = crypto_register_alg(&aes_cfb64_alg[0]); + if (dd->caps.has_cfb64) { + err = crypto_register_alg(&aes_cfb64_alg); if (err) goto err_aes_cfb64_alg; } @@ -1036,10 +1239,32 @@ err_aes_algs: return err; } +static void atmel_aes_get_cap(struct atmel_aes_dev *dd) +{ + dd->caps.has_dualbuff = 0; + dd->caps.has_cfb64 = 0; + dd->caps.max_burst_size = 1; + + /* keep only major version number */ + switch (dd->hw_version & 0xff0) { + case 0x130: + dd->caps.has_dualbuff = 1; + dd->caps.has_cfb64 = 1; + dd->caps.max_burst_size = 4; + break; + case 0x120: + break; + default: + dev_warn(dd->dev, + "Unmanaged aes version, set minimum capabilities\n"); + break; + } +} + static int atmel_aes_probe(struct platform_device *pdev) { struct atmel_aes_dev *aes_dd; - struct aes_platform_data *pdata; + struct crypto_platform_data *pdata; struct device *dev = &pdev->dev; struct resource *aes_res; unsigned long aes_phys_size; @@ -1099,7 +1324,7 @@ static int atmel_aes_probe(struct platform_device *pdev) } /* Initializing the clock */ - aes_dd->iclk = clk_get(&pdev->dev, NULL); + aes_dd->iclk = clk_get(&pdev->dev, "aes_clk"); if (IS_ERR(aes_dd->iclk)) { dev_err(dev, "clock intialization failed.\n"); err = PTR_ERR(aes_dd->iclk); @@ -1113,7 +1338,15 @@ static int atmel_aes_probe(struct platform_device *pdev) goto aes_io_err; } - err = atmel_aes_dma_init(aes_dd); + atmel_aes_hw_version_init(aes_dd); + + atmel_aes_get_cap(aes_dd); + + err = atmel_aes_buff_init(aes_dd); + if (err) + goto err_aes_buff; + + err = atmel_aes_dma_init(aes_dd, pdata); if (err) goto err_aes_dma; @@ -1135,6 +1368,8 @@ err_algs: spin_unlock(&atmel_aes.lock); atmel_aes_dma_cleanup(aes_dd); err_aes_dma: + atmel_aes_buff_cleanup(aes_dd); +err_aes_buff: iounmap(aes_dd->io_base); aes_io_err: clk_put(aes_dd->iclk); -- cgit v0.10.2 From 1f858040c2f78013fd2b10ddeb9dc157c3362b04 Mon Sep 17 00:00:00 2001 From: Nicolas Royer Date: Wed, 20 Feb 2013 17:10:25 +0100 Subject: crypto: atmel-tdes - add support for latest release of the IP (0x700) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update from previous IP release (0x600): - add DMA support (previous IP release use PDC) Signed-off-by: Nicolas Royer Acked-by: Nicolas Ferre Acked-by: Eric Bénard Tested-by: Eric Bénard Signed-off-by: Herbert Xu diff --git a/drivers/crypto/atmel-tdes-regs.h b/drivers/crypto/atmel-tdes-regs.h index 5ac2a90..f86734d 100644 --- a/drivers/crypto/atmel-tdes-regs.h +++ b/drivers/crypto/atmel-tdes-regs.h @@ -69,6 +69,8 @@ #define TDES_XTEARNDR_XTEA_RNDS_MASK (0x3F << 0) #define TDES_XTEARNDR_XTEA_RNDS_OFFSET 0 +#define TDES_HW_VERSION 0xFC + #define TDES_RPR 0x100 #define TDES_RCR 0x104 #define TDES_TPR 0x108 diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 7c73fbb..4a99564 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -38,29 +38,35 @@ #include #include #include +#include #include "atmel-tdes-regs.h" /* TDES flags */ -#define TDES_FLAGS_MODE_MASK 0x007f +#define TDES_FLAGS_MODE_MASK 0x00ff #define TDES_FLAGS_ENCRYPT BIT(0) #define TDES_FLAGS_CBC BIT(1) #define TDES_FLAGS_CFB BIT(2) #define TDES_FLAGS_CFB8 BIT(3) #define TDES_FLAGS_CFB16 BIT(4) #define TDES_FLAGS_CFB32 BIT(5) -#define TDES_FLAGS_OFB BIT(6) +#define TDES_FLAGS_CFB64 BIT(6) +#define TDES_FLAGS_OFB BIT(7) #define TDES_FLAGS_INIT BIT(16) #define TDES_FLAGS_FAST BIT(17) #define TDES_FLAGS_BUSY BIT(18) +#define TDES_FLAGS_DMA BIT(19) -#define ATMEL_TDES_QUEUE_LENGTH 1 +#define ATMEL_TDES_QUEUE_LENGTH 50 #define CFB8_BLOCK_SIZE 1 #define CFB16_BLOCK_SIZE 2 #define CFB32_BLOCK_SIZE 4 -#define CFB64_BLOCK_SIZE 8 +struct atmel_tdes_caps { + bool has_dma; + u32 has_cfb_3keys; +}; struct atmel_tdes_dev; @@ -70,12 +76,19 @@ struct atmel_tdes_ctx { int keylen; u32 key[3*DES_KEY_SIZE / sizeof(u32)]; unsigned long flags; + + u16 block_size; }; struct atmel_tdes_reqctx { unsigned long mode; }; +struct atmel_tdes_dma { + struct dma_chan *chan; + struct dma_slave_config dma_conf; +}; + struct atmel_tdes_dev { struct list_head list; unsigned long phys_base; @@ -99,8 +112,10 @@ struct atmel_tdes_dev { size_t total; struct scatterlist *in_sg; + unsigned int nb_in_sg; size_t in_offset; struct scatterlist *out_sg; + unsigned int nb_out_sg; size_t out_offset; size_t buflen; @@ -109,10 +124,16 @@ struct atmel_tdes_dev { void *buf_in; int dma_in; dma_addr_t dma_addr_in; + struct atmel_tdes_dma dma_lch_in; void *buf_out; int dma_out; dma_addr_t dma_addr_out; + struct atmel_tdes_dma dma_lch_out; + + struct atmel_tdes_caps caps; + + u32 hw_version; }; struct atmel_tdes_drv { @@ -207,6 +228,31 @@ static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd) return 0; } +static inline unsigned int atmel_tdes_get_version(struct atmel_tdes_dev *dd) +{ + return atmel_tdes_read(dd, TDES_HW_VERSION) & 0x00000fff; +} + +static void atmel_tdes_hw_version_init(struct atmel_tdes_dev *dd) +{ + atmel_tdes_hw_init(dd); + + dd->hw_version = atmel_tdes_get_version(dd); + + dev_info(dd->dev, + "version: 0x%x\n", dd->hw_version); + + clk_disable_unprepare(dd->iclk); +} + +static void atmel_tdes_dma_callback(void *data) +{ + struct atmel_tdes_dev *dd = data; + + /* dma_lch_out - completed */ + tasklet_schedule(&dd->done_task); +} + static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd) { int err; @@ -217,7 +263,9 @@ static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd) if (err) return err; - atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS); + if (!dd->caps.has_dma) + atmel_tdes_write(dd, TDES_PTCR, + TDES_PTCR_TXTDIS | TDES_PTCR_RXTDIS); /* MR register must be set before IV registers */ if (dd->ctx->keylen > (DES_KEY_SIZE << 1)) { @@ -241,6 +289,8 @@ static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd) valmr |= TDES_MR_CFBS_16b; else if (dd->flags & TDES_FLAGS_CFB32) valmr |= TDES_MR_CFBS_32b; + else if (dd->flags & TDES_FLAGS_CFB64) + valmr |= TDES_MR_CFBS_64b; } else if (dd->flags & TDES_FLAGS_OFB) { valmr |= TDES_MR_OPMOD_OFB; } @@ -262,7 +312,7 @@ static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd) return 0; } -static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd) +static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd) { int err = 0; size_t count; @@ -288,7 +338,7 @@ static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd) return err; } -static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd) +static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd) { int err = -ENOMEM; @@ -333,7 +383,7 @@ err_alloc: return err; } -static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd) +static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd) { dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, DMA_FROM_DEVICE); @@ -343,7 +393,7 @@ static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd) free_page((unsigned long)dd->buf_in); } -static int atmel_tdes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in, +static int atmel_tdes_crypt_pdc(struct crypto_tfm *tfm, dma_addr_t dma_addr_in, dma_addr_t dma_addr_out, int length) { struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm); @@ -379,7 +429,76 @@ static int atmel_tdes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in, return 0; } -static int atmel_tdes_crypt_dma_start(struct atmel_tdes_dev *dd) +static int atmel_tdes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in, + dma_addr_t dma_addr_out, int length) +{ + struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm); + struct atmel_tdes_dev *dd = ctx->dd; + struct scatterlist sg[2]; + struct dma_async_tx_descriptor *in_desc, *out_desc; + + dd->dma_size = length; + + if (!(dd->flags & TDES_FLAGS_FAST)) { + dma_sync_single_for_device(dd->dev, dma_addr_in, length, + DMA_TO_DEVICE); + } + + if (dd->flags & TDES_FLAGS_CFB8) { + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_1_BYTE; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_1_BYTE; + } else if (dd->flags & TDES_FLAGS_CFB16) { + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_2_BYTES; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_2_BYTES; + } else { + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + } + + dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); + dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf); + + dd->flags |= TDES_FLAGS_DMA; + + sg_init_table(&sg[0], 1); + sg_dma_address(&sg[0]) = dma_addr_in; + sg_dma_len(&sg[0]) = length; + + sg_init_table(&sg[1], 1); + sg_dma_address(&sg[1]) = dma_addr_out; + sg_dma_len(&sg[1]) = length; + + in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0], + 1, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!in_desc) + return -EINVAL; + + out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1], + 1, DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!out_desc) + return -EINVAL; + + out_desc->callback = atmel_tdes_dma_callback; + out_desc->callback_param = dd; + + dmaengine_submit(out_desc); + dma_async_issue_pending(dd->dma_lch_out.chan); + + dmaengine_submit(in_desc); + dma_async_issue_pending(dd->dma_lch_in.chan); + + return 0; +} + +static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd) { struct crypto_tfm *tfm = crypto_ablkcipher_tfm( crypto_ablkcipher_reqtfm(dd->req)); @@ -387,23 +506,23 @@ static int atmel_tdes_crypt_dma_start(struct atmel_tdes_dev *dd) size_t count; dma_addr_t addr_in, addr_out; - if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) { + if ((!dd->in_offset) && (!dd->out_offset)) { /* check for alignment */ - in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)); - out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)); - + in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) && + IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size); + out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) && + IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size); fast = in && out; + + if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg)) + fast = 0; } + if (fast) { count = min(dd->total, sg_dma_len(dd->in_sg)); count = min(count, sg_dma_len(dd->out_sg)); - if (count != dd->total) { - pr_err("request length != buffer length\n"); - return -EINVAL; - } - err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); if (!err) { dev_err(dd->dev, "dma_map_sg() error\n"); @@ -433,13 +552,16 @@ static int atmel_tdes_crypt_dma_start(struct atmel_tdes_dev *dd) addr_out = dd->dma_addr_out; dd->flags &= ~TDES_FLAGS_FAST; - } dd->total -= count; - err = atmel_tdes_crypt_dma(tfm, addr_in, addr_out, count); - if (err) { + if (dd->caps.has_dma) + err = atmel_tdes_crypt_dma(tfm, addr_in, addr_out, count); + else + err = atmel_tdes_crypt_pdc(tfm, addr_in, addr_out, count); + + if (err && (dd->flags & TDES_FLAGS_FAST)) { dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); } @@ -447,7 +569,6 @@ static int atmel_tdes_crypt_dma_start(struct atmel_tdes_dev *dd) return err; } - static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err) { struct ablkcipher_request *req = dd->req; @@ -506,7 +627,7 @@ static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd, err = atmel_tdes_write_ctrl(dd); if (!err) - err = atmel_tdes_crypt_dma_start(dd); + err = atmel_tdes_crypt_start(dd); if (err) { /* des_task will not finish it, so do it here */ atmel_tdes_finish_req(dd, err); @@ -516,41 +637,145 @@ static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd, return ret; } +static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd) +{ + int err = -EINVAL; + size_t count; + + if (dd->flags & TDES_FLAGS_DMA) { + err = 0; + if (dd->flags & TDES_FLAGS_FAST) { + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); + } else { + dma_sync_single_for_device(dd->dev, dd->dma_addr_out, + dd->dma_size, DMA_FROM_DEVICE); + + /* copy data */ + count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, + dd->buf_out, dd->buflen, dd->dma_size, 1); + if (count != dd->dma_size) { + err = -EINVAL; + pr_err("not all data converted: %u\n", count); + } + } + } + return err; +} static int atmel_tdes_crypt(struct ablkcipher_request *req, unsigned long mode) { struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx( crypto_ablkcipher_reqtfm(req)); struct atmel_tdes_reqctx *rctx = ablkcipher_request_ctx(req); - struct atmel_tdes_dev *dd; if (mode & TDES_FLAGS_CFB8) { if (!IS_ALIGNED(req->nbytes, CFB8_BLOCK_SIZE)) { pr_err("request size is not exact amount of CFB8 blocks\n"); return -EINVAL; } + ctx->block_size = CFB8_BLOCK_SIZE; } else if (mode & TDES_FLAGS_CFB16) { if (!IS_ALIGNED(req->nbytes, CFB16_BLOCK_SIZE)) { pr_err("request size is not exact amount of CFB16 blocks\n"); return -EINVAL; } + ctx->block_size = CFB16_BLOCK_SIZE; } else if (mode & TDES_FLAGS_CFB32) { if (!IS_ALIGNED(req->nbytes, CFB32_BLOCK_SIZE)) { pr_err("request size is not exact amount of CFB32 blocks\n"); return -EINVAL; } - } else if (!IS_ALIGNED(req->nbytes, DES_BLOCK_SIZE)) { - pr_err("request size is not exact amount of DES blocks\n"); - return -EINVAL; + ctx->block_size = CFB32_BLOCK_SIZE; + } else { + if (!IS_ALIGNED(req->nbytes, DES_BLOCK_SIZE)) { + pr_err("request size is not exact amount of DES blocks\n"); + return -EINVAL; + } + ctx->block_size = DES_BLOCK_SIZE; } - dd = atmel_tdes_find_dev(ctx); - if (!dd) + rctx->mode = mode; + + return atmel_tdes_handle_queue(ctx->dd, req); +} + +static bool atmel_tdes_filter(struct dma_chan *chan, void *slave) +{ + struct at_dma_slave *sl = slave; + + if (sl && sl->dma_dev == chan->device->dev) { + chan->private = sl; + return true; + } else { + return false; + } +} + +static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd, + struct crypto_platform_data *pdata) +{ + int err = -ENOMEM; + dma_cap_mask_t mask_in, mask_out; + + if (pdata && pdata->dma_slave->txdata.dma_dev && + pdata->dma_slave->rxdata.dma_dev) { + + /* Try to grab 2 DMA channels */ + dma_cap_zero(mask_in); + dma_cap_set(DMA_SLAVE, mask_in); + + dd->dma_lch_in.chan = dma_request_channel(mask_in, + atmel_tdes_filter, &pdata->dma_slave->rxdata); + + if (!dd->dma_lch_in.chan) + goto err_dma_in; + + dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; + dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + + TDES_IDATA1R; + dd->dma_lch_in.dma_conf.src_maxburst = 1; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = 1; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.device_fc = false; + + dma_cap_zero(mask_out); + dma_cap_set(DMA_SLAVE, mask_out); + dd->dma_lch_out.chan = dma_request_channel(mask_out, + atmel_tdes_filter, &pdata->dma_slave->txdata); + + if (!dd->dma_lch_out.chan) + goto err_dma_out; + + dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; + dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + + TDES_ODATA1R; + dd->dma_lch_out.dma_conf.src_maxburst = 1; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.dst_maxburst = 1; + dd->dma_lch_out.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.device_fc = false; + + return 0; + } else { return -ENODEV; + } - rctx->mode = mode; +err_dma_out: + dma_release_channel(dd->dma_lch_in.chan); +err_dma_in: + return err; +} - return atmel_tdes_handle_queue(dd, req); +static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd) +{ + dma_release_channel(dd->dma_lch_in.chan); + dma_release_channel(dd->dma_lch_out.chan); } static int atmel_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, @@ -590,7 +815,8 @@ static int atmel_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, /* * HW bug in cfb 3-keys mode. */ - if (strstr(alg_name, "cfb") && (keylen != 2*DES_KEY_SIZE)) { + if (!ctx->dd->caps.has_cfb_3keys && strstr(alg_name, "cfb") + && (keylen != 2*DES_KEY_SIZE)) { crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } else if ((keylen != 2*DES_KEY_SIZE) && (keylen != 3*DES_KEY_SIZE)) { @@ -678,8 +904,15 @@ static int atmel_tdes_ofb_decrypt(struct ablkcipher_request *req) static int atmel_tdes_cra_init(struct crypto_tfm *tfm) { + struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm); + struct atmel_tdes_dev *dd; + tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_tdes_reqctx); + dd = atmel_tdes_find_dev(ctx); + if (!dd) + return -ENODEV; + return 0; } @@ -695,7 +928,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x7, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -715,7 +948,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x7, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -736,7 +969,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x7, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -778,7 +1011,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB16_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x1, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -799,7 +1032,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB32_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x3, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -820,7 +1053,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x7, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -841,7 +1074,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x7, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -861,7 +1094,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x7, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -882,7 +1115,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x7, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -924,7 +1157,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB16_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x1, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -945,7 +1178,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB32_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x3, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -966,7 +1199,7 @@ static struct crypto_alg tdes_algs[] = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, + .cra_alignmask = 0x7, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, @@ -994,14 +1227,24 @@ static void atmel_tdes_done_task(unsigned long data) struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *) data; int err; - err = atmel_tdes_crypt_dma_stop(dd); + if (!(dd->flags & TDES_FLAGS_DMA)) + err = atmel_tdes_crypt_pdc_stop(dd); + else + err = atmel_tdes_crypt_dma_stop(dd); err = dd->err ? : err; if (dd->total && !err) { - err = atmel_tdes_crypt_dma_start(dd); + if (dd->flags & TDES_FLAGS_FAST) { + dd->in_sg = sg_next(dd->in_sg); + dd->out_sg = sg_next(dd->out_sg); + if (!dd->in_sg || !dd->out_sg) + err = -EINVAL; + } if (!err) - return; + err = atmel_tdes_crypt_start(dd); + if (!err) + return; /* DMA started. Not fininishing. */ } atmel_tdes_finish_req(dd, err); @@ -1053,9 +1296,31 @@ err_tdes_algs: return err; } +static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd) +{ + + dd->caps.has_dma = 0; + dd->caps.has_cfb_3keys = 0; + + /* keep only major version number */ + switch (dd->hw_version & 0xf00) { + case 0x700: + dd->caps.has_dma = 1; + dd->caps.has_cfb_3keys = 1; + break; + case 0x600: + break; + default: + dev_warn(dd->dev, + "Unmanaged tdes version, set minimum capabilities\n"); + break; + } +} + static int atmel_tdes_probe(struct platform_device *pdev) { struct atmel_tdes_dev *tdes_dd; + struct crypto_platform_data *pdata; struct device *dev = &pdev->dev; struct resource *tdes_res; unsigned long tdes_phys_size; @@ -1109,7 +1374,7 @@ static int atmel_tdes_probe(struct platform_device *pdev) } /* Initializing the clock */ - tdes_dd->iclk = clk_get(&pdev->dev, NULL); + tdes_dd->iclk = clk_get(&pdev->dev, "tdes_clk"); if (IS_ERR(tdes_dd->iclk)) { dev_err(dev, "clock intialization failed.\n"); err = PTR_ERR(tdes_dd->iclk); @@ -1123,9 +1388,25 @@ static int atmel_tdes_probe(struct platform_device *pdev) goto tdes_io_err; } - err = atmel_tdes_dma_init(tdes_dd); + atmel_tdes_hw_version_init(tdes_dd); + + atmel_tdes_get_cap(tdes_dd); + + err = atmel_tdes_buff_init(tdes_dd); if (err) - goto err_tdes_dma; + goto err_tdes_buff; + + if (tdes_dd->caps.has_dma) { + pdata = pdev->dev.platform_data; + if (!pdata) { + dev_err(&pdev->dev, "platform data not available\n"); + err = -ENXIO; + goto err_pdata; + } + err = atmel_tdes_dma_init(tdes_dd, pdata); + if (err) + goto err_tdes_dma; + } spin_lock(&atmel_tdes.lock); list_add_tail(&tdes_dd->list, &atmel_tdes.dev_list); @@ -1143,8 +1424,12 @@ err_algs: spin_lock(&atmel_tdes.lock); list_del(&tdes_dd->list); spin_unlock(&atmel_tdes.lock); - atmel_tdes_dma_cleanup(tdes_dd); + if (tdes_dd->caps.has_dma) + atmel_tdes_dma_cleanup(tdes_dd); err_tdes_dma: +err_pdata: + atmel_tdes_buff_cleanup(tdes_dd); +err_tdes_buff: iounmap(tdes_dd->io_base); tdes_io_err: clk_put(tdes_dd->iclk); @@ -1178,7 +1463,10 @@ static int atmel_tdes_remove(struct platform_device *pdev) tasklet_kill(&tdes_dd->done_task); tasklet_kill(&tdes_dd->queue_task); - atmel_tdes_dma_cleanup(tdes_dd); + if (tdes_dd->caps.has_dma) + atmel_tdes_dma_cleanup(tdes_dd); + + atmel_tdes_buff_cleanup(tdes_dd); iounmap(tdes_dd->io_base); -- cgit v0.10.2 From d4905b38d1f6b60761a6fd16f45ebd1fac8b6e1f Mon Sep 17 00:00:00 2001 From: Nicolas Royer Date: Wed, 20 Feb 2013 17:10:26 +0100 Subject: crypto: atmel-sha - add support for latest release of the IP (0x410) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates from IP release 0x320 to 0x400: - add DMA support (previous IP revision use PDC) - add DMA double input buffer support - add SHA224 support Update from IP release 0x400 to 0x410: - add SHA384 and SHA512 support Signed-off-by: Nicolas Royer Acked-by: Nicolas Ferre Acked-by: Eric Bénard Tested-by: Eric Bénard Signed-off-by: Herbert Xu diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 87ec4d0..e66fb0a 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -361,15 +361,17 @@ config CRYPTO_DEV_ATMEL_TDES will be called atmel-tdes. config CRYPTO_DEV_ATMEL_SHA - tristate "Support for Atmel SHA1/SHA256 hw accelerator" + tristate "Support for Atmel SHA hw accelerator" depends on ARCH_AT91 select CRYPTO_SHA1 select CRYPTO_SHA256 + select CRYPTO_SHA512 select CRYPTO_ALGAPI help - Some Atmel processors have SHA1/SHA256 hw accelerator. + Some Atmel processors have SHA1/SHA224/SHA256/SHA384/SHA512 + hw accelerator. Select this if you want to use the Atmel module for - SHA1/SHA256 algorithms. + SHA1/SHA224/SHA256/SHA384/SHA512 algorithms. To compile this driver as a module, choose M here: the module will be called atmel-sha. diff --git a/drivers/crypto/atmel-sha-regs.h b/drivers/crypto/atmel-sha-regs.h index dc53a20..83b2d74 100644 --- a/drivers/crypto/atmel-sha-regs.h +++ b/drivers/crypto/atmel-sha-regs.h @@ -14,10 +14,13 @@ #define SHA_MR_MODE_MANUAL 0x0 #define SHA_MR_MODE_AUTO 0x1 #define SHA_MR_MODE_PDC 0x2 -#define SHA_MR_DUALBUFF (1 << 3) #define SHA_MR_PROCDLY (1 << 4) #define SHA_MR_ALGO_SHA1 (0 << 8) #define SHA_MR_ALGO_SHA256 (1 << 8) +#define SHA_MR_ALGO_SHA384 (2 << 8) +#define SHA_MR_ALGO_SHA512 (3 << 8) +#define SHA_MR_ALGO_SHA224 (4 << 8) +#define SHA_MR_DUALBUFF (1 << 16) #define SHA_IER 0x10 #define SHA_IDR 0x14 @@ -33,6 +36,8 @@ #define SHA_ISR_URAT_MR (0x2 << 12) #define SHA_ISR_URAT_WO (0x5 << 12) +#define SHA_HW_VERSION 0xFC + #define SHA_TPR 0x108 #define SHA_TCR 0x10C #define SHA_TNPR 0x118 diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 4918e94..eaed8bf 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "atmel-sha-regs.h" /* SHA flags */ @@ -52,11 +53,12 @@ #define SHA_FLAGS_FINUP BIT(16) #define SHA_FLAGS_SG BIT(17) #define SHA_FLAGS_SHA1 BIT(18) -#define SHA_FLAGS_SHA256 BIT(19) -#define SHA_FLAGS_ERROR BIT(20) -#define SHA_FLAGS_PAD BIT(21) - -#define SHA_FLAGS_DUALBUFF BIT(24) +#define SHA_FLAGS_SHA224 BIT(19) +#define SHA_FLAGS_SHA256 BIT(20) +#define SHA_FLAGS_SHA384 BIT(21) +#define SHA_FLAGS_SHA512 BIT(22) +#define SHA_FLAGS_ERROR BIT(23) +#define SHA_FLAGS_PAD BIT(24) #define SHA_OP_UPDATE 1 #define SHA_OP_FINAL 2 @@ -65,6 +67,12 @@ #define ATMEL_SHA_DMA_THRESHOLD 56 +struct atmel_sha_caps { + bool has_dma; + bool has_dualbuff; + bool has_sha224; + bool has_sha_384_512; +}; struct atmel_sha_dev; @@ -73,8 +81,8 @@ struct atmel_sha_reqctx { unsigned long flags; unsigned long op; - u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32)); - size_t digcnt; + u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32)); + u64 digcnt[2]; size_t bufcnt; size_t buflen; dma_addr_t dma_addr; @@ -84,6 +92,8 @@ struct atmel_sha_reqctx { unsigned int offset; /* offset in current sg */ unsigned int total; /* total request */ + size_t block_size; + u8 buffer[0] __aligned(sizeof(u32)); }; @@ -97,7 +107,12 @@ struct atmel_sha_ctx { }; -#define ATMEL_SHA_QUEUE_LENGTH 1 +#define ATMEL_SHA_QUEUE_LENGTH 50 + +struct atmel_sha_dma { + struct dma_chan *chan; + struct dma_slave_config dma_conf; +}; struct atmel_sha_dev { struct list_head list; @@ -114,6 +129,12 @@ struct atmel_sha_dev { unsigned long flags; struct crypto_queue queue; struct ahash_request *req; + + struct atmel_sha_dma dma_lch_in; + + struct atmel_sha_caps caps; + + u32 hw_version; }; struct atmel_sha_drv { @@ -137,14 +158,6 @@ static inline void atmel_sha_write(struct atmel_sha_dev *dd, writel_relaxed(value, dd->io_base + offset); } -static void atmel_sha_dualbuff_test(struct atmel_sha_dev *dd) -{ - atmel_sha_write(dd, SHA_MR, SHA_MR_DUALBUFF); - - if (atmel_sha_read(dd, SHA_MR) & SHA_MR_DUALBUFF) - dd->flags |= SHA_FLAGS_DUALBUFF; -} - static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx) { size_t count; @@ -176,31 +189,58 @@ static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx) } /* - * The purpose of this padding is to ensure that the padded message - * is a multiple of 512 bits. The bit "1" is appended at the end of - * the message followed by "padlen-1" zero bits. Then a 64 bits block - * equals to the message length in bits is appended. + * The purpose of this padding is to ensure that the padded message is a + * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). + * The bit "1" is appended at the end of the message followed by + * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or + * 128 bits block (SHA384/SHA512) equals to the message length in bits + * is appended. * - * padlen is calculated as followed: + * For SHA1/SHA224/SHA256, padlen is calculated as followed: * - if message length < 56 bytes then padlen = 56 - message length * - else padlen = 64 + 56 - message length + * + * For SHA384/SHA512, padlen is calculated as followed: + * - if message length < 112 bytes then padlen = 112 - message length + * - else padlen = 128 + 112 - message length */ static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length) { unsigned int index, padlen; - u64 bits; - u64 size; - - bits = (ctx->bufcnt + ctx->digcnt + length) << 3; - size = cpu_to_be64(bits); - - index = ctx->bufcnt & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - *(ctx->buffer + ctx->bufcnt) = 0x80; - memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); - memcpy(ctx->buffer + ctx->bufcnt + padlen, &size, 8); - ctx->bufcnt += padlen + 8; - ctx->flags |= SHA_FLAGS_PAD; + u64 bits[2]; + u64 size[2]; + + size[0] = ctx->digcnt[0]; + size[1] = ctx->digcnt[1]; + + size[0] += ctx->bufcnt; + if (size[0] < ctx->bufcnt) + size[1]++; + + size[0] += length; + if (size[0] < length) + size[1]++; + + bits[1] = cpu_to_be64(size[0] << 3); + bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61); + + if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) { + index = ctx->bufcnt & 0x7f; + padlen = (index < 112) ? (112 - index) : ((128+112) - index); + *(ctx->buffer + ctx->bufcnt) = 0x80; + memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); + memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); + ctx->bufcnt += padlen + 16; + ctx->flags |= SHA_FLAGS_PAD; + } else { + index = ctx->bufcnt & 0x3f; + padlen = (index < 56) ? (56 - index) : ((64+56) - index); + *(ctx->buffer + ctx->bufcnt) = 0x80; + memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); + memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); + ctx->bufcnt += padlen + 8; + ctx->flags |= SHA_FLAGS_PAD; + } } static int atmel_sha_init(struct ahash_request *req) @@ -231,13 +271,35 @@ static int atmel_sha_init(struct ahash_request *req) dev_dbg(dd->dev, "init: digest size: %d\n", crypto_ahash_digestsize(tfm)); - if (crypto_ahash_digestsize(tfm) == SHA1_DIGEST_SIZE) + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: ctx->flags |= SHA_FLAGS_SHA1; - else if (crypto_ahash_digestsize(tfm) == SHA256_DIGEST_SIZE) + ctx->block_size = SHA1_BLOCK_SIZE; + break; + case SHA224_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA224; + ctx->block_size = SHA224_BLOCK_SIZE; + break; + case SHA256_DIGEST_SIZE: ctx->flags |= SHA_FLAGS_SHA256; + ctx->block_size = SHA256_BLOCK_SIZE; + break; + case SHA384_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA384; + ctx->block_size = SHA384_BLOCK_SIZE; + break; + case SHA512_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA512; + ctx->block_size = SHA512_BLOCK_SIZE; + break; + default: + return -EINVAL; + break; + } ctx->bufcnt = 0; - ctx->digcnt = 0; + ctx->digcnt[0] = 0; + ctx->digcnt[1] = 0; ctx->buflen = SHA_BUFFER_LEN; return 0; @@ -249,19 +311,28 @@ static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma) u32 valcr = 0, valmr = SHA_MR_MODE_AUTO; if (likely(dma)) { - atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE); + if (!dd->caps.has_dma) + atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE); valmr = SHA_MR_MODE_PDC; - if (dd->flags & SHA_FLAGS_DUALBUFF) - valmr = SHA_MR_DUALBUFF; + if (dd->caps.has_dualbuff) + valmr |= SHA_MR_DUALBUFF; } else { atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); } - if (ctx->flags & SHA_FLAGS_SHA256) + if (ctx->flags & SHA_FLAGS_SHA1) + valmr |= SHA_MR_ALGO_SHA1; + else if (ctx->flags & SHA_FLAGS_SHA224) + valmr |= SHA_MR_ALGO_SHA224; + else if (ctx->flags & SHA_FLAGS_SHA256) valmr |= SHA_MR_ALGO_SHA256; + else if (ctx->flags & SHA_FLAGS_SHA384) + valmr |= SHA_MR_ALGO_SHA384; + else if (ctx->flags & SHA_FLAGS_SHA512) + valmr |= SHA_MR_ALGO_SHA512; /* Setting CR_FIRST only for the first iteration */ - if (!ctx->digcnt) + if (!(ctx->digcnt[0] || ctx->digcnt[1])) valcr = SHA_CR_FIRST; atmel_sha_write(dd, SHA_CR, valcr); @@ -275,13 +346,15 @@ static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf, int count, len32; const u32 *buffer = (const u32 *)buf; - dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n", - ctx->digcnt, length, final); + dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", + ctx->digcnt[1], ctx->digcnt[0], length, final); atmel_sha_write_ctrl(dd, 0); /* should be non-zero before next lines to disable clocks later */ - ctx->digcnt += length; + ctx->digcnt[0] += length; + if (ctx->digcnt[0] < length) + ctx->digcnt[1]++; if (final) dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ @@ -302,8 +375,8 @@ static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); int len32; - dev_dbg(dd->dev, "xmit_pdc: digcnt: %d, length: %d, final: %d\n", - ctx->digcnt, length1, final); + dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", + ctx->digcnt[1], ctx->digcnt[0], length1, final); len32 = DIV_ROUND_UP(length1, sizeof(u32)); atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTDIS); @@ -317,7 +390,9 @@ static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, atmel_sha_write_ctrl(dd, 1); /* should be non-zero before next lines to disable clocks later */ - ctx->digcnt += length1; + ctx->digcnt[0] += length1; + if (ctx->digcnt[0] < length1) + ctx->digcnt[1]++; if (final) dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ @@ -330,6 +405,86 @@ static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, return -EINPROGRESS; } +static void atmel_sha_dma_callback(void *data) +{ + struct atmel_sha_dev *dd = data; + + /* dma_lch_in - completed - wait DATRDY */ + atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); +} + +static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, + size_t length1, dma_addr_t dma_addr2, size_t length2, int final) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); + struct dma_async_tx_descriptor *in_desc; + struct scatterlist sg[2]; + + dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", + ctx->digcnt[1], ctx->digcnt[0], length1, final); + + if (ctx->flags & (SHA_FLAGS_SHA1 | SHA_FLAGS_SHA224 | + SHA_FLAGS_SHA256)) { + dd->dma_lch_in.dma_conf.src_maxburst = 16; + dd->dma_lch_in.dma_conf.dst_maxburst = 16; + } else { + dd->dma_lch_in.dma_conf.src_maxburst = 32; + dd->dma_lch_in.dma_conf.dst_maxburst = 32; + } + + dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); + + if (length2) { + sg_init_table(sg, 2); + sg_dma_address(&sg[0]) = dma_addr1; + sg_dma_len(&sg[0]) = length1; + sg_dma_address(&sg[1]) = dma_addr2; + sg_dma_len(&sg[1]) = length2; + in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + } else { + sg_init_table(sg, 1); + sg_dma_address(&sg[0]) = dma_addr1; + sg_dma_len(&sg[0]) = length1; + in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + } + if (!in_desc) + return -EINVAL; + + in_desc->callback = atmel_sha_dma_callback; + in_desc->callback_param = dd; + + atmel_sha_write_ctrl(dd, 1); + + /* should be non-zero before next lines to disable clocks later */ + ctx->digcnt[0] += length1; + if (ctx->digcnt[0] < length1) + ctx->digcnt[1]++; + + if (final) + dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ + + dd->flags |= SHA_FLAGS_DMA_ACTIVE; + + /* Start DMA transfer */ + dmaengine_submit(in_desc); + dma_async_issue_pending(dd->dma_lch_in.chan); + + return -EINPROGRESS; +} + +static int atmel_sha_xmit_start(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, + size_t length1, dma_addr_t dma_addr2, size_t length2, int final) +{ + if (dd->caps.has_dma) + return atmel_sha_xmit_dma(dd, dma_addr1, length1, + dma_addr2, length2, final); + else + return atmel_sha_xmit_pdc(dd, dma_addr1, length1, + dma_addr2, length2, final); +} + static int atmel_sha_update_cpu(struct atmel_sha_dev *dd) { struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); @@ -337,7 +492,6 @@ static int atmel_sha_update_cpu(struct atmel_sha_dev *dd) atmel_sha_append_sg(ctx); atmel_sha_fill_padding(ctx, 0); - bufcnt = ctx->bufcnt; ctx->bufcnt = 0; @@ -349,17 +503,17 @@ static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd, size_t length, int final) { ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, - ctx->buflen + SHA1_BLOCK_SIZE, DMA_TO_DEVICE); + ctx->buflen + ctx->block_size, DMA_TO_DEVICE); if (dma_mapping_error(dd->dev, ctx->dma_addr)) { dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen + - SHA1_BLOCK_SIZE); + ctx->block_size); return -EINVAL; } ctx->flags &= ~SHA_FLAGS_SG; /* next call does not fail... so no unmap in the case of error */ - return atmel_sha_xmit_pdc(dd, ctx->dma_addr, length, 0, 0, final); + return atmel_sha_xmit_start(dd, ctx->dma_addr, length, 0, 0, final); } static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd) @@ -372,8 +526,8 @@ static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd) final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; - dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n", - ctx->bufcnt, ctx->digcnt, final); + dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: 0x%llx 0x%llx, final: %d\n", + ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final); if (final) atmel_sha_fill_padding(ctx, 0); @@ -400,30 +554,25 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) if (ctx->bufcnt || ctx->offset) return atmel_sha_update_dma_slow(dd); - dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n", - ctx->digcnt, ctx->bufcnt, ctx->total); + dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %u, total: %u\n", + ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total); sg = ctx->sg; if (!IS_ALIGNED(sg->offset, sizeof(u32))) return atmel_sha_update_dma_slow(dd); - if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, SHA1_BLOCK_SIZE)) - /* size is not SHA1_BLOCK_SIZE aligned */ + if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->block_size)) + /* size is not ctx->block_size aligned */ return atmel_sha_update_dma_slow(dd); length = min(ctx->total, sg->length); if (sg_is_last(sg)) { if (!(ctx->flags & SHA_FLAGS_FINUP)) { - /* not last sg must be SHA1_BLOCK_SIZE aligned */ - tail = length & (SHA1_BLOCK_SIZE - 1); + /* not last sg must be ctx->block_size aligned */ + tail = length & (ctx->block_size - 1); length -= tail; - if (length == 0) { - /* offset where to start slow */ - ctx->offset = length; - return atmel_sha_update_dma_slow(dd); - } } } @@ -434,7 +583,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) /* Add padding */ if (final) { - tail = length & (SHA1_BLOCK_SIZE - 1); + tail = length & (ctx->block_size - 1); length -= tail; ctx->total += tail; ctx->offset = length; /* offset where to start slow */ @@ -445,10 +594,10 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) atmel_sha_fill_padding(ctx, length); ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, - ctx->buflen + SHA1_BLOCK_SIZE, DMA_TO_DEVICE); + ctx->buflen + ctx->block_size, DMA_TO_DEVICE); if (dma_mapping_error(dd->dev, ctx->dma_addr)) { dev_err(dd->dev, "dma %u bytes error\n", - ctx->buflen + SHA1_BLOCK_SIZE); + ctx->buflen + ctx->block_size); return -EINVAL; } @@ -456,7 +605,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) ctx->flags &= ~SHA_FLAGS_SG; count = ctx->bufcnt; ctx->bufcnt = 0; - return atmel_sha_xmit_pdc(dd, ctx->dma_addr, count, 0, + return atmel_sha_xmit_start(dd, ctx->dma_addr, count, 0, 0, final); } else { ctx->sg = sg; @@ -470,7 +619,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) count = ctx->bufcnt; ctx->bufcnt = 0; - return atmel_sha_xmit_pdc(dd, sg_dma_address(ctx->sg), + return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, ctx->dma_addr, count, final); } } @@ -483,7 +632,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) ctx->flags |= SHA_FLAGS_SG; /* next call does not fail... so no unmap in the case of error */ - return atmel_sha_xmit_pdc(dd, sg_dma_address(ctx->sg), length, 0, + return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, 0, 0, final); } @@ -498,12 +647,13 @@ static int atmel_sha_update_dma_stop(struct atmel_sha_dev *dd) if (ctx->sg) ctx->offset = 0; } - if (ctx->flags & SHA_FLAGS_PAD) + if (ctx->flags & SHA_FLAGS_PAD) { dma_unmap_single(dd->dev, ctx->dma_addr, - ctx->buflen + SHA1_BLOCK_SIZE, DMA_TO_DEVICE); + ctx->buflen + ctx->block_size, DMA_TO_DEVICE); + } } else { dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen + - SHA1_BLOCK_SIZE, DMA_TO_DEVICE); + ctx->block_size, DMA_TO_DEVICE); } return 0; @@ -515,8 +665,8 @@ static int atmel_sha_update_req(struct atmel_sha_dev *dd) struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); int err; - dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n", - ctx->total, ctx->digcnt, (ctx->flags & SHA_FLAGS_FINUP) != 0); + dev_dbg(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n", + ctx->total, ctx->digcnt[1], ctx->digcnt[0]); if (ctx->flags & SHA_FLAGS_CPU) err = atmel_sha_update_cpu(dd); @@ -524,8 +674,8 @@ static int atmel_sha_update_req(struct atmel_sha_dev *dd) err = atmel_sha_update_dma_start(dd); /* wait for dma completion before can take more data */ - dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", - err, ctx->digcnt); + dev_dbg(dd->dev, "update: err: %d, digcnt: 0x%llx 0%llx\n", + err, ctx->digcnt[1], ctx->digcnt[0]); return err; } @@ -562,12 +712,21 @@ static void atmel_sha_copy_hash(struct ahash_request *req) u32 *hash = (u32 *)ctx->digest; int i; - if (likely(ctx->flags & SHA_FLAGS_SHA1)) + if (ctx->flags & SHA_FLAGS_SHA1) for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); - else + else if (ctx->flags & SHA_FLAGS_SHA224) + for (i = 0; i < SHA224_DIGEST_SIZE / sizeof(u32); i++) + hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); + else if (ctx->flags & SHA_FLAGS_SHA256) for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(u32); i++) hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); + else if (ctx->flags & SHA_FLAGS_SHA384) + for (i = 0; i < SHA384_DIGEST_SIZE / sizeof(u32); i++) + hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); + else + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(u32); i++) + hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); } static void atmel_sha_copy_ready_hash(struct ahash_request *req) @@ -577,10 +736,16 @@ static void atmel_sha_copy_ready_hash(struct ahash_request *req) if (!req->result) return; - if (likely(ctx->flags & SHA_FLAGS_SHA1)) + if (ctx->flags & SHA_FLAGS_SHA1) memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE); - else + else if (ctx->flags & SHA_FLAGS_SHA224) + memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE); + else if (ctx->flags & SHA_FLAGS_SHA256) memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE); + else if (ctx->flags & SHA_FLAGS_SHA384) + memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE); + else + memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE); } static int atmel_sha_finish(struct ahash_request *req) @@ -589,11 +754,11 @@ static int atmel_sha_finish(struct ahash_request *req) struct atmel_sha_dev *dd = ctx->dd; int err = 0; - if (ctx->digcnt) + if (ctx->digcnt[0] || ctx->digcnt[1]) atmel_sha_copy_ready_hash(req); - dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, - ctx->bufcnt); + dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %d\n", ctx->digcnt[1], + ctx->digcnt[0], ctx->bufcnt); return err; } @@ -628,9 +793,8 @@ static int atmel_sha_hw_init(struct atmel_sha_dev *dd) { clk_prepare_enable(dd->iclk); - if (SHA_FLAGS_INIT & dd->flags) { + if (!(SHA_FLAGS_INIT & dd->flags)) { atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST); - atmel_sha_dualbuff_test(dd); dd->flags |= SHA_FLAGS_INIT; dd->err = 0; } @@ -638,6 +802,23 @@ static int atmel_sha_hw_init(struct atmel_sha_dev *dd) return 0; } +static inline unsigned int atmel_sha_get_version(struct atmel_sha_dev *dd) +{ + return atmel_sha_read(dd, SHA_HW_VERSION) & 0x00000fff; +} + +static void atmel_sha_hw_version_init(struct atmel_sha_dev *dd) +{ + atmel_sha_hw_init(dd); + + dd->hw_version = atmel_sha_get_version(dd); + + dev_info(dd->dev, + "version: 0x%x\n", dd->hw_version); + + clk_disable_unprepare(dd->iclk); +} + static int atmel_sha_handle_queue(struct atmel_sha_dev *dd, struct ahash_request *req) { @@ -682,10 +863,9 @@ static int atmel_sha_handle_queue(struct atmel_sha_dev *dd, if (ctx->op == SHA_OP_UPDATE) { err = atmel_sha_update_req(dd); - if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP)) { + if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP)) /* no final() after finup() */ err = atmel_sha_final_req(dd); - } } else if (ctx->op == SHA_OP_FINAL) { err = atmel_sha_final_req(dd); } @@ -808,7 +988,7 @@ static int atmel_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) } crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct atmel_sha_reqctx) + - SHA_BUFFER_LEN + SHA256_BLOCK_SIZE); + SHA_BUFFER_LEN + SHA512_BLOCK_SIZE); return 0; } @@ -826,7 +1006,7 @@ static void atmel_sha_cra_exit(struct crypto_tfm *tfm) tctx->fallback = NULL; } -static struct ahash_alg sha_algs[] = { +static struct ahash_alg sha_1_256_algs[] = { { .init = atmel_sha_init, .update = atmel_sha_update, @@ -875,6 +1055,79 @@ static struct ahash_alg sha_algs[] = { }, }; +static struct ahash_alg sha_224_alg = { + .init = atmel_sha_init, + .update = atmel_sha_update, + .final = atmel_sha_final, + .finup = atmel_sha_finup, + .digest = atmel_sha_digest, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "sha224", + .cra_driver_name = "atmel-sha224", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct atmel_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = atmel_sha_cra_init, + .cra_exit = atmel_sha_cra_exit, + } + } +}; + +static struct ahash_alg sha_384_512_algs[] = { +{ + .init = atmel_sha_init, + .update = atmel_sha_update, + .final = atmel_sha_final, + .finup = atmel_sha_finup, + .digest = atmel_sha_digest, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .base = { + .cra_name = "sha384", + .cra_driver_name = "atmel-sha384", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct atmel_sha_ctx), + .cra_alignmask = 0x3, + .cra_module = THIS_MODULE, + .cra_init = atmel_sha_cra_init, + .cra_exit = atmel_sha_cra_exit, + } + } +}, +{ + .init = atmel_sha_init, + .update = atmel_sha_update, + .final = atmel_sha_final, + .finup = atmel_sha_finup, + .digest = atmel_sha_digest, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .base = { + .cra_name = "sha512", + .cra_driver_name = "atmel-sha512", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct atmel_sha_ctx), + .cra_alignmask = 0x3, + .cra_module = THIS_MODULE, + .cra_init = atmel_sha_cra_init, + .cra_exit = atmel_sha_cra_exit, + } + } +}, +}; + static void atmel_sha_done_task(unsigned long data) { struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; @@ -941,32 +1194,142 @@ static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd) { int i; - for (i = 0; i < ARRAY_SIZE(sha_algs); i++) - crypto_unregister_ahash(&sha_algs[i]); + for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) + crypto_unregister_ahash(&sha_1_256_algs[i]); + + if (dd->caps.has_sha224) + crypto_unregister_ahash(&sha_224_alg); + + if (dd->caps.has_sha_384_512) { + for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) + crypto_unregister_ahash(&sha_384_512_algs[i]); + } } static int atmel_sha_register_algs(struct atmel_sha_dev *dd) { int err, i, j; - for (i = 0; i < ARRAY_SIZE(sha_algs); i++) { - err = crypto_register_ahash(&sha_algs[i]); + for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) { + err = crypto_register_ahash(&sha_1_256_algs[i]); if (err) - goto err_sha_algs; + goto err_sha_1_256_algs; + } + + if (dd->caps.has_sha224) { + err = crypto_register_ahash(&sha_224_alg); + if (err) + goto err_sha_224_algs; + } + + if (dd->caps.has_sha_384_512) { + for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) { + err = crypto_register_ahash(&sha_384_512_algs[i]); + if (err) + goto err_sha_384_512_algs; + } } return 0; -err_sha_algs: +err_sha_384_512_algs: + for (j = 0; j < i; j++) + crypto_unregister_ahash(&sha_384_512_algs[j]); + crypto_unregister_ahash(&sha_224_alg); +err_sha_224_algs: + i = ARRAY_SIZE(sha_1_256_algs); +err_sha_1_256_algs: for (j = 0; j < i; j++) - crypto_unregister_ahash(&sha_algs[j]); + crypto_unregister_ahash(&sha_1_256_algs[j]); return err; } +static bool atmel_sha_filter(struct dma_chan *chan, void *slave) +{ + struct at_dma_slave *sl = slave; + + if (sl && sl->dma_dev == chan->device->dev) { + chan->private = sl; + return true; + } else { + return false; + } +} + +static int atmel_sha_dma_init(struct atmel_sha_dev *dd, + struct crypto_platform_data *pdata) +{ + int err = -ENOMEM; + dma_cap_mask_t mask_in; + + if (pdata && pdata->dma_slave->rxdata.dma_dev) { + /* Try to grab DMA channel */ + dma_cap_zero(mask_in); + dma_cap_set(DMA_SLAVE, mask_in); + + dd->dma_lch_in.chan = dma_request_channel(mask_in, + atmel_sha_filter, &pdata->dma_slave->rxdata); + + if (!dd->dma_lch_in.chan) + return err; + + dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; + dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + + SHA_REG_DIN(0); + dd->dma_lch_in.dma_conf.src_maxburst = 1; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = 1; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.device_fc = false; + + return 0; + } + + return -ENODEV; +} + +static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd) +{ + dma_release_channel(dd->dma_lch_in.chan); +} + +static void atmel_sha_get_cap(struct atmel_sha_dev *dd) +{ + + dd->caps.has_dma = 0; + dd->caps.has_dualbuff = 0; + dd->caps.has_sha224 = 0; + dd->caps.has_sha_384_512 = 0; + + /* keep only major version number */ + switch (dd->hw_version & 0xff0) { + case 0x410: + dd->caps.has_dma = 1; + dd->caps.has_dualbuff = 1; + dd->caps.has_sha224 = 1; + dd->caps.has_sha_384_512 = 1; + break; + case 0x400: + dd->caps.has_dma = 1; + dd->caps.has_dualbuff = 1; + dd->caps.has_sha224 = 1; + break; + case 0x320: + break; + default: + dev_warn(dd->dev, + "Unmanaged sha version, set minimum capabilities\n"); + break; + } +} + static int atmel_sha_probe(struct platform_device *pdev) { struct atmel_sha_dev *sha_dd; + struct crypto_platform_data *pdata; struct device *dev = &pdev->dev; struct resource *sha_res; unsigned long sha_phys_size; @@ -1018,7 +1381,7 @@ static int atmel_sha_probe(struct platform_device *pdev) } /* Initializing the clock */ - sha_dd->iclk = clk_get(&pdev->dev, NULL); + sha_dd->iclk = clk_get(&pdev->dev, "sha_clk"); if (IS_ERR(sha_dd->iclk)) { dev_err(dev, "clock intialization failed.\n"); err = PTR_ERR(sha_dd->iclk); @@ -1032,6 +1395,22 @@ static int atmel_sha_probe(struct platform_device *pdev) goto sha_io_err; } + atmel_sha_hw_version_init(sha_dd); + + atmel_sha_get_cap(sha_dd); + + if (sha_dd->caps.has_dma) { + pdata = pdev->dev.platform_data; + if (!pdata) { + dev_err(&pdev->dev, "platform data not available\n"); + err = -ENXIO; + goto err_pdata; + } + err = atmel_sha_dma_init(sha_dd, pdata); + if (err) + goto err_sha_dma; + } + spin_lock(&atmel_sha.lock); list_add_tail(&sha_dd->list, &atmel_sha.dev_list); spin_unlock(&atmel_sha.lock); @@ -1048,6 +1427,10 @@ err_algs: spin_lock(&atmel_sha.lock); list_del(&sha_dd->list); spin_unlock(&atmel_sha.lock); + if (sha_dd->caps.has_dma) + atmel_sha_dma_cleanup(sha_dd); +err_sha_dma: +err_pdata: iounmap(sha_dd->io_base); sha_io_err: clk_put(sha_dd->iclk); @@ -1078,6 +1461,9 @@ static int atmel_sha_remove(struct platform_device *pdev) tasklet_kill(&sha_dd->done_task); + if (sha_dd->caps.has_dma) + atmel_sha_dma_cleanup(sha_dd); + iounmap(sha_dd->io_base); clk_put(sha_dd->iclk); @@ -1102,6 +1488,6 @@ static struct platform_driver atmel_sha_driver = { module_platform_driver(atmel_sha_driver); -MODULE_DESCRIPTION("Atmel SHA1/SHA256 hw acceleration support."); +MODULE_DESCRIPTION("Atmel SHA (1/256/224/384/512) hw acceleration support."); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); -- cgit v0.10.2 From 918731fa2808352387a872799d96f1e293e5f658 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Thu, 21 Feb 2013 11:04:22 -0800 Subject: crypto: crc32c - Update the links to the white papers on CRC32C calculations with PCLMULQDQ instructions. Herbert, The following patch update the stale link to the CRC32C white paper that was referenced. Tim Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S index cf1a7ec..19ec73f 100644 --- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S +++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S @@ -1,9 +1,10 @@ /* * Implement fast CRC32C with PCLMULQDQ instructions. (x86_64) * - * The white paper on CRC32C calculations with PCLMULQDQ instruction can be + * The white papers on CRC32C calculations with PCLMULQDQ instruction can be * downloaded from: - * http://download.intel.com/design/intarch/papers/323405.pdf + * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/crc-iscsi-polynomial-crc32-instruction-paper.pdf + * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-paper.pdf * * Copyright (C) 2012 Intel Corporation. * -- cgit v0.10.2 From a84fb791cb467851772a9196c05531be4abaf1bb Mon Sep 17 00:00:00 2001 From: Mathias Krause Date: Sun, 24 Feb 2013 14:09:12 +0100 Subject: crypto: user - constify netlink dispatch table There is no need to modify the netlink dispatch table at runtime and making it const even makes the resulting object file slightly smaller. Cc: Steffen Klassert Signed-off-by: Mathias Krause Signed-off-by: Herbert Xu diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c index dfd511f..1512e41 100644 --- a/crypto/crypto_user.c +++ b/crypto/crypto_user.c @@ -440,7 +440,7 @@ static const struct nla_policy crypto_policy[CRYPTOCFGA_MAX+1] = { #undef MSGSIZE -static struct crypto_link { +static const struct crypto_link { int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **); int (*dump)(struct sk_buff *, struct netlink_callback *); int (*done)(struct netlink_callback *); @@ -456,7 +456,7 @@ static struct crypto_link { static int crypto_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh) { struct nlattr *attrs[CRYPTOCFGA_MAX+1]; - struct crypto_link *link; + const struct crypto_link *link; int type, err; type = nlh->nlmsg_type; -- cgit v0.10.2 From fb1dd794802022f4413d36e47bd40f8c7bacd456 Mon Sep 17 00:00:00 2001 From: Syam Sidhardhan Date: Mon, 25 Feb 2013 03:57:39 +0530 Subject: crypto: bfin_crc - Fix possible NULL pointer dereference If we define dev_dbg(), then there is a possible NULL pointer dereference. Signed-off-by: Syam Sidhardhan Signed-off-by: Herbert Xu diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c index 827913d..d797f31 100644 --- a/drivers/crypto/bfin_crc.c +++ b/drivers/crypto/bfin_crc.c @@ -151,7 +151,7 @@ static int bfin_crypto_crc_init(struct ahash_request *req) struct bfin_crypto_crc_reqctx *ctx = ahash_request_ctx(req); struct bfin_crypto_crc *crc; - dev_dbg(crc->dev, "crc_init\n"); + dev_dbg(ctx->crc->dev, "crc_init\n"); spin_lock_bh(&crc_list.lock); list_for_each_entry(crc, &crc_list.dev_list, list) { crc_ctx->crc = crc; @@ -160,7 +160,7 @@ static int bfin_crypto_crc_init(struct ahash_request *req) spin_unlock_bh(&crc_list.lock); if (sg_count(req->src) > CRC_MAX_DMA_DESC) { - dev_dbg(crc->dev, "init: requested sg list is too big > %d\n", + dev_dbg(ctx->crc->dev, "init: requested sg list is too big > %d\n", CRC_MAX_DMA_DESC); return -EINVAL; } @@ -175,7 +175,7 @@ static int bfin_crypto_crc_init(struct ahash_request *req) /* init crc results */ put_unaligned_le32(crc_ctx->key, req->result); - dev_dbg(crc->dev, "init: digest size: %d\n", + dev_dbg(ctx->crc->dev, "init: digest size: %d\n", crypto_ahash_digestsize(tfm)); return bfin_crypto_crc_init_hw(crc, crc_ctx->key); -- cgit v0.10.2 From e68af48251ffb2a8aad4664ea68e0363893dda26 Mon Sep 17 00:00:00 2001 From: Joel A Fernandes Date: Tue, 26 Feb 2013 10:04:31 -0600 Subject: crypto: omap-sham - Use pm_runtime_put instead of pm_runtime_put_sync in tasklet After DMA is complete, the omap_sham_finish_req function is called as a part of the done_task tasklet. During this its atomic and any calls to pm functions should not assume they wont sleep. The patch replaces a call to pm_runtime_put_sync (which can sleep) with pm_runtime_put thus fixing a kernel panic observed on AM33xx SoC during SHA operation. Tested on an AM33xx SoC device (beaglebone board). To reproduce the problem, used the tcrypt kernel module as: modprobe tcrypt sec=2 mode=403 Signed-off-by: Joel A Fernandes Cc: David S. Miller Acked-by: Mark A. Greer Signed-off-by: Herbert Xu diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 3d1611f..1cfdea1 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -923,7 +923,7 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY)); - pm_runtime_put_sync(dd->dev); + pm_runtime_put(dd->dev); if (req->base.complete) req->base.complete(&req->base, err); -- cgit v0.10.2 From bbbaa37428abac025fe11b83933473888ddf6fc1 Mon Sep 17 00:00:00 2001 From: Joel A Fernandes Date: Tue, 26 Feb 2013 10:04:32 -0600 Subject: crypto: omap-aes - Use pm_runtime_put instead of pm_runtime_put_sync in tasklet After DMA is complete, the omap_aes_finish_req function is called as a part of the done_task tasklet. During this its atomic and any calls to pm functions should not assume they wont sleep. The patch replaces a call to pm_runtime_put_sync (which can sleep) with pm_runtime_put thus fixing a kernel panic observed on AM33xx SoC during AES operation. Tested on an AM33xx SoC device (beaglebone board). To reproduce the problem, I used the tcrypt kernel module as: modprobe tcrypt sec=2 mode=500 Signed-off-by: Joel A Fernandes Cc: David S. Miller Acked-by: Mark A. Greer Signed-off-by: Herbert Xu diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 6aa425f..1e5a17a 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -636,7 +636,7 @@ static void omap_aes_finish_req(struct omap_aes_dev *dd, int err) pr_debug("err: %d\n", err); - pm_runtime_put_sync(dd->dev); + pm_runtime_put(dd->dev); dd->flags &= ~FLAGS_BUSY; req->base.complete(&req->base, err); -- cgit v0.10.2 From 94e51df9d60c818a7974f545fd32f6690a9605d9 Mon Sep 17 00:00:00 2001 From: Sachin Kamat Date: Mon, 4 Mar 2013 15:09:42 +0530 Subject: crypto: omap-aes - Use module_platform_driver macro module_platform_driver() makes the code simpler by eliminating boilerplate code. Signed-off-by: Sachin Kamat Signed-off-by: Herbert Xu diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 1e5a17a..ee15b0f 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1248,18 +1248,7 @@ static struct platform_driver omap_aes_driver = { }, }; -static int __init omap_aes_mod_init(void) -{ - return platform_driver_register(&omap_aes_driver); -} - -static void __exit omap_aes_mod_exit(void) -{ - platform_driver_unregister(&omap_aes_driver); -} - -module_init(omap_aes_mod_init); -module_exit(omap_aes_mod_exit); +module_platform_driver(omap_aes_driver); MODULE_DESCRIPTION("OMAP AES hw acceleration support."); MODULE_LICENSE("GPL v2"); -- cgit v0.10.2 From 0261370268193755c5ff1bef3d9d6339f314052c Mon Sep 17 00:00:00 2001 From: Sachin Kamat Date: Mon, 4 Mar 2013 15:09:43 +0530 Subject: crypto: omap-sham - Use module_platform_driver macro module_platform_driver() makes the code simpler by eliminating boilerplate code. Signed-off-by: Sachin Kamat Signed-off-by: Herbert Xu diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 1cfdea1..a1e1b47 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -1813,18 +1813,7 @@ static struct platform_driver omap_sham_driver = { }, }; -static int __init omap_sham_mod_init(void) -{ - return platform_driver_register(&omap_sham_driver); -} - -static void __exit omap_sham_mod_exit(void) -{ - platform_driver_unregister(&omap_sham_driver); -} - -module_init(omap_sham_mod_init); -module_exit(omap_sham_mod_exit); +module_platform_driver(omap_sham_driver); MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support."); MODULE_LICENSE("GPL v2"); -- cgit v0.10.2 From ae8488a507357fd4fd2c825ac423b39ea1041353 Mon Sep 17 00:00:00 2001 From: Paul Bolle Date: Tue, 5 Mar 2013 14:33:16 +0100 Subject: crypto: caam - fix typo "CRYPTO_AHASH" The Kconfig entry for CAAM's hash algorithm implementations has always selected CRYPTO_AHASH. But there's no corresponding Kconfig symbol. It seems it was intended to select CRYPTO_HASH, like other crypto drivers do. That would apparently (indirectly) select CRYPTO_HASH2, which would enable the ahash functionality this driver uses. Signed-off-by: Paul Bolle Reviewed-by: Kim Phillips Signed-off-by: Herbert Xu diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index 65c7668..b44091c 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -78,7 +78,7 @@ config CRYPTO_DEV_FSL_CAAM_AHASH_API tristate "Register hash algorithm implementations with Crypto API" depends on CRYPTO_DEV_FSL_CAAM default y - select CRYPTO_AHASH + select CRYPTO_HASH help Selecting this will offload ahash for users of the scatterlist crypto API to the SEC4 via job ring. -- cgit v0.10.2 From 6375bcf7867fb18241fda47bfaeec85b925221f3 Mon Sep 17 00:00:00 2001 From: Tang Chen Date: Thu, 7 Mar 2013 18:38:17 +0800 Subject: hwrng: Fix a wrong comment in Documentation/hw_random.txt Seeing from the comment, there should be three reasons for removing request_mem_region. Change the comment "two" to "three". Signed-off-by: Tang Chen Acked-by: Rob Landley Signed-off-by: Herbert Xu diff --git a/Documentation/hw_random.txt b/Documentation/hw_random.txt index 690f525..026e237 100644 --- a/Documentation/hw_random.txt +++ b/Documentation/hw_random.txt @@ -63,7 +63,7 @@ Intel RNG Driver notes: * FIXME: support poll(2) - NOTE: request_mem_region was removed, for two reasons: + NOTE: request_mem_region was removed, for three reasons: 1) Only one RNG is supported by this driver, 2) The location used by the RNG is a fixed location in MMIO-addressable memory, 3) users with properly working BIOS e820 handling will always -- cgit v0.10.2 From 5de8875281e1db024d67cbd5c792264194bfca2a Mon Sep 17 00:00:00 2001 From: Javier Martin Date: Fri, 1 Mar 2013 12:37:53 +0100 Subject: crypto: sahara - Add driver for SAHARA2 accelerator. SAHARA2 HW module is included in the i.MX27 SoC from Freescale. It is capable of performing cipher algorithms such as AES, 3DES..., hashing and RNG too. This driver provides support for AES-CBC and AES-ECB by now. Reviewed-by: Arnd Bergmann Signed-off-by: Javier Martin Signed-off-by: Herbert Xu diff --git a/Documentation/devicetree/bindings/crypto/fsl-imx-sahara.txt b/Documentation/devicetree/bindings/crypto/fsl-imx-sahara.txt new file mode 100644 index 0000000..5c65ecc --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/fsl-imx-sahara.txt @@ -0,0 +1,15 @@ +Freescale SAHARA Cryptographic Accelerator included in some i.MX chips. +Currently only i.MX27 is supported. + +Required properties: +- compatible : Should be "fsl,-sahara" +- reg : Should contain SAHARA registers location and length +- interrupts : Should contain SAHARA interrupt number + +Example: + +sah@10025000 { + compatible = "fsl,imx27-sahara"; + reg = < 0x10025000 0x800>; + interrupts = <75>; +}; diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index e66fb0a..dffb855 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -276,6 +276,16 @@ config CRYPTO_DEV_PICOXCELL Saying m here will build a module named pipcoxcell_crypto. +config CRYPTO_DEV_SAHARA + tristate "Support for SAHARA crypto accelerator" + depends on ARCH_MXC && EXPERIMENTAL && OF + select CRYPTO_BLKCIPHER + select CRYPTO_AES + select CRYPTO_ECB + help + This option enables support for the SAHARA HW crypto accelerator + found in some Freescale i.MX chips. + config CRYPTO_DEV_S5P tristate "Support for Samsung S5PV210 crypto accelerator" depends on ARCH_S5PV210 diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 880a47b..38ce13d 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes.o obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o +obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o obj-$(CONFIG_CRYPTO_DEV_TEGRA_AES) += tegra-aes.o obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c new file mode 100644 index 0000000..a97bb6c --- /dev/null +++ b/drivers/crypto/sahara.c @@ -0,0 +1,1070 @@ +/* + * Cryptographic API. + * + * Support for SAHARA cryptographic accelerator. + * + * Copyright (c) 2013 Vista Silicon S.L. + * Author: Javier Martin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * Based on omap-aes.c and tegra-aes.c + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SAHARA_NAME "sahara" +#define SAHARA_VERSION_3 3 +#define SAHARA_TIMEOUT_MS 1000 +#define SAHARA_MAX_HW_DESC 2 +#define SAHARA_MAX_HW_LINK 20 + +#define FLAGS_MODE_MASK 0x000f +#define FLAGS_ENCRYPT BIT(0) +#define FLAGS_CBC BIT(1) +#define FLAGS_NEW_KEY BIT(3) +#define FLAGS_BUSY 4 + +#define SAHARA_HDR_BASE 0x00800000 +#define SAHARA_HDR_SKHA_ALG_AES 0 +#define SAHARA_HDR_SKHA_OP_ENC (1 << 2) +#define SAHARA_HDR_SKHA_MODE_ECB (0 << 3) +#define SAHARA_HDR_SKHA_MODE_CBC (1 << 3) +#define SAHARA_HDR_FORM_DATA (5 << 16) +#define SAHARA_HDR_FORM_KEY (8 << 16) +#define SAHARA_HDR_LLO (1 << 24) +#define SAHARA_HDR_CHA_SKHA (1 << 28) +#define SAHARA_HDR_CHA_MDHA (2 << 28) +#define SAHARA_HDR_PARITY_BIT (1 << 31) + +/* SAHARA can only process one request at a time */ +#define SAHARA_QUEUE_LENGTH 1 + +#define SAHARA_REG_VERSION 0x00 +#define SAHARA_REG_DAR 0x04 +#define SAHARA_REG_CONTROL 0x08 +#define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24) +#define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16) +#define SAHARA_CONTROL_RNG_AUTORSD (1 << 7) +#define SAHARA_CONTROL_ENABLE_INT (1 << 4) +#define SAHARA_REG_CMD 0x0C +#define SAHARA_CMD_RESET (1 << 0) +#define SAHARA_CMD_CLEAR_INT (1 << 8) +#define SAHARA_CMD_CLEAR_ERR (1 << 9) +#define SAHARA_CMD_SINGLE_STEP (1 << 10) +#define SAHARA_CMD_MODE_BATCH (1 << 16) +#define SAHARA_CMD_MODE_DEBUG (1 << 18) +#define SAHARA_REG_STATUS 0x10 +#define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7) +#define SAHARA_STATE_IDLE 0 +#define SAHARA_STATE_BUSY 1 +#define SAHARA_STATE_ERR 2 +#define SAHARA_STATE_FAULT 3 +#define SAHARA_STATE_COMPLETE 4 +#define SAHARA_STATE_COMP_FLAG (1 << 2) +#define SAHARA_STATUS_DAR_FULL (1 << 3) +#define SAHARA_STATUS_ERROR (1 << 4) +#define SAHARA_STATUS_SECURE (1 << 5) +#define SAHARA_STATUS_FAIL (1 << 6) +#define SAHARA_STATUS_INIT (1 << 7) +#define SAHARA_STATUS_RNG_RESEED (1 << 8) +#define SAHARA_STATUS_ACTIVE_RNG (1 << 9) +#define SAHARA_STATUS_ACTIVE_MDHA (1 << 10) +#define SAHARA_STATUS_ACTIVE_SKHA (1 << 11) +#define SAHARA_STATUS_MODE_BATCH (1 << 16) +#define SAHARA_STATUS_MODE_DEDICATED (1 << 17) +#define SAHARA_STATUS_MODE_DEBUG (1 << 18) +#define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff) +#define SAHARA_REG_ERRSTATUS 0x14 +#define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf) +#define SAHARA_ERRSOURCE_CHA 14 +#define SAHARA_ERRSOURCE_DMA 15 +#define SAHARA_ERRSTATUS_DMA_DIR (1 << 8) +#define SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3) +#define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7) +#define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff) +#define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3) +#define SAHARA_REG_FADDR 0x18 +#define SAHARA_REG_CDAR 0x1C +#define SAHARA_REG_IDAR 0x20 + +struct sahara_hw_desc { + u32 hdr; + u32 len1; + dma_addr_t p1; + u32 len2; + dma_addr_t p2; + dma_addr_t next; +}; + +struct sahara_hw_link { + u32 len; + dma_addr_t p; + dma_addr_t next; +}; + +struct sahara_ctx { + struct sahara_dev *dev; + unsigned long flags; + int keylen; + u8 key[AES_KEYSIZE_128]; + struct crypto_ablkcipher *fallback; +}; + +struct sahara_aes_reqctx { + unsigned long mode; +}; + +struct sahara_dev { + struct device *device; + void __iomem *regs_base; + struct clk *clk_ipg; + struct clk *clk_ahb; + + struct sahara_ctx *ctx; + spinlock_t lock; + struct crypto_queue queue; + unsigned long flags; + + struct tasklet_struct done_task; + struct tasklet_struct queue_task; + + struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC]; + dma_addr_t hw_phys_desc[SAHARA_MAX_HW_DESC]; + + u8 *key_base; + dma_addr_t key_phys_base; + + u8 *iv_base; + dma_addr_t iv_phys_base; + + struct sahara_hw_link *hw_link[SAHARA_MAX_HW_LINK]; + dma_addr_t hw_phys_link[SAHARA_MAX_HW_LINK]; + + struct ablkcipher_request *req; + size_t total; + struct scatterlist *in_sg; + unsigned int nb_in_sg; + struct scatterlist *out_sg; + unsigned int nb_out_sg; + + u32 error; + struct timer_list watchdog; +}; + +static struct sahara_dev *dev_ptr; + +static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg) +{ + writel(data, dev->regs_base + reg); +} + +static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg) +{ + return readl(dev->regs_base + reg); +} + +static u32 sahara_aes_key_hdr(struct sahara_dev *dev) +{ + u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES | + SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO | + SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; + + if (dev->flags & FLAGS_CBC) { + hdr |= SAHARA_HDR_SKHA_MODE_CBC; + hdr ^= SAHARA_HDR_PARITY_BIT; + } + + if (dev->flags & FLAGS_ENCRYPT) { + hdr |= SAHARA_HDR_SKHA_OP_ENC; + hdr ^= SAHARA_HDR_PARITY_BIT; + } + + return hdr; +} + +static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev) +{ + return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA | + SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; +} + +static int sahara_sg_length(struct scatterlist *sg, + unsigned int total) +{ + int sg_nb; + unsigned int len; + struct scatterlist *sg_list; + + sg_nb = 0; + sg_list = sg; + + while (total) { + len = min(sg_list->length, total); + + sg_nb++; + total -= len; + + sg_list = sg_next(sg_list); + if (!sg_list) + total = 0; + } + + return sg_nb; +} + +static char *sahara_err_src[16] = { + "No error", + "Header error", + "Descriptor length error", + "Descriptor length or pointer error", + "Link length error", + "Link pointer error", + "Input buffer error", + "Output buffer error", + "Output buffer starvation", + "Internal state fault", + "General descriptor problem", + "Reserved", + "Descriptor address error", + "Link address error", + "CHA error", + "DMA error" +}; + +static char *sahara_err_dmasize[4] = { + "Byte transfer", + "Half-word transfer", + "Word transfer", + "Reserved" +}; + +static char *sahara_err_dmasrc[8] = { + "No error", + "AHB bus error", + "Internal IP bus error", + "Parity error", + "DMA crosses 256 byte boundary", + "DMA is busy", + "Reserved", + "DMA HW error" +}; + +static char *sahara_cha_errsrc[12] = { + "Input buffer non-empty", + "Illegal address", + "Illegal mode", + "Illegal data size", + "Illegal key size", + "Write during processing", + "CTX read during processing", + "HW error", + "Input buffer disabled/underflow", + "Output buffer disabled/overflow", + "DES key parity error", + "Reserved" +}; + +static char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" }; + +static void sahara_decode_error(struct sahara_dev *dev, unsigned int error) +{ + u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error); + u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error)); + + dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error); + + dev_err(dev->device, " - %s.\n", sahara_err_src[source]); + + if (source == SAHARA_ERRSOURCE_DMA) { + if (error & SAHARA_ERRSTATUS_DMA_DIR) + dev_err(dev->device, " * DMA read.\n"); + else + dev_err(dev->device, " * DMA write.\n"); + + dev_err(dev->device, " * %s.\n", + sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]); + dev_err(dev->device, " * %s.\n", + sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]); + } else if (source == SAHARA_ERRSOURCE_CHA) { + dev_err(dev->device, " * %s.\n", + sahara_cha_errsrc[chasrc]); + dev_err(dev->device, " * %s.\n", + sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]); + } + dev_err(dev->device, "\n"); +} + +static char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" }; + +static void sahara_decode_status(struct sahara_dev *dev, unsigned int status) +{ + u8 state; + + if (!IS_ENABLED(DEBUG)) + return; + + state = SAHARA_STATUS_GET_STATE(status); + + dev_dbg(dev->device, "%s: Status Register = 0x%08x\n", + __func__, status); + + dev_dbg(dev->device, " - State = %d:\n", state); + if (state & SAHARA_STATE_COMP_FLAG) + dev_dbg(dev->device, " * Descriptor completed. IRQ pending.\n"); + + dev_dbg(dev->device, " * %s.\n", + sahara_state[state & ~SAHARA_STATE_COMP_FLAG]); + + if (status & SAHARA_STATUS_DAR_FULL) + dev_dbg(dev->device, " - DAR Full.\n"); + if (status & SAHARA_STATUS_ERROR) + dev_dbg(dev->device, " - Error.\n"); + if (status & SAHARA_STATUS_SECURE) + dev_dbg(dev->device, " - Secure.\n"); + if (status & SAHARA_STATUS_FAIL) + dev_dbg(dev->device, " - Fail.\n"); + if (status & SAHARA_STATUS_RNG_RESEED) + dev_dbg(dev->device, " - RNG Reseed Request.\n"); + if (status & SAHARA_STATUS_ACTIVE_RNG) + dev_dbg(dev->device, " - RNG Active.\n"); + if (status & SAHARA_STATUS_ACTIVE_MDHA) + dev_dbg(dev->device, " - MDHA Active.\n"); + if (status & SAHARA_STATUS_ACTIVE_SKHA) + dev_dbg(dev->device, " - SKHA Active.\n"); + + if (status & SAHARA_STATUS_MODE_BATCH) + dev_dbg(dev->device, " - Batch Mode.\n"); + else if (status & SAHARA_STATUS_MODE_DEDICATED) + dev_dbg(dev->device, " - Decidated Mode.\n"); + else if (status & SAHARA_STATUS_MODE_DEBUG) + dev_dbg(dev->device, " - Debug Mode.\n"); + + dev_dbg(dev->device, " - Internal state = 0x%02x\n", + SAHARA_STATUS_GET_ISTATE(status)); + + dev_dbg(dev->device, "Current DAR: 0x%08x\n", + sahara_read(dev, SAHARA_REG_CDAR)); + dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n", + sahara_read(dev, SAHARA_REG_IDAR)); +} + +static void sahara_dump_descriptors(struct sahara_dev *dev) +{ + int i; + + if (!IS_ENABLED(DEBUG)) + return; + + for (i = 0; i < SAHARA_MAX_HW_DESC; i++) { + dev_dbg(dev->device, "Descriptor (%d) (0x%08x):\n", + i, dev->hw_phys_desc[i]); + dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr); + dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1); + dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1); + dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2); + dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2); + dev_dbg(dev->device, "\tnext = 0x%08x\n", + dev->hw_desc[i]->next); + } + dev_dbg(dev->device, "\n"); +} + +static void sahara_dump_links(struct sahara_dev *dev) +{ + int i; + + if (!IS_ENABLED(DEBUG)) + return; + + for (i = 0; i < SAHARA_MAX_HW_LINK; i++) { + dev_dbg(dev->device, "Link (%d) (0x%08x):\n", + i, dev->hw_phys_link[i]); + dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len); + dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p); + dev_dbg(dev->device, "\tnext = 0x%08x\n", + dev->hw_link[i]->next); + } + dev_dbg(dev->device, "\n"); +} + +static void sahara_aes_done_task(unsigned long data) +{ + struct sahara_dev *dev = (struct sahara_dev *)data; + + dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, + DMA_TO_DEVICE); + dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_FROM_DEVICE); + + spin_lock(&dev->lock); + clear_bit(FLAGS_BUSY, &dev->flags); + spin_unlock(&dev->lock); + + dev->req->base.complete(&dev->req->base, dev->error); +} + +void sahara_watchdog(unsigned long data) +{ + struct sahara_dev *dev = (struct sahara_dev *)data; + unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS); + unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS); + + sahara_decode_status(dev, stat); + sahara_decode_error(dev, err); + dev->error = -ETIMEDOUT; + sahara_aes_done_task(data); +} + +static int sahara_hw_descriptor_create(struct sahara_dev *dev) +{ + struct sahara_ctx *ctx = dev->ctx; + struct scatterlist *sg; + int ret; + int i, j; + + /* Copy new key if necessary */ + if (ctx->flags & FLAGS_NEW_KEY) { + memcpy(dev->key_base, ctx->key, ctx->keylen); + ctx->flags &= ~FLAGS_NEW_KEY; + + if (dev->flags & FLAGS_CBC) { + dev->hw_desc[0]->len1 = AES_BLOCK_SIZE; + dev->hw_desc[0]->p1 = dev->iv_phys_base; + } else { + dev->hw_desc[0]->len1 = 0; + dev->hw_desc[0]->p1 = 0; + } + dev->hw_desc[0]->len2 = ctx->keylen; + dev->hw_desc[0]->p2 = dev->key_phys_base; + dev->hw_desc[0]->next = dev->hw_phys_desc[1]; + } + dev->hw_desc[0]->hdr = sahara_aes_key_hdr(dev); + + dev->nb_in_sg = sahara_sg_length(dev->in_sg, dev->total); + dev->nb_out_sg = sahara_sg_length(dev->out_sg, dev->total); + if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) { + dev_err(dev->device, "not enough hw links (%d)\n", + dev->nb_in_sg + dev->nb_out_sg); + return -EINVAL; + } + + ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_TO_DEVICE); + if (ret != dev->nb_in_sg) { + dev_err(dev->device, "couldn't map in sg\n"); + goto unmap_in; + } + ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg, + DMA_FROM_DEVICE); + if (ret != dev->nb_out_sg) { + dev_err(dev->device, "couldn't map out sg\n"); + goto unmap_out; + } + + /* Create input links */ + dev->hw_desc[1]->p1 = dev->hw_phys_link[0]; + sg = dev->in_sg; + for (i = 0; i < dev->nb_in_sg; i++) { + dev->hw_link[i]->len = sg->length; + dev->hw_link[i]->p = sg->dma_address; + if (i == (dev->nb_in_sg - 1)) { + dev->hw_link[i]->next = 0; + } else { + dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; + sg = sg_next(sg); + } + } + + /* Create output links */ + dev->hw_desc[1]->p2 = dev->hw_phys_link[i]; + sg = dev->out_sg; + for (j = i; j < dev->nb_out_sg + i; j++) { + dev->hw_link[j]->len = sg->length; + dev->hw_link[j]->p = sg->dma_address; + if (j == (dev->nb_out_sg + i - 1)) { + dev->hw_link[j]->next = 0; + } else { + dev->hw_link[j]->next = dev->hw_phys_link[j + 1]; + sg = sg_next(sg); + } + } + + /* Fill remaining fields of hw_desc[1] */ + dev->hw_desc[1]->hdr = sahara_aes_data_link_hdr(dev); + dev->hw_desc[1]->len1 = dev->total; + dev->hw_desc[1]->len2 = dev->total; + dev->hw_desc[1]->next = 0; + + sahara_dump_descriptors(dev); + sahara_dump_links(dev); + + /* Start processing descriptor chain. */ + mod_timer(&dev->watchdog, + jiffies + msecs_to_jiffies(SAHARA_TIMEOUT_MS)); + sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR); + + return 0; + +unmap_out: + dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, + DMA_TO_DEVICE); +unmap_in: + dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_FROM_DEVICE); + + return -EINVAL; +} + +static void sahara_aes_queue_task(unsigned long data) +{ + struct sahara_dev *dev = (struct sahara_dev *)data; + struct crypto_async_request *async_req, *backlog; + struct sahara_ctx *ctx; + struct sahara_aes_reqctx *rctx; + struct ablkcipher_request *req; + int ret; + + spin_lock(&dev->lock); + backlog = crypto_get_backlog(&dev->queue); + async_req = crypto_dequeue_request(&dev->queue); + if (!async_req) + clear_bit(FLAGS_BUSY, &dev->flags); + spin_unlock(&dev->lock); + + if (!async_req) + return; + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + req = ablkcipher_request_cast(async_req); + + /* Request is ready to be dispatched by the device */ + dev_dbg(dev->device, + "dispatch request (nbytes=%d, src=%p, dst=%p)\n", + req->nbytes, req->src, req->dst); + + /* assign new request to device */ + dev->req = req; + dev->total = req->nbytes; + dev->in_sg = req->src; + dev->out_sg = req->dst; + + rctx = ablkcipher_request_ctx(req); + ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); + rctx->mode &= FLAGS_MODE_MASK; + dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode; + + if ((dev->flags & FLAGS_CBC) && req->info) + memcpy(dev->iv_base, req->info, AES_KEYSIZE_128); + + /* assign new context to device */ + ctx->dev = dev; + dev->ctx = ctx; + + ret = sahara_hw_descriptor_create(dev); + if (ret < 0) { + spin_lock(&dev->lock); + clear_bit(FLAGS_BUSY, &dev->flags); + spin_unlock(&dev->lock); + dev->req->base.complete(&dev->req->base, ret); + } +} + +static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm); + int ret; + + ctx->keylen = keylen; + + /* SAHARA only supports 128bit keys */ + if (keylen == AES_KEYSIZE_128) { + memcpy(ctx->key, key, keylen); + ctx->flags |= FLAGS_NEW_KEY; + return 0; + } + + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) + return -EINVAL; + + /* + * The requested key size is not supported by HW, do a fallback. + */ + ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; + ctx->fallback->base.crt_flags |= + (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen); + if (ret) { + struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm); + + tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm_aux->crt_flags |= + (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK); + } + return ret; +} + +static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode) +{ + struct sahara_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req); + struct sahara_dev *dev = dev_ptr; + int err = 0; + int busy; + + dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n", + req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC)); + + if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { + dev_err(dev->device, + "request size is not exact amount of AES blocks\n"); + return -EINVAL; + } + + ctx->dev = dev; + + rctx->mode = mode; + spin_lock_bh(&dev->lock); + err = ablkcipher_enqueue_request(&dev->queue, req); + busy = test_and_set_bit(FLAGS_BUSY, &dev->flags); + spin_unlock_bh(&dev->lock); + + if (!busy) + tasklet_schedule(&dev->queue_task); + + return err; +} + +static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req) +{ + struct crypto_tfm *tfm = + crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); + struct sahara_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + int err; + + if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { + ablkcipher_request_set_tfm(req, ctx->fallback); + err = crypto_ablkcipher_encrypt(req); + ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + return err; + } + + return sahara_aes_crypt(req, FLAGS_ENCRYPT); +} + +static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req) +{ + struct crypto_tfm *tfm = + crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); + struct sahara_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + int err; + + if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { + ablkcipher_request_set_tfm(req, ctx->fallback); + err = crypto_ablkcipher_decrypt(req); + ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + return err; + } + + return sahara_aes_crypt(req, 0); +} + +static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req) +{ + struct crypto_tfm *tfm = + crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); + struct sahara_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + int err; + + if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { + ablkcipher_request_set_tfm(req, ctx->fallback); + err = crypto_ablkcipher_encrypt(req); + ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + return err; + } + + return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); +} + +static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req) +{ + struct crypto_tfm *tfm = + crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); + struct sahara_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + int err; + + if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { + ablkcipher_request_set_tfm(req, ctx->fallback); + err = crypto_ablkcipher_decrypt(req); + ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + return err; + } + + return sahara_aes_crypt(req, FLAGS_CBC); +} + +static int sahara_aes_cra_init(struct crypto_tfm *tfm) +{ + const char *name = tfm->__crt_alg->cra_name; + struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->fallback = crypto_alloc_ablkcipher(name, 0, + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback)) { + pr_err("Error allocating fallback algo %s\n", name); + return PTR_ERR(ctx->fallback); + } + + tfm->crt_ablkcipher.reqsize = sizeof(struct sahara_aes_reqctx); + + return 0; +} + +static void sahara_aes_cra_exit(struct crypto_tfm *tfm) +{ + struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->fallback) + crypto_free_ablkcipher(ctx->fallback); + ctx->fallback = NULL; +} + +static struct crypto_alg aes_algs[] = { +{ + .cra_name = "ecb(aes)", + .cra_driver_name = "sahara-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sahara_ctx), + .cra_alignmask = 0x0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = sahara_aes_cra_init, + .cra_exit = sahara_aes_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE , + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = sahara_aes_setkey, + .encrypt = sahara_aes_ecb_encrypt, + .decrypt = sahara_aes_ecb_decrypt, + } +}, { + .cra_name = "cbc(aes)", + .cra_driver_name = "sahara-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sahara_ctx), + .cra_alignmask = 0x0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = sahara_aes_cra_init, + .cra_exit = sahara_aes_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE , + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = sahara_aes_setkey, + .encrypt = sahara_aes_cbc_encrypt, + .decrypt = sahara_aes_cbc_decrypt, + } +} +}; + +static irqreturn_t sahara_irq_handler(int irq, void *data) +{ + struct sahara_dev *dev = (struct sahara_dev *)data; + unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS); + unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS); + + del_timer(&dev->watchdog); + + sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR, + SAHARA_REG_CMD); + + sahara_decode_status(dev, stat); + + if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) { + return IRQ_NONE; + } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) { + dev->error = 0; + } else { + sahara_decode_error(dev, err); + dev->error = -EINVAL; + } + + tasklet_schedule(&dev->done_task); + + return IRQ_HANDLED; +} + + +static int sahara_register_algs(struct sahara_dev *dev) +{ + int err, i, j; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + INIT_LIST_HEAD(&aes_algs[i].cra_list); + err = crypto_register_alg(&aes_algs[i]); + if (err) + goto err_aes_algs; + } + + return 0; + +err_aes_algs: + for (j = 0; j < i; j++) + crypto_unregister_alg(&aes_algs[j]); + + return err; +} + +static void sahara_unregister_algs(struct sahara_dev *dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) + crypto_unregister_alg(&aes_algs[i]); +} + +static struct platform_device_id sahara_platform_ids[] = { + { .name = "sahara-imx27" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(platform, sahara_platform_ids); + +static struct of_device_id sahara_dt_ids[] = { + { .compatible = "fsl,imx27-sahara" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(platform, sahara_dt_ids); + +static int sahara_probe(struct platform_device *pdev) +{ + struct sahara_dev *dev; + struct resource *res; + u32 version; + int irq; + int err; + int i; + + dev = devm_kzalloc(&pdev->dev, sizeof(struct sahara_dev), GFP_KERNEL); + if (dev == NULL) { + dev_err(&pdev->dev, "unable to alloc data struct.\n"); + return -ENOMEM; + } + + dev->device = &pdev->dev; + platform_set_drvdata(pdev, dev); + + /* Get the base address */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(&pdev->dev, "failed to get memory region resource\n"); + return -ENODEV; + } + + if (devm_request_mem_region(&pdev->dev, res->start, + resource_size(res), SAHARA_NAME) == NULL) { + dev_err(&pdev->dev, "failed to request memory region\n"); + return -ENOENT; + } + dev->regs_base = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!dev->regs_base) { + dev_err(&pdev->dev, "failed to ioremap address region\n"); + return -ENOENT; + } + + /* Get the IRQ */ + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(&pdev->dev, "failed to get irq resource\n"); + return irq; + } + + if (devm_request_irq(&pdev->dev, irq, sahara_irq_handler, + 0, SAHARA_NAME, dev) < 0) { + dev_err(&pdev->dev, "failed to request irq\n"); + return -ENOENT; + } + + /* clocks */ + dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); + if (IS_ERR(dev->clk_ipg)) { + dev_err(&pdev->dev, "Could not get ipg clock\n"); + return PTR_ERR(dev->clk_ipg); + } + + dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb"); + if (IS_ERR(dev->clk_ahb)) { + dev_err(&pdev->dev, "Could not get ahb clock\n"); + return PTR_ERR(dev->clk_ahb); + } + + /* Allocate HW descriptors */ + dev->hw_desc[0] = dma_alloc_coherent(&pdev->dev, + SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), + &dev->hw_phys_desc[0], GFP_KERNEL); + if (!dev->hw_desc[0]) { + dev_err(&pdev->dev, "Could not allocate hw descriptors\n"); + return -ENOMEM; + } + dev->hw_desc[1] = dev->hw_desc[0] + 1; + dev->hw_phys_desc[1] = dev->hw_phys_desc[0] + + sizeof(struct sahara_hw_desc); + + /* Allocate space for iv and key */ + dev->key_base = dma_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, + &dev->key_phys_base, GFP_KERNEL); + if (!dev->key_base) { + dev_err(&pdev->dev, "Could not allocate memory for key\n"); + err = -ENOMEM; + goto err_key; + } + dev->iv_base = dev->key_base + AES_KEYSIZE_128; + dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128; + + /* Allocate space for HW links */ + dev->hw_link[0] = dma_alloc_coherent(&pdev->dev, + SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), + &dev->hw_phys_link[0], GFP_KERNEL); + if (!dev->hw_link) { + dev_err(&pdev->dev, "Could not allocate hw links\n"); + err = -ENOMEM; + goto err_link; + } + for (i = 1; i < SAHARA_MAX_HW_LINK; i++) { + dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] + + sizeof(struct sahara_hw_link); + dev->hw_link[i] = dev->hw_link[i - 1] + 1; + } + + crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH); + + dev_ptr = dev; + + tasklet_init(&dev->queue_task, sahara_aes_queue_task, + (unsigned long)dev); + tasklet_init(&dev->done_task, sahara_aes_done_task, + (unsigned long)dev); + + init_timer(&dev->watchdog); + dev->watchdog.function = &sahara_watchdog; + dev->watchdog.data = (unsigned long)dev; + + clk_prepare_enable(dev->clk_ipg); + clk_prepare_enable(dev->clk_ahb); + + version = sahara_read(dev, SAHARA_REG_VERSION); + if (version != SAHARA_VERSION_3) { + dev_err(&pdev->dev, "SAHARA version %d not supported\n", + version); + err = -ENODEV; + goto err_algs; + } + + sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH, + SAHARA_REG_CMD); + sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) | + SAHARA_CONTROL_SET_MAXBURST(8) | + SAHARA_CONTROL_RNG_AUTORSD | + SAHARA_CONTROL_ENABLE_INT, + SAHARA_REG_CONTROL); + + err = sahara_register_algs(dev); + if (err) + goto err_algs; + + dev_info(&pdev->dev, "SAHARA version %d initialized\n", version); + + return 0; + +err_algs: + dma_free_coherent(&pdev->dev, + SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), + dev->hw_link[0], dev->hw_phys_link[0]); + clk_disable_unprepare(dev->clk_ipg); + clk_disable_unprepare(dev->clk_ahb); + dev_ptr = NULL; +err_link: + dma_free_coherent(&pdev->dev, + 2 * AES_KEYSIZE_128, + dev->key_base, dev->key_phys_base); +err_key: + dma_free_coherent(&pdev->dev, + SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), + dev->hw_desc[0], dev->hw_phys_desc[0]); + + return err; +} + +static int sahara_remove(struct platform_device *pdev) +{ + struct sahara_dev *dev = platform_get_drvdata(pdev); + + dma_free_coherent(&pdev->dev, + SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), + dev->hw_link[0], dev->hw_phys_link[0]); + dma_free_coherent(&pdev->dev, + 2 * AES_KEYSIZE_128, + dev->key_base, dev->key_phys_base); + dma_free_coherent(&pdev->dev, + SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), + dev->hw_desc[0], dev->hw_phys_desc[0]); + + tasklet_kill(&dev->done_task); + tasklet_kill(&dev->queue_task); + + sahara_unregister_algs(dev); + + clk_disable_unprepare(dev->clk_ipg); + clk_disable_unprepare(dev->clk_ahb); + + dev_ptr = NULL; + + return 0; +} + +static struct platform_driver sahara_driver = { + .probe = sahara_probe, + .remove = sahara_remove, + .driver = { + .name = SAHARA_NAME, + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(sahara_dt_ids), + }, + .id_table = sahara_platform_ids, +}; + +module_platform_driver(sahara_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Javier Martin "); +MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator"); -- cgit v0.10.2 From 1643a35fea3300c7df63c91596d3246c05b43a76 Mon Sep 17 00:00:00 2001 From: Mihnea Dobrescu-Balaur Date: Mon, 11 Mar 2013 12:48:10 +0200 Subject: crypto: ux500 - replace kmalloc and then memcpy with kmemdup Signed-off-by: Mihnea Dobrescu-Balaur Signed-off-by: Herbert Xu diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index 632c333..8d16d3a 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -1368,14 +1368,12 @@ static int hash_setkey(struct crypto_ahash *tfm, /** * Freed in final. */ - ctx->key = kmalloc(keylen, GFP_KERNEL); + ctx->key = kmemdup(key, keylen, GFP_KERNEL); if (!ctx->key) { pr_err(DEV_DBG_NAME " [%s] Failed to allocate ctx->key " "for %d\n", __func__, alg); return -ENOMEM; } - - memcpy(ctx->key, key, keylen); ctx->keylen = keylen; return ret; -- cgit v0.10.2 From a80c5422b1426699c279b74fe8d3e8a4e3c902de Mon Sep 17 00:00:00 2001 From: Jingoo Han Date: Tue, 12 Mar 2013 14:46:22 +0900 Subject: hwrng: exynos - add CONFIG_PM_SLEEP/CONFIG_PM_RUNTIME to suspend/resume This patch adds CONFIG_PM_SLEEP to suspend/resume functions to fix the following build warning when CONFIG_PM_SLEEP is not selected. drivers/char/hw_random/exynos-rng.c:147:12: warning: 'exynos_rng_runtime_suspend' defined but not used [-Wunused-function] drivers/char/hw_random/exynos-rng.c:157:12: warning: 'exynos_rng_runtime_resume' defined but not used [-Wunused-function] Add CONFIG_PM_RUNTIME to suspend/resume functions to fix the build error. It is because UNIVERSAL_DEV_PM_OPS macro is related to both CONFIG_PM_SLEEP and CONFIG_PM_RUNTIME. drivers/char/hw_random/exynos-rng.c:167:8: error: 'exynos_rng_runtime_suspend' undeclared here (not in a function) drivers/char/hw_random/exynos-rng.c:167:8: error: 'exynos_rng_runtime_resume' undeclared here (not in a function) Signed-off-by: Jingoo Han Reported-by: David Rientjes Signed-off-by: Herbert Xu diff --git a/drivers/char/hw_random/exynos-rng.c b/drivers/char/hw_random/exynos-rng.c index ac47631..402ccfb 100644 --- a/drivers/char/hw_random/exynos-rng.c +++ b/drivers/char/hw_random/exynos-rng.c @@ -144,6 +144,7 @@ static int exynos_rng_remove(struct platform_device *pdev) return 0; } +#if defined(CONFIG_PM_SLEEP) || defined(CONFIG_PM_RUNTIME) static int exynos_rng_runtime_suspend(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); @@ -161,7 +162,7 @@ static int exynos_rng_runtime_resume(struct device *dev) return clk_prepare_enable(exynos_rng->clk); } - +#endif static UNIVERSAL_DEV_PM_OPS(exynos_rng_pm_ops, exynos_rng_runtime_suspend, exynos_rng_runtime_resume, NULL); -- cgit v0.10.2 From 575c1bd549ff6bc097b3669b086e54e01169379f Mon Sep 17 00:00:00 2001 From: Vakul Garg Date: Tue, 12 Mar 2013 13:55:21 +0530 Subject: crypto: caam - set RDB bit in security configuration register This change is required for post SEC-5.0 devices which have RNG4. Setting RDB in security configuration register allows CAAM to use the "Random Data Buffer" to be filled by a single request. The Random Data Buffer is large enough for ten packets to get their IVs from a single request. If the Random Data Buffer is not enabled, then each IV causes a separate request, and RNG4 hardware cannot keep up resulting in lower IPSEC throughput if random IVs are used. Signed-off-by: Vakul Garg Signed-off-by: Herbert Xu diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 1c56f63..19faea2 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -304,6 +304,9 @@ static int caam_probe(struct platform_device *pdev) caam_remove(pdev); return ret; } + + /* Enable RDB bit so that RNG works faster */ + setbits32(&topregs->ctrl.scfgr, SCFGR_RDBENABLE); } /* NOTE: RTIC detection ought to go here, around Si time */ diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index 3223fc6..cd6feda 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -252,7 +252,8 @@ struct caam_ctrl { /* Read/Writable */ u32 rsvd1; u32 mcr; /* MCFG Master Config Register */ - u32 rsvd2[2]; + u32 rsvd2; + u32 scfgr; /* SCFGR, Security Config Register */ /* Bus Access Configuration Section 010-11f */ /* Read/Writable */ @@ -299,6 +300,7 @@ struct caam_ctrl { #define MCFGR_WDFAIL 0x20000000 /* DECO watchdog force-fail */ #define MCFGR_DMA_RESET 0x10000000 #define MCFGR_LONG_PTR 0x00010000 /* Use >32-bit desc addressing */ +#define SCFGR_RDBENABLE 0x00000400 /* AXI read cache control */ #define MCFGR_ARCACHE_SHIFT 12 -- cgit v0.10.2 From cb7d5662d7a13471eff81b25277a229d7173aabd Mon Sep 17 00:00:00 2001 From: Vakul Garg Date: Tue, 12 Mar 2013 14:09:24 +0530 Subject: crypto: caam - Fix missing init of '.type' in AEAD algos. Following AEAD algo templates are updated for '.type' initialization. (a) authenc(hmac(sha224),cbc(aes)) (b) authenc(hmac(sha384),cbc(aes)) (c) authenc(hmac(sha224),cbc(des3_ede)) (d) authenc(hmac(sha384),cbc(des3_ede)) (e) authenc(hmac(sha224),cbc(des)) (f) authenc(hmac(sha384),cbc(des)) Signed-off-by: Vakul Garg Reviewed-by: Kim Phillips Signed-off-by: Andy Fleming Signed-off-by: Herbert Xu diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index b2a0a07..42420fb 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1697,6 +1697,7 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha224),cbc(aes))", .driver_name = "authenc-hmac-sha224-cbc-aes-caam", .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, @@ -1736,6 +1737,7 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha384),cbc(aes))", .driver_name = "authenc-hmac-sha384-cbc-aes-caam", .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, @@ -1814,6 +1816,7 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha224),cbc(des3_ede))", .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam", .blocksize = DES3_EDE_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, @@ -1853,6 +1856,7 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha384),cbc(des3_ede))", .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam", .blocksize = DES3_EDE_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, @@ -1930,6 +1934,7 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha224),cbc(des))", .driver_name = "authenc-hmac-sha224-cbc-des-caam", .blocksize = DES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, @@ -1969,6 +1974,7 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha384),cbc(des))", .driver_name = "authenc-hmac-sha384-cbc-des-caam", .blocksize = DES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, -- cgit v0.10.2 From 264878e6a7eb02841100beb0d999987ef3b5e17f Mon Sep 17 00:00:00 2001 From: Fabio Estevam Date: Wed, 13 Mar 2013 00:57:27 -0300 Subject: hwrng: mxc-rnga - Use devm_ioremap_resource() Using devm_ioremap_resource() can make the code cleaner and simpler. Signed-off-by: Fabio Estevam Signed-off-by: Herbert Xu diff --git a/drivers/char/hw_random/mxc-rnga.c b/drivers/char/hw_random/mxc-rnga.c index f05d857..93fc741 100644 --- a/drivers/char/hw_random/mxc-rnga.c +++ b/drivers/char/hw_random/mxc-rnga.c @@ -142,7 +142,7 @@ static void mxc_rnga_cleanup(struct hwrng *rng) static int __init mxc_rnga_probe(struct platform_device *pdev) { int err = -ENODEV; - struct resource *res, *mem; + struct resource *res; struct mxc_rng *mxc_rng; mxc_rng = devm_kzalloc(&pdev->dev, sizeof(struct mxc_rng), @@ -172,15 +172,9 @@ static int __init mxc_rnga_probe(struct platform_device *pdev) goto err_region; } - mem = request_mem_region(res->start, resource_size(res), pdev->name); - if (mem == NULL) { - err = -EBUSY; - goto err_region; - } - - mxc_rng->mem = ioremap(res->start, resource_size(res)); - if (!mxc_rng->mem) { - err = -ENOMEM; + mxc_rng->mem = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(mxc_rng->mem)) { + err = PTR_ERR(mxc_rng->mem); goto err_ioremap; } @@ -195,8 +189,6 @@ static int __init mxc_rnga_probe(struct platform_device *pdev) return 0; err_ioremap: - release_mem_region(res->start, resource_size(res)); - err_region: clk_disable_unprepare(mxc_rng->clk); @@ -206,15 +198,10 @@ out: static int __exit mxc_rnga_remove(struct platform_device *pdev) { - struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); struct mxc_rng *mxc_rng = platform_get_drvdata(pdev); hwrng_unregister(&mxc_rng->rng); - iounmap(mxc_rng->mem); - - release_mem_region(res->start, resource_size(res)); - clk_disable_unprepare(mxc_rng->clk); return 0; -- cgit v0.10.2 From 5cec26e9843e4f16974e5a30a3ffada1557b661d Mon Sep 17 00:00:00 2001 From: Sachin Kamat Date: Thu, 14 Mar 2013 15:46:58 +0530 Subject: crypto: picoxcell - Use of_match_ptr() macro This eliminates having an #ifdef returning NULL for the case when OF is disabled. Signed-off-by: Sachin Kamat Acked-by: Jamie Iles Signed-off-by: Herbert Xu diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index 2096d46..ac30724 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -1688,8 +1688,6 @@ static const struct of_device_id spacc_of_id_table[] = { { .compatible = "picochip,spacc-l2" }, {} }; -#else /* CONFIG_OF */ -#define spacc_of_id_table NULL #endif /* CONFIG_OF */ static bool spacc_is_compatible(struct platform_device *pdev, @@ -1874,7 +1872,7 @@ static struct platform_driver spacc_driver = { #ifdef CONFIG_PM .pm = &spacc_pm_ops, #endif /* CONFIG_PM */ - .of_match_table = spacc_of_id_table, + .of_match_table = of_match_ptr(spacc_of_id_table), }, .id_table = spacc_id_table, }; -- cgit v0.10.2 From eb16796302bad44fd346d4e0e15f8086132d1363 Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Fri, 22 Mar 2013 21:18:44 +0800 Subject: crypto: ux500 - fix error return code in hash_dma_final() Fix to return a negative error code from the error handling case instead of 0, as returned elsewhere in this function. Signed-off-by: Wei Yongjun Acked-by: Linus Walleij Signed-off-by: Herbert Xu diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index 8d16d3a..d99b671 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -939,6 +939,7 @@ static int hash_dma_final(struct ahash_request *req) if (!ctx->device->dma.nents) { dev_err(device_data->dev, "[%s] " "ctx->device->dma.nents = 0", __func__); + ret = ctx->device->dma.nents; goto out; } @@ -946,6 +947,7 @@ static int hash_dma_final(struct ahash_request *req) if (bytes_written != req->nbytes) { dev_err(device_data->dev, "[%s] " "hash_dma_write() failed!", __func__); + ret = bytes_written; goto out; } -- cgit v0.10.2 From eca17269976fe82cdd713095473bee4842dbd41a Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sun, 24 Mar 2013 14:32:01 +0200 Subject: crypto: x86/crc32-pclmul - assembly clean-ups: use ENTRY/ENDPROC Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/crc32-pclmul_asm.S b/arch/x86/crypto/crc32-pclmul_asm.S index c8335014..94c27df 100644 --- a/arch/x86/crypto/crc32-pclmul_asm.S +++ b/arch/x86/crypto/crc32-pclmul_asm.S @@ -101,9 +101,8 @@ * uint crc32_pclmul_le_16(unsigned char const *buffer, * size_t len, uint crc32) */ -.globl crc32_pclmul_le_16 -.align 4, 0x90 -crc32_pclmul_le_16:/* buffer and buffer size are 16 bytes aligned */ + +ENTRY(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligned */ movdqa (BUF), %xmm1 movdqa 0x10(BUF), %xmm2 movdqa 0x20(BUF), %xmm3 @@ -244,3 +243,4 @@ fold_64: pextrd $0x01, %xmm1, %eax ret +ENDPROC(crc32_pclmul_le_16) -- cgit v0.10.2 From 873b9cafa8cae695d92f0faabe7166af10078b78 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sun, 24 Mar 2013 15:34:07 +0200 Subject: crypto: x86 - build AVX block cipher implementations only if assembler supports AVX instructions These modules require AVX support in assembler, so add new check to Makefile for this. Other option would be to use CONFIG_AS_AVX inside source files, but that would result dummy/empty/no-fuctionality modules being created. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index ca96761..83681a3 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -2,6 +2,8 @@ # Arch-specific CryptoAPI modules. # +avx_supported := $(call as-instr,vpxor %xmm0$(comma)%xmm0$(comma)%xmm0,yes,no) + obj-$(CONFIG_CRYPTO_ABLK_HELPER_X86) += ablk_helper.o obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o @@ -12,16 +14,11 @@ obj-$(CONFIG_CRYPTO_SERPENT_SSE2_586) += serpent-sse2-i586.o obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o -obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX_X86_64) += camellia-aesni-avx-x86_64.o -obj-$(CONFIG_CRYPTO_CAST5_AVX_X86_64) += cast5-avx-x86_64.o -obj-$(CONFIG_CRYPTO_CAST6_AVX_X86_64) += cast6-avx-x86_64.o obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o -obj-$(CONFIG_CRYPTO_TWOFISH_AVX_X86_64) += twofish-avx-x86_64.o obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o obj-$(CONFIG_CRYPTO_SERPENT_SSE2_X86_64) += serpent-sse2-x86_64.o -obj-$(CONFIG_CRYPTO_SERPENT_AVX_X86_64) += serpent-avx-x86_64.o obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o @@ -29,6 +26,16 @@ obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o +# These modules require assembler to support AVX. +ifeq ($(avx_supported),yes) + obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX_X86_64) += \ + camellia-aesni-avx-x86_64.o + obj-$(CONFIG_CRYPTO_CAST5_AVX_X86_64) += cast5-avx-x86_64.o + obj-$(CONFIG_CRYPTO_CAST6_AVX_X86_64) += cast6-avx-x86_64.o + obj-$(CONFIG_CRYPTO_TWOFISH_AVX_X86_64) += twofish-avx-x86_64.o + obj-$(CONFIG_CRYPTO_SERPENT_AVX_X86_64) += serpent-avx-x86_64.o +endif + aes-i586-y := aes-i586-asm_32.o aes_glue.o twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o @@ -36,17 +43,22 @@ serpent-sse2-i586-y := serpent-sse2-i586-asm_32.o serpent_sse2_glue.o aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o -camellia-aesni-avx-x86_64-y := camellia-aesni-avx-asm_64.o \ - camellia_aesni_avx_glue.o -cast5-avx-x86_64-y := cast5-avx-x86_64-asm_64.o cast5_avx_glue.o -cast6-avx-x86_64-y := cast6-avx-x86_64-asm_64.o cast6_avx_glue.o blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o -twofish-avx-x86_64-y := twofish-avx-x86_64-asm_64.o twofish_avx_glue.o salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o serpent-sse2-x86_64-y := serpent-sse2-x86_64-asm_64.o serpent_sse2_glue.o -serpent-avx-x86_64-y := serpent-avx-x86_64-asm_64.o serpent_avx_glue.o + +ifeq ($(avx_supported),yes) + camellia-aesni-avx-x86_64-y := camellia-aesni-avx-asm_64.o \ + camellia_aesni_avx_glue.o + cast5-avx-x86_64-y := cast5-avx-x86_64-asm_64.o cast5_avx_glue.o + cast6-avx-x86_64-y := cast6-avx-x86_64-asm_64.o cast6_avx_glue.o + twofish-avx-x86_64-y := twofish-avx-x86_64-asm_64.o \ + twofish_avx_glue.o + serpent-avx-x86_64-y := serpent-avx-x86_64-asm_64.o \ + serpent_avx_glue.o +endif aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o -- cgit v0.10.2 From 35d2c9d0c3cfd90850dc647250610587743e1f29 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:58:49 -0700 Subject: crypto: sha256 - Expose SHA256 generic routine to be callable externally. Other SHA256 routine may need to use the generic routine when FPU is not available. Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index c3ed4ec..5433667 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -246,7 +246,7 @@ static int sha256_init(struct shash_desc *desc) return 0; } -static int sha256_update(struct shash_desc *desc, const u8 *data, +int crypto_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct sha256_state *sctx = shash_desc_ctx(desc); @@ -277,6 +277,7 @@ static int sha256_update(struct shash_desc *desc, const u8 *data, return 0; } +EXPORT_SYMBOL(crypto_sha256_update); static int sha256_final(struct shash_desc *desc, u8 *out) { @@ -293,10 +294,10 @@ static int sha256_final(struct shash_desc *desc, u8 *out) /* Pad out to 56 mod 64. */ index = sctx->count & 0x3f; pad_len = (index < 56) ? (56 - index) : ((64+56) - index); - sha256_update(desc, padding, pad_len); + crypto_sha256_update(desc, padding, pad_len); /* Append length (before padding) */ - sha256_update(desc, (const u8 *)&bits, sizeof(bits)); + crypto_sha256_update(desc, (const u8 *)&bits, sizeof(bits)); /* Store state in digest */ for (i = 0; i < 8; i++) @@ -339,7 +340,7 @@ static int sha256_import(struct shash_desc *desc, const void *in) static struct shash_alg sha256_algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_init, - .update = sha256_update, + .update = crypto_sha256_update, .final = sha256_final, .export = sha256_export, .import = sha256_import, @@ -355,7 +356,7 @@ static struct shash_alg sha256_algs[2] = { { }, { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_init, - .update = sha256_update, + .update = crypto_sha256_update, .final = sha224_final, .descsize = sizeof(struct sha256_state), .base = { diff --git a/include/crypto/sha.h b/include/crypto/sha.h index c6c9c1f..f46ff61 100644 --- a/include/crypto/sha.h +++ b/include/crypto/sha.h @@ -87,4 +87,6 @@ struct shash_desc; extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len); +extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len); #endif -- cgit v0.10.2 From 46d208a2bdf5c3d4a60f2363318f600d64493f60 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:58:58 -0700 Subject: crypto: sha256 - Optimized sha256 x86_64 assembly routine using Supplemental SSE3 instructions. Provides SHA256 x86_64 assembly routine optimized with SSSE3 instructions. Speedup of 40% or more has been measured over the generic implementation. Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/sha256-ssse3-asm.S b/arch/x86/crypto/sha256-ssse3-asm.S new file mode 100644 index 0000000..98d3c39 --- /dev/null +++ b/arch/x86/crypto/sha256-ssse3-asm.S @@ -0,0 +1,506 @@ +######################################################################## +# Implement fast SHA-256 with SSSE3 instructions. (x86_64) +# +# Copyright (C) 2013 Intel Corporation. +# +# Authors: +# James Guilford +# Kirk Yap +# Tim Chen +# +# This software is available to you under a choice of one of two +# licenses. You may choose to be licensed under the terms of the GNU +# General Public License (GPL) Version 2, available from the file +# COPYING in the main directory of this source tree, or the +# OpenIB.org BSD license below: +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +######################################################################## +# +# This code is described in an Intel White-Paper: +# "Fast SHA-256 Implementations on Intel Architecture Processors" +# +# To find it, surf to http://www.intel.com/p/en_US/embedded +# and search for that title. +# +######################################################################## + +#include + +## assume buffers not aligned +#define MOVDQ movdqu + +################################ Define Macros + +# addm [mem], reg +# Add reg to mem using reg-mem add and store +.macro addm p1 p2 + add \p1, \p2 + mov \p2, \p1 +.endm + +################################ + +# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask +# Load xmm with mem and byte swap each dword +.macro COPY_XMM_AND_BSWAP p1 p2 p3 + MOVDQ \p2, \p1 + pshufb \p3, \p1 +.endm + +################################ + +X0 = %xmm4 +X1 = %xmm5 +X2 = %xmm6 +X3 = %xmm7 + +XTMP0 = %xmm0 +XTMP1 = %xmm1 +XTMP2 = %xmm2 +XTMP3 = %xmm3 +XTMP4 = %xmm8 +XFER = %xmm9 + +SHUF_00BA = %xmm10 # shuffle xBxA -> 00BA +SHUF_DC00 = %xmm11 # shuffle xDxC -> DC00 +BYTE_FLIP_MASK = %xmm12 + +NUM_BLKS = %rdx # 3rd arg +CTX = %rsi # 2nd arg +INP = %rdi # 1st arg + +SRND = %rdi # clobbers INP +c = %ecx +d = %r8d +e = %edx +TBL = %rbp +a = %eax +b = %ebx + +f = %r9d +g = %r10d +h = %r11d + +y0 = %r13d +y1 = %r14d +y2 = %r15d + + + +_INP_END_SIZE = 8 +_INP_SIZE = 8 +_XFER_SIZE = 8 +_XMM_SAVE_SIZE = 0 + +_INP_END = 0 +_INP = _INP_END + _INP_END_SIZE +_XFER = _INP + _INP_SIZE +_XMM_SAVE = _XFER + _XFER_SIZE +STACK_SIZE = _XMM_SAVE + _XMM_SAVE_SIZE + +# rotate_Xs +# Rotate values of symbols X0...X3 +.macro rotate_Xs +X_ = X0 +X0 = X1 +X1 = X2 +X2 = X3 +X3 = X_ +.endm + +# ROTATE_ARGS +# Rotate values of symbols a...h +.macro ROTATE_ARGS +TMP_ = h +h = g +g = f +f = e +e = d +d = c +c = b +b = a +a = TMP_ +.endm + +.macro FOUR_ROUNDS_AND_SCHED + ## compute s0 four at a time and s1 two at a time + ## compute W[-16] + W[-7] 4 at a time + movdqa X3, XTMP0 + mov e, y0 # y0 = e + ror $(25-11), y0 # y0 = e >> (25-11) + mov a, y1 # y1 = a + palignr $4, X2, XTMP0 # XTMP0 = W[-7] + ror $(22-13), y1 # y1 = a >> (22-13) + xor e, y0 # y0 = e ^ (e >> (25-11)) + mov f, y2 # y2 = f + ror $(11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) + movdqa X1, XTMP1 + xor a, y1 # y1 = a ^ (a >> (22-13) + xor g, y2 # y2 = f^g + paddd X0, XTMP0 # XTMP0 = W[-7] + W[-16] + xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and e, y2 # y2 = (f^g)&e + ror $(13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) + ## compute s0 + palignr $4, X0, XTMP1 # XTMP1 = W[-15] + xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror $6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor g, y2 # y2 = CH = ((f^g)&e)^g + movdqa XTMP1, XTMP2 # XTMP2 = W[-15] + ror $2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y0, y2 # y2 = S1 + CH + add _XFER(%rsp) , y2 # y2 = k + w + S1 + CH + movdqa XTMP1, XTMP3 # XTMP3 = W[-15] + mov a, y0 # y0 = a + add y2, h # h = h + S1 + CH + k + w + mov a, y2 # y2 = a + pslld $(32-7), XTMP1 # + or c, y0 # y0 = a|c + add h, d # d = d + h + S1 + CH + k + w + and c, y2 # y2 = a&c + psrld $7, XTMP2 # + and b, y0 # y0 = (a|c)&b + add y1, h # h = h + S1 + CH + k + w + S0 + por XTMP2, XTMP1 # XTMP1 = W[-15] ror 7 + or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) + add y0, h # h = h + S1 + CH + k + w + S0 + MAJ + # + ROTATE_ARGS # + movdqa XTMP3, XTMP2 # XTMP2 = W[-15] + mov e, y0 # y0 = e + mov a, y1 # y1 = a + movdqa XTMP3, XTMP4 # XTMP4 = W[-15] + ror $(25-11), y0 # y0 = e >> (25-11) + xor e, y0 # y0 = e ^ (e >> (25-11)) + mov f, y2 # y2 = f + ror $(22-13), y1 # y1 = a >> (22-13) + pslld $(32-18), XTMP3 # + xor a, y1 # y1 = a ^ (a >> (22-13) + ror $(11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor g, y2 # y2 = f^g + psrld $18, XTMP2 # + ror $(13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and e, y2 # y2 = (f^g)&e + ror $6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP3, XTMP1 + xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor g, y2 # y2 = CH = ((f^g)&e)^g + psrld $3, XTMP4 # XTMP4 = W[-15] >> 3 + add y0, y2 # y2 = S1 + CH + add (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH + ror $2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + pxor XTMP2, XTMP1 # XTMP1 = W[-15] ror 7 ^ W[-15] ror 18 + mov a, y0 # y0 = a + add y2, h # h = h + S1 + CH + k + w + mov a, y2 # y2 = a + pxor XTMP4, XTMP1 # XTMP1 = s0 + or c, y0 # y0 = a|c + add h, d # d = d + h + S1 + CH + k + w + and c, y2 # y2 = a&c + ## compute low s1 + pshufd $0b11111010, X3, XTMP2 # XTMP2 = W[-2] {BBAA} + and b, y0 # y0 = (a|c)&b + add y1, h # h = h + S1 + CH + k + w + S0 + paddd XTMP1, XTMP0 # XTMP0 = W[-16] + W[-7] + s0 + or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) + add y0, h # h = h + S1 + CH + k + w + S0 + MAJ + + ROTATE_ARGS + movdqa XTMP2, XTMP3 # XTMP3 = W[-2] {BBAA} + mov e, y0 # y0 = e + mov a, y1 # y1 = a + ror $(25-11), y0 # y0 = e >> (25-11) + movdqa XTMP2, XTMP4 # XTMP4 = W[-2] {BBAA} + xor e, y0 # y0 = e ^ (e >> (25-11)) + ror $(22-13), y1 # y1 = a >> (22-13) + mov f, y2 # y2 = f + xor a, y1 # y1 = a ^ (a >> (22-13) + ror $(11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq $17, XTMP2 # XTMP2 = W[-2] ror 17 {xBxA} + xor g, y2 # y2 = f^g + psrlq $19, XTMP3 # XTMP3 = W[-2] ror 19 {xBxA} + xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and e, y2 # y2 = (f^g)&e + psrld $10, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA} + ror $(13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor g, y2 # y2 = CH = ((f^g)&e)^g + ror $6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP3, XTMP2 + add y0, y2 # y2 = S1 + CH + ror $2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH + pxor XTMP2, XTMP4 # XTMP4 = s1 {xBxA} + mov a, y0 # y0 = a + add y2, h # h = h + S1 + CH + k + w + mov a, y2 # y2 = a + pshufb SHUF_00BA, XTMP4 # XTMP4 = s1 {00BA} + or c, y0 # y0 = a|c + add h, d # d = d + h + S1 + CH + k + w + and c, y2 # y2 = a&c + paddd XTMP4, XTMP0 # XTMP0 = {..., ..., W[1], W[0]} + and b, y0 # y0 = (a|c)&b + add y1, h # h = h + S1 + CH + k + w + S0 + ## compute high s1 + pshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {BBAA} + or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) + add y0, h # h = h + S1 + CH + k + w + S0 + MAJ + # + ROTATE_ARGS # + movdqa XTMP2, XTMP3 # XTMP3 = W[-2] {DDCC} + mov e, y0 # y0 = e + ror $(25-11), y0 # y0 = e >> (25-11) + mov a, y1 # y1 = a + movdqa XTMP2, X0 # X0 = W[-2] {DDCC} + ror $(22-13), y1 # y1 = a >> (22-13) + xor e, y0 # y0 = e ^ (e >> (25-11)) + mov f, y2 # y2 = f + ror $(11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq $17, XTMP2 # XTMP2 = W[-2] ror 17 {xDxC} + xor a, y1 # y1 = a ^ (a >> (22-13) + xor g, y2 # y2 = f^g + psrlq $19, XTMP3 # XTMP3 = W[-2] ror 19 {xDxC} + xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25 + and e, y2 # y2 = (f^g)&e + ror $(13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) + psrld $10, X0 # X0 = W[-2] >> 10 {DDCC} + xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22 + ror $6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>2 + xor g, y2 # y2 = CH = ((f^g)&e)^g + pxor XTMP3, XTMP2 # + ror $2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>2 + add y0, y2 # y2 = S1 + CH + add (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH + pxor XTMP2, X0 # X0 = s1 {xDxC} + mov a, y0 # y0 = a + add y2, h # h = h + S1 + CH + k + w + mov a, y2 # y2 = a + pshufb SHUF_DC00, X0 # X0 = s1 {DC00} + or c, y0 # y0 = a|c + add h, d # d = d + h + S1 + CH + k + w + and c, y2 # y2 = a&c + paddd XTMP0, X0 # X0 = {W[3], W[2], W[1], W[0]} + and b, y0 # y0 = (a|c)&b + add y1, h # h = h + S1 + CH + k + w + S0 + or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) + add y0, h # h = h + S1 + CH + k + w + S0 + MAJ + + ROTATE_ARGS + rotate_Xs +.endm + +## input is [rsp + _XFER + %1 * 4] +.macro DO_ROUND round + mov e, y0 # y0 = e + ror $(25-11), y0 # y0 = e >> (25-11) + mov a, y1 # y1 = a + xor e, y0 # y0 = e ^ (e >> (25-11)) + ror $(22-13), y1 # y1 = a >> (22-13) + mov f, y2 # y2 = f + xor a, y1 # y1 = a ^ (a >> (22-13) + ror $(11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor g, y2 # y2 = f^g + xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + ror $(13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) + and e, y2 # y2 = (f^g)&e + xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror $6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor g, y2 # y2 = CH = ((f^g)&e)^g + add y0, y2 # y2 = S1 + CH + ror $2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + offset = \round * 4 + _XFER + add offset(%rsp), y2 # y2 = k + w + S1 + CH + mov a, y0 # y0 = a + add y2, h # h = h + S1 + CH + k + w + mov a, y2 # y2 = a + or c, y0 # y0 = a|c + add h, d # d = d + h + S1 + CH + k + w + and c, y2 # y2 = a&c + and b, y0 # y0 = (a|c)&b + add y1, h # h = h + S1 + CH + k + w + S0 + or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) + add y0, h # h = h + S1 + CH + k + w + S0 + MAJ + ROTATE_ARGS +.endm + +######################################################################## +## void sha256_transform_ssse3(void *input_data, UINT32 digest[8], UINT64 num_blks) +## arg 1 : pointer to input data +## arg 2 : pointer to digest +## arg 3 : Num blocks +######################################################################## +.text +ENTRY(sha256_transform_ssse3) +.align 32 + pushq %rbx + pushq %rbp + pushq %r13 + pushq %r14 + pushq %r15 + pushq %r12 + + mov %rsp, %r12 + subq $STACK_SIZE, %rsp + and $~15, %rsp + + shl $6, NUM_BLKS # convert to bytes + jz done_hash + add INP, NUM_BLKS + mov NUM_BLKS, _INP_END(%rsp) # pointer to end of data + + ## load initial digest + mov 4*0(CTX), a + mov 4*1(CTX), b + mov 4*2(CTX), c + mov 4*3(CTX), d + mov 4*4(CTX), e + mov 4*5(CTX), f + mov 4*6(CTX), g + mov 4*7(CTX), h + + movdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK + movdqa _SHUF_00BA(%rip), SHUF_00BA + movdqa _SHUF_DC00(%rip), SHUF_DC00 + +loop0: + lea K256(%rip), TBL + + ## byte swap first 16 dwords + COPY_XMM_AND_BSWAP X0, 0*16(INP), BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X1, 1*16(INP), BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X2, 2*16(INP), BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X3, 3*16(INP), BYTE_FLIP_MASK + + mov INP, _INP(%rsp) + + ## schedule 48 input dwords, by doing 3 rounds of 16 each + mov $3, SRND +.align 16 +loop1: + movdqa (TBL), XFER + paddd X0, XFER + movdqa XFER, _XFER(%rsp) + FOUR_ROUNDS_AND_SCHED + + movdqa 1*16(TBL), XFER + paddd X0, XFER + movdqa XFER, _XFER(%rsp) + FOUR_ROUNDS_AND_SCHED + + movdqa 2*16(TBL), XFER + paddd X0, XFER + movdqa XFER, _XFER(%rsp) + FOUR_ROUNDS_AND_SCHED + + movdqa 3*16(TBL), XFER + paddd X0, XFER + movdqa XFER, _XFER(%rsp) + add $4*16, TBL + FOUR_ROUNDS_AND_SCHED + + sub $1, SRND + jne loop1 + + mov $2, SRND +loop2: + paddd (TBL), X0 + movdqa X0, _XFER(%rsp) + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + paddd 1*16(TBL), X1 + movdqa X1, _XFER(%rsp) + add $2*16, TBL + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + + movdqa X2, X0 + movdqa X3, X1 + + sub $1, SRND + jne loop2 + + addm (4*0)(CTX),a + addm (4*1)(CTX),b + addm (4*2)(CTX),c + addm (4*3)(CTX),d + addm (4*4)(CTX),e + addm (4*5)(CTX),f + addm (4*6)(CTX),g + addm (4*7)(CTX),h + + mov _INP(%rsp), INP + add $64, INP + cmp _INP_END(%rsp), INP + jne loop0 + +done_hash: + + mov %r12, %rsp + + popq %r12 + popq %r15 + popq %r14 + popq %r13 + popq %rbp + popq %rbx + + ret +ENDPROC(sha256_transform_ssse3) + +.data +.align 64 +K256: + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +PSHUFFLE_BYTE_FLIP_MASK: + .octa 0x0c0d0e0f08090a0b0405060700010203 + +# shuffle xBxA -> 00BA +_SHUF_00BA: + .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100 + +# shuffle xDxC -> DC00 +_SHUF_DC00: + .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF -- cgit v0.10.2 From ec2b4c851f4da48a51b79a69843beb135e3db8c2 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:59:05 -0700 Subject: crypto: sha256 - Optimized sha256 x86_64 assembly routine with AVX instructions. Provides SHA256 x86_64 assembly routine optimized with SSE and AVX instructions. Speedup of 60% or more has been measured over the generic implementation. Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/sha256-avx-asm.S b/arch/x86/crypto/sha256-avx-asm.S new file mode 100644 index 0000000..56610c4 --- /dev/null +++ b/arch/x86/crypto/sha256-avx-asm.S @@ -0,0 +1,496 @@ +######################################################################## +# Implement fast SHA-256 with AVX1 instructions. (x86_64) +# +# Copyright (C) 2013 Intel Corporation. +# +# Authors: +# James Guilford +# Kirk Yap +# Tim Chen +# +# This software is available to you under a choice of one of two +# licenses. You may choose to be licensed under the terms of the GNU +# General Public License (GPL) Version 2, available from the file +# COPYING in the main directory of this source tree, or the +# OpenIB.org BSD license below: +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +######################################################################## +# +# This code is described in an Intel White-Paper: +# "Fast SHA-256 Implementations on Intel Architecture Processors" +# +# To find it, surf to http://www.intel.com/p/en_US/embedded +# and search for that title. +# +######################################################################## +# This code schedules 1 block at a time, with 4 lanes per block +######################################################################## + +#ifdef CONFIG_AS_AVX +#include + +## assume buffers not aligned +#define VMOVDQ vmovdqu + +################################ Define Macros + +# addm [mem], reg +# Add reg to mem using reg-mem add and store +.macro addm p1 p2 + add \p1, \p2 + mov \p2, \p1 +.endm + + +.macro MY_ROR p1 p2 + shld $(32-(\p1)), \p2, \p2 +.endm + +################################ + +# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask +# Load xmm with mem and byte swap each dword +.macro COPY_XMM_AND_BSWAP p1 p2 p3 + VMOVDQ \p2, \p1 + vpshufb \p3, \p1, \p1 +.endm + +################################ + +X0 = %xmm4 +X1 = %xmm5 +X2 = %xmm6 +X3 = %xmm7 + +XTMP0 = %xmm0 +XTMP1 = %xmm1 +XTMP2 = %xmm2 +XTMP3 = %xmm3 +XTMP4 = %xmm8 +XFER = %xmm9 +XTMP5 = %xmm11 + +SHUF_00BA = %xmm10 # shuffle xBxA -> 00BA +SHUF_DC00 = %xmm12 # shuffle xDxC -> DC00 +BYTE_FLIP_MASK = %xmm13 + +NUM_BLKS = %rdx # 3rd arg +CTX = %rsi # 2nd arg +INP = %rdi # 1st arg + +SRND = %rdi # clobbers INP +c = %ecx +d = %r8d +e = %edx +TBL = %rbp +a = %eax +b = %ebx + +f = %r9d +g = %r10d +h = %r11d + +y0 = %r13d +y1 = %r14d +y2 = %r15d + + +_INP_END_SIZE = 8 +_INP_SIZE = 8 +_XFER_SIZE = 8 +_XMM_SAVE_SIZE = 0 + +_INP_END = 0 +_INP = _INP_END + _INP_END_SIZE +_XFER = _INP + _INP_SIZE +_XMM_SAVE = _XFER + _XFER_SIZE +STACK_SIZE = _XMM_SAVE + _XMM_SAVE_SIZE + +# rotate_Xs +# Rotate values of symbols X0...X3 +.macro rotate_Xs +X_ = X0 +X0 = X1 +X1 = X2 +X2 = X3 +X3 = X_ +.endm + +# ROTATE_ARGS +# Rotate values of symbols a...h +.macro ROTATE_ARGS +TMP_ = h +h = g +g = f +f = e +e = d +d = c +c = b +b = a +a = TMP_ +.endm + +.macro FOUR_ROUNDS_AND_SCHED + ## compute s0 four at a time and s1 two at a time + ## compute W[-16] + W[-7] 4 at a time + + mov e, y0 # y0 = e + MY_ROR (25-11), y0 # y0 = e >> (25-11) + mov a, y1 # y1 = a + vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7] + MY_ROR (22-13), y1 # y1 = a >> (22-13) + xor e, y0 # y0 = e ^ (e >> (25-11)) + mov f, y2 # y2 = f + MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor a, y1 # y1 = a ^ (a >> (22-13) + xor g, y2 # y2 = f^g + vpaddd X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16] + xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and e, y2 # y2 = (f^g)&e + MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) + ## compute s0 + vpalignr $4, X0, X1, XTMP1 # XTMP1 = W[-15] + xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor g, y2 # y2 = CH = ((f^g)&e)^g + MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y0, y2 # y2 = S1 + CH + add _XFER(%rsp), y2 # y2 = k + w + S1 + CH + mov a, y0 # y0 = a + add y2, h # h = h + S1 + CH + k + w + mov a, y2 # y2 = a + vpsrld $7, XTMP1, XTMP2 + or c, y0 # y0 = a|c + add h, d # d = d + h + S1 + CH + k + w + and c, y2 # y2 = a&c + vpslld $(32-7), XTMP1, XTMP3 + and b, y0 # y0 = (a|c)&b + add y1, h # h = h + S1 + CH + k + w + S0 + vpor XTMP2, XTMP3, XTMP3 # XTMP1 = W[-15] MY_ROR 7 + or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) + add y0, h # h = h + S1 + CH + k + w + S0 + MAJ + ROTATE_ARGS + mov e, y0 # y0 = e + mov a, y1 # y1 = a + MY_ROR (25-11), y0 # y0 = e >> (25-11) + xor e, y0 # y0 = e ^ (e >> (25-11)) + mov f, y2 # y2 = f + MY_ROR (22-13), y1 # y1 = a >> (22-13) + vpsrld $18, XTMP1, XTMP2 # + xor a, y1 # y1 = a ^ (a >> (22-13) + MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor g, y2 # y2 = f^g + vpsrld $3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3 + MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and e, y2 # y2 = (f^g)&e + MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + vpslld $(32-18), XTMP1, XTMP1 + xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor g, y2 # y2 = CH = ((f^g)&e)^g + vpxor XTMP1, XTMP3, XTMP3 # + add y0, y2 # y2 = S1 + CH + add (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH + MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + vpxor XTMP2, XTMP3, XTMP3 # XTMP1 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR + mov a, y0 # y0 = a + add y2, h # h = h + S1 + CH + k + w + mov a, y2 # y2 = a + vpxor XTMP4, XTMP3, XTMP1 # XTMP1 = s0 + or c, y0 # y0 = a|c + add h, d # d = d + h + S1 + CH + k + w + and c, y2 # y2 = a&c + ## compute low s1 + vpshufd $0b11111010, X3, XTMP2 # XTMP2 = W[-2] {BBAA} + and b, y0 # y0 = (a|c)&b + add y1, h # h = h + S1 + CH + k + w + S0 + vpaddd XTMP1, XTMP0, XTMP0 # XTMP0 = W[-16] + W[-7] + s0 + or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) + add y0, h # h = h + S1 + CH + k + w + S0 + MAJ + ROTATE_ARGS + mov e, y0 # y0 = e + mov a, y1 # y1 = a + MY_ROR (25-11), y0 # y0 = e >> (25-11) + xor e, y0 # y0 = e ^ (e >> (25-11)) + MY_ROR (22-13), y1 # y1 = a >> (22-13) + mov f, y2 # y2 = f + xor a, y1 # y1 = a ^ (a >> (22-13) + MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) + vpsrld $10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA} + xor g, y2 # y2 = f^g + vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] MY_ROR 19 {xBxA} + xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and e, y2 # y2 = (f^g)&e + vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] MY_ROR 17 {xBxA} + MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor g, y2 # y2 = CH = ((f^g)&e)^g + MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + vpxor XTMP3, XTMP2, XTMP2 # + add y0, y2 # y2 = S1 + CH + MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH + vpxor XTMP2, XTMP4, XTMP4 # XTMP4 = s1 {xBxA} + mov a, y0 # y0 = a + add y2, h # h = h + S1 + CH + k + w + mov a, y2 # y2 = a + vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA} + or c, y0 # y0 = a|c + add h, d # d = d + h + S1 + CH + k + w + and c, y2 # y2 = a&c + vpaddd XTMP4, XTMP0, XTMP0 # XTMP0 = {..., ..., W[1], W[0]} + and b, y0 # y0 = (a|c)&b + add y1, h # h = h + S1 + CH + k + w + S0 + ## compute high s1 + vpshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {DDCC} + or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) + add y0, h # h = h + S1 + CH + k + w + S0 + MAJ + ROTATE_ARGS + mov e, y0 # y0 = e + MY_ROR (25-11), y0 # y0 = e >> (25-11) + mov a, y1 # y1 = a + MY_ROR (22-13), y1 # y1 = a >> (22-13) + xor e, y0 # y0 = e ^ (e >> (25-11)) + mov f, y2 # y2 = f + MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) + vpsrld $10, XTMP2, XTMP5 # XTMP5 = W[-2] >> 10 {DDCC} + xor a, y1 # y1 = a ^ (a >> (22-13) + xor g, y2 # y2 = f^g + vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] MY_ROR 19 {xDxC} + xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and e, y2 # y2 = (f^g)&e + MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) + vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] MY_ROR 17 {xDxC} + xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor g, y2 # y2 = CH = ((f^g)&e)^g + vpxor XTMP3, XTMP2, XTMP2 + MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y0, y2 # y2 = S1 + CH + add (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH + vpxor XTMP2, XTMP5, XTMP5 # XTMP5 = s1 {xDxC} + mov a, y0 # y0 = a + add y2, h # h = h + S1 + CH + k + w + mov a, y2 # y2 = a + vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00} + or c, y0 # y0 = a|c + add h, d # d = d + h + S1 + CH + k + w + and c, y2 # y2 = a&c + vpaddd XTMP0, XTMP5, X0 # X0 = {W[3], W[2], W[1], W[0]} + and b, y0 # y0 = (a|c)&b + add y1, h # h = h + S1 + CH + k + w + S0 + or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) + add y0, h # h = h + S1 + CH + k + w + S0 + MAJ + ROTATE_ARGS + rotate_Xs +.endm + +## input is [rsp + _XFER + %1 * 4] +.macro DO_ROUND round + mov e, y0 # y0 = e + MY_ROR (25-11), y0 # y0 = e >> (25-11) + mov a, y1 # y1 = a + xor e, y0 # y0 = e ^ (e >> (25-11)) + MY_ROR (22-13), y1 # y1 = a >> (22-13) + mov f, y2 # y2 = f + xor a, y1 # y1 = a ^ (a >> (22-13) + MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor g, y2 # y2 = f^g + xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) + and e, y2 # y2 = (f^g)&e + xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor g, y2 # y2 = CH = ((f^g)&e)^g + add y0, y2 # y2 = S1 + CH + MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + offset = \round * 4 + _XFER # + add offset(%rsp), y2 # y2 = k + w + S1 + CH + mov a, y0 # y0 = a + add y2, h # h = h + S1 + CH + k + w + mov a, y2 # y2 = a + or c, y0 # y0 = a|c + add h, d # d = d + h + S1 + CH + k + w + and c, y2 # y2 = a&c + and b, y0 # y0 = (a|c)&b + add y1, h # h = h + S1 + CH + k + w + S0 + or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) + add y0, h # h = h + S1 + CH + k + w + S0 + MAJ + ROTATE_ARGS +.endm + +######################################################################## +## void sha256_transform_avx(void *input_data, UINT32 digest[8], UINT64 num_blks) +## arg 1 : pointer to input data +## arg 2 : pointer to digest +## arg 3 : Num blocks +######################################################################## +.text +ENTRY(sha256_transform_avx) +.align 32 + pushq %rbx + pushq %rbp + pushq %r13 + pushq %r14 + pushq %r15 + pushq %r12 + + mov %rsp, %r12 + subq $STACK_SIZE, %rsp # allocate stack space + and $~15, %rsp # align stack pointer + + shl $6, NUM_BLKS # convert to bytes + jz done_hash + add INP, NUM_BLKS # pointer to end of data + mov NUM_BLKS, _INP_END(%rsp) + + ## load initial digest + mov 4*0(CTX), a + mov 4*1(CTX), b + mov 4*2(CTX), c + mov 4*3(CTX), d + mov 4*4(CTX), e + mov 4*5(CTX), f + mov 4*6(CTX), g + mov 4*7(CTX), h + + vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK + vmovdqa _SHUF_00BA(%rip), SHUF_00BA + vmovdqa _SHUF_DC00(%rip), SHUF_DC00 +loop0: + lea K256(%rip), TBL + + ## byte swap first 16 dwords + COPY_XMM_AND_BSWAP X0, 0*16(INP), BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X1, 1*16(INP), BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X2, 2*16(INP), BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X3, 3*16(INP), BYTE_FLIP_MASK + + mov INP, _INP(%rsp) + + ## schedule 48 input dwords, by doing 3 rounds of 16 each + mov $3, SRND +.align 16 +loop1: + vpaddd (TBL), X0, XFER + vmovdqa XFER, _XFER(%rsp) + FOUR_ROUNDS_AND_SCHED + + vpaddd 1*16(TBL), X0, XFER + vmovdqa XFER, _XFER(%rsp) + FOUR_ROUNDS_AND_SCHED + + vpaddd 2*16(TBL), X0, XFER + vmovdqa XFER, _XFER(%rsp) + FOUR_ROUNDS_AND_SCHED + + vpaddd 3*16(TBL), X0, XFER + vmovdqa XFER, _XFER(%rsp) + add $4*16, TBL + FOUR_ROUNDS_AND_SCHED + + sub $1, SRND + jne loop1 + + mov $2, SRND +loop2: + vpaddd (TBL), X0, XFER + vmovdqa XFER, _XFER(%rsp) + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + + vpaddd 1*16(TBL), X1, XFER + vmovdqa XFER, _XFER(%rsp) + add $2*16, TBL + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + + vmovdqa X2, X0 + vmovdqa X3, X1 + + sub $1, SRND + jne loop2 + + addm (4*0)(CTX),a + addm (4*1)(CTX),b + addm (4*2)(CTX),c + addm (4*3)(CTX),d + addm (4*4)(CTX),e + addm (4*5)(CTX),f + addm (4*6)(CTX),g + addm (4*7)(CTX),h + + mov _INP(%rsp), INP + add $64, INP + cmp _INP_END(%rsp), INP + jne loop0 + +done_hash: + + mov %r12, %rsp + + popq %r12 + popq %r15 + popq %r14 + popq %r13 + popq %rbp + popq %rbx + ret +ENDPROC(sha256_transform_avx) + +.data +.align 64 +K256: + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +PSHUFFLE_BYTE_FLIP_MASK: + .octa 0x0c0d0e0f08090a0b0405060700010203 + +# shuffle xBxA -> 00BA +_SHUF_00BA: + .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100 + +# shuffle xDxC -> DC00 +_SHUF_DC00: + .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF +#endif -- cgit v0.10.2 From d34a460092d857f1616e39eed7eac6f40cea2225 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:59:10 -0700 Subject: crypto: sha256 - Optimized sha256 x86_64 routine using AVX2's RORX instructions Provides SHA256 x86_64 assembly routine optimized with SSE, AVX and AVX2's RORX instructions. Speedup of 70% or more has been measured over the generic implementation. Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/sha256-avx2-asm.S b/arch/x86/crypto/sha256-avx2-asm.S new file mode 100644 index 0000000..9e86944 --- /dev/null +++ b/arch/x86/crypto/sha256-avx2-asm.S @@ -0,0 +1,772 @@ +######################################################################## +# Implement fast SHA-256 with AVX2 instructions. (x86_64) +# +# Copyright (C) 2013 Intel Corporation. +# +# Authors: +# James Guilford +# Kirk Yap +# Tim Chen +# +# This software is available to you under a choice of one of two +# licenses. You may choose to be licensed under the terms of the GNU +# General Public License (GPL) Version 2, available from the file +# COPYING in the main directory of this source tree, or the +# OpenIB.org BSD license below: +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +######################################################################## +# +# This code is described in an Intel White-Paper: +# "Fast SHA-256 Implementations on Intel Architecture Processors" +# +# To find it, surf to http://www.intel.com/p/en_US/embedded +# and search for that title. +# +######################################################################## +# This code schedules 2 blocks at a time, with 4 lanes per block +######################################################################## + +#ifdef CONFIG_AS_AVX2 +#include + +## assume buffers not aligned +#define VMOVDQ vmovdqu + +################################ Define Macros + +# addm [mem], reg +# Add reg to mem using reg-mem add and store +.macro addm p1 p2 + add \p1, \p2 + mov \p2, \p1 +.endm + +################################ + +X0 = %ymm4 +X1 = %ymm5 +X2 = %ymm6 +X3 = %ymm7 + +# XMM versions of above +XWORD0 = %xmm4 +XWORD1 = %xmm5 +XWORD2 = %xmm6 +XWORD3 = %xmm7 + +XTMP0 = %ymm0 +XTMP1 = %ymm1 +XTMP2 = %ymm2 +XTMP3 = %ymm3 +XTMP4 = %ymm8 +XFER = %ymm9 +XTMP5 = %ymm11 + +SHUF_00BA = %ymm10 # shuffle xBxA -> 00BA +SHUF_DC00 = %ymm12 # shuffle xDxC -> DC00 +BYTE_FLIP_MASK = %ymm13 + +X_BYTE_FLIP_MASK = %xmm13 # XMM version of BYTE_FLIP_MASK + +NUM_BLKS = %rdx # 3rd arg +CTX = %rsi # 2nd arg +INP = %rdi # 1st arg +c = %ecx +d = %r8d +e = %edx # clobbers NUM_BLKS +y3 = %edi # clobbers INP + + +TBL = %rbp +SRND = CTX # SRND is same register as CTX + +a = %eax +b = %ebx +f = %r9d +g = %r10d +h = %r11d +old_h = %r11d + +T1 = %r12d +y0 = %r13d +y1 = %r14d +y2 = %r15d + + +_XFER_SIZE = 2*64*4 # 2 blocks, 64 rounds, 4 bytes/round +_XMM_SAVE_SIZE = 0 +_INP_END_SIZE = 8 +_INP_SIZE = 8 +_CTX_SIZE = 8 +_RSP_SIZE = 8 + +_XFER = 0 +_XMM_SAVE = _XFER + _XFER_SIZE +_INP_END = _XMM_SAVE + _XMM_SAVE_SIZE +_INP = _INP_END + _INP_END_SIZE +_CTX = _INP + _INP_SIZE +_RSP = _CTX + _CTX_SIZE +STACK_SIZE = _RSP + _RSP_SIZE + +# rotate_Xs +# Rotate values of symbols X0...X3 +.macro rotate_Xs + X_ = X0 + X0 = X1 + X1 = X2 + X2 = X3 + X3 = X_ +.endm + +# ROTATE_ARGS +# Rotate values of symbols a...h +.macro ROTATE_ARGS + old_h = h + TMP_ = h + h = g + g = f + f = e + e = d + d = c + c = b + b = a + a = TMP_ +.endm + +.macro FOUR_ROUNDS_AND_SCHED disp +################################### RND N + 0 ############################ + + mov a, y3 # y3 = a # MAJA + rorx $25, e, y0 # y0 = e >> 25 # S1A + rorx $11, e, y1 # y1 = e >> 11 # S1B + + addl \disp(%rsp, SRND), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7] + mov f, y2 # y2 = f # CH + rorx $13, a, T1 # T1 = a >> 13 # S0B + + xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 + xor g, y2 # y2 = f^g # CH + vpaddd X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16]# y1 = (e >> 6)# S1 + rorx $6, e, y1 # y1 = (e >> 6) # S1 + + and e, y2 # y2 = (f^g)&e # CH + xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 + rorx $22, a, y1 # y1 = a >> 22 # S0A + add h, d # d = k + w + h + d # -- + + and b, y3 # y3 = (a|c)&b # MAJA + vpalignr $4, X0, X1, XTMP1 # XTMP1 = W[-15] + xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 + rorx $2, a, T1 # T1 = (a >> 2) # S0 + + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + vpsrld $7, XTMP1, XTMP2 + xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 + mov a, T1 # T1 = a # MAJB + and c, T1 # T1 = a&c # MAJB + + add y0, y2 # y2 = S1 + CH # -- + vpslld $(32-7), XTMP1, XTMP3 + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + vpor XTMP2, XTMP3, XTMP3 # XTMP3 = W[-15] ror 7 + + vpsrld $18, XTMP1, XTMP2 + add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + add y3, h # h = t1 + S0 + MAJ # -- + + + ROTATE_ARGS + +################################### RND N + 1 ############################ + + mov a, y3 # y3 = a # MAJA + rorx $25, e, y0 # y0 = e >> 25 # S1A + rorx $11, e, y1 # y1 = e >> 11 # S1B + offset = \disp + 1*4 + addl offset(%rsp, SRND), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + + vpsrld $3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3 + mov f, y2 # y2 = f # CH + rorx $13, a, T1 # T1 = a >> 13 # S0B + xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 + xor g, y2 # y2 = f^g # CH + + + rorx $6, e, y1 # y1 = (e >> 6) # S1 + xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 + rorx $22, a, y1 # y1 = a >> 22 # S0A + and e, y2 # y2 = (f^g)&e # CH + add h, d # d = k + w + h + d # -- + + vpslld $(32-18), XTMP1, XTMP1 + and b, y3 # y3 = (a|c)&b # MAJA + xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 + + vpxor XTMP1, XTMP3, XTMP3 + rorx $2, a, T1 # T1 = (a >> 2) # S0 + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + + vpxor XTMP2, XTMP3, XTMP3 # XTMP3 = W[-15] ror 7 ^ W[-15] ror 18 + xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 + mov a, T1 # T1 = a # MAJB + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + vpxor XTMP4, XTMP3, XTMP1 # XTMP1 = s0 + vpshufd $0b11111010, X3, XTMP2 # XTMP2 = W[-2] {BBAA} + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + vpaddd XTMP1, XTMP0, XTMP0 # XTMP0 = W[-16] + W[-7] + s0 + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + add y3, h # h = t1 + S0 + MAJ # -- + + vpsrld $10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA} + + + ROTATE_ARGS + +################################### RND N + 2 ############################ + + mov a, y3 # y3 = a # MAJA + rorx $25, e, y0 # y0 = e >> 25 # S1A + offset = \disp + 2*4 + addl offset(%rsp, SRND), h # h = k + w + h # -- + + vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xBxA} + rorx $11, e, y1 # y1 = e >> 11 # S1B + or c, y3 # y3 = a|c # MAJA + mov f, y2 # y2 = f # CH + xor g, y2 # y2 = f^g # CH + + rorx $13, a, T1 # T1 = a >> 13 # S0B + xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 + vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] ror 17 {xBxA} + and e, y2 # y2 = (f^g)&e # CH + + rorx $6, e, y1 # y1 = (e >> 6) # S1 + vpxor XTMP3, XTMP2, XTMP2 + add h, d # d = k + w + h + d # -- + and b, y3 # y3 = (a|c)&b # MAJA + + xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 + rorx $22, a, y1 # y1 = a >> 22 # S0A + vpxor XTMP2, XTMP4, XTMP4 # XTMP4 = s1 {xBxA} + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + + vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA} + xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 + rorx $2, a ,T1 # T1 = (a >> 2) # S0 + vpaddd XTMP4, XTMP0, XTMP0 # XTMP0 = {..., ..., W[1], W[0]} + + xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 + mov a, T1 # T1 = a # MAJB + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + vpshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {DDCC} + + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1,h # h = k + w + h + S0 # -- + add y2,d # d = k + w + h + d + S1 + CH = d + t1 # -- + add y2,h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + + add y3,h # h = t1 + S0 + MAJ # -- + + + ROTATE_ARGS + +################################### RND N + 3 ############################ + + mov a, y3 # y3 = a # MAJA + rorx $25, e, y0 # y0 = e >> 25 # S1A + rorx $11, e, y1 # y1 = e >> 11 # S1B + offset = \disp + 3*4 + addl offset(%rsp, SRND), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + + vpsrld $10, XTMP2, XTMP5 # XTMP5 = W[-2] >> 10 {DDCC} + mov f, y2 # y2 = f # CH + rorx $13, a, T1 # T1 = a >> 13 # S0B + xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 + xor g, y2 # y2 = f^g # CH + + + vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xDxC} + rorx $6, e, y1 # y1 = (e >> 6) # S1 + and e, y2 # y2 = (f^g)&e # CH + add h, d # d = k + w + h + d # -- + and b, y3 # y3 = (a|c)&b # MAJA + + vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] ror 17 {xDxC} + xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + + vpxor XTMP3, XTMP2, XTMP2 + rorx $22, a, y1 # y1 = a >> 22 # S0A + add y0, y2 # y2 = S1 + CH # -- + + vpxor XTMP2, XTMP5, XTMP5 # XTMP5 = s1 {xDxC} + xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + rorx $2, a, T1 # T1 = (a >> 2) # S0 + vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00} + + vpaddd XTMP0, XTMP5, X0 # X0 = {W[3], W[2], W[1], W[0]} + xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 + mov a, T1 # T1 = a # MAJB + and c, T1 # T1 = a&c # MAJB + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + + add y1, h # h = k + w + h + S0 # -- + add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + add y3, h # h = t1 + S0 + MAJ # -- + + ROTATE_ARGS + rotate_Xs +.endm + +.macro DO_4ROUNDS disp +################################### RND N + 0 ########################### + + mov f, y2 # y2 = f # CH + rorx $25, e, y0 # y0 = e >> 25 # S1A + rorx $11, e, y1 # y1 = e >> 11 # S1B + xor g, y2 # y2 = f^g # CH + + xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 + rorx $6, e, y1 # y1 = (e >> 6) # S1 + and e, y2 # y2 = (f^g)&e # CH + + xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 + rorx $13, a, T1 # T1 = a >> 13 # S0B + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + rorx $22, a, y1 # y1 = a >> 22 # S0A + mov a, y3 # y3 = a # MAJA + + xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 + rorx $2, a, T1 # T1 = (a >> 2) # S0 + addl \disp(%rsp, SRND), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 + mov a, T1 # T1 = a # MAJB + and b, y3 # y3 = (a|c)&b # MAJA + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + + add h, d # d = k + w + h + d # -- + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + ROTATE_ARGS + +################################### RND N + 1 ########################### + + add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + mov f, y2 # y2 = f # CH + rorx $25, e, y0 # y0 = e >> 25 # S1A + rorx $11, e, y1 # y1 = e >> 11 # S1B + xor g, y2 # y2 = f^g # CH + + xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 + rorx $6, e, y1 # y1 = (e >> 6) # S1 + and e, y2 # y2 = (f^g)&e # CH + add y3, old_h # h = t1 + S0 + MAJ # -- + + xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 + rorx $13, a, T1 # T1 = a >> 13 # S0B + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + rorx $22, a, y1 # y1 = a >> 22 # S0A + mov a, y3 # y3 = a # MAJA + + xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 + rorx $2, a, T1 # T1 = (a >> 2) # S0 + offset = 4*1 + \disp + addl offset(%rsp, SRND), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 + mov a, T1 # T1 = a # MAJB + and b, y3 # y3 = (a|c)&b # MAJA + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + + add h, d # d = k + w + h + d # -- + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + ROTATE_ARGS + +################################### RND N + 2 ############################## + + add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + mov f, y2 # y2 = f # CH + rorx $25, e, y0 # y0 = e >> 25 # S1A + rorx $11, e, y1 # y1 = e >> 11 # S1B + xor g, y2 # y2 = f^g # CH + + xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 + rorx $6, e, y1 # y1 = (e >> 6) # S1 + and e, y2 # y2 = (f^g)&e # CH + add y3, old_h # h = t1 + S0 + MAJ # -- + + xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 + rorx $13, a, T1 # T1 = a >> 13 # S0B + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + rorx $22, a, y1 # y1 = a >> 22 # S0A + mov a, y3 # y3 = a # MAJA + + xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 + rorx $2, a, T1 # T1 = (a >> 2) # S0 + offset = 4*2 + \disp + addl offset(%rsp, SRND), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 + mov a, T1 # T1 = a # MAJB + and b, y3 # y3 = (a|c)&b # MAJA + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + + add h, d # d = k + w + h + d # -- + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + ROTATE_ARGS + +################################### RND N + 3 ########################### + + add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + mov f, y2 # y2 = f # CH + rorx $25, e, y0 # y0 = e >> 25 # S1A + rorx $11, e, y1 # y1 = e >> 11 # S1B + xor g, y2 # y2 = f^g # CH + + xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 + rorx $6, e, y1 # y1 = (e >> 6) # S1 + and e, y2 # y2 = (f^g)&e # CH + add y3, old_h # h = t1 + S0 + MAJ # -- + + xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 + rorx $13, a, T1 # T1 = a >> 13 # S0B + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + rorx $22, a, y1 # y1 = a >> 22 # S0A + mov a, y3 # y3 = a # MAJA + + xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 + rorx $2, a, T1 # T1 = (a >> 2) # S0 + offset = 4*3 + \disp + addl offset(%rsp, SRND), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 + mov a, T1 # T1 = a # MAJB + and b, y3 # y3 = (a|c)&b # MAJA + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + + add h, d # d = k + w + h + d # -- + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + + add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + + add y3, h # h = t1 + S0 + MAJ # -- + + ROTATE_ARGS + +.endm + +######################################################################## +## void sha256_transform_rorx(void *input_data, UINT32 digest[8], UINT64 num_blks) +## arg 1 : pointer to input data +## arg 2 : pointer to digest +## arg 3 : Num blocks +######################################################################## +.text +ENTRY(sha256_transform_rorx) +.align 32 + pushq %rbx + pushq %rbp + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + + mov %rsp, %rax + subq $STACK_SIZE, %rsp + and $-32, %rsp # align rsp to 32 byte boundary + mov %rax, _RSP(%rsp) + + + shl $6, NUM_BLKS # convert to bytes + jz done_hash + lea -64(INP, NUM_BLKS), NUM_BLKS # pointer to last block + mov NUM_BLKS, _INP_END(%rsp) + + cmp NUM_BLKS, INP + je only_one_block + + ## load initial digest + mov (CTX), a + mov 4*1(CTX), b + mov 4*2(CTX), c + mov 4*3(CTX), d + mov 4*4(CTX), e + mov 4*5(CTX), f + mov 4*6(CTX), g + mov 4*7(CTX), h + + vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK + vmovdqa _SHUF_00BA(%rip), SHUF_00BA + vmovdqa _SHUF_DC00(%rip), SHUF_DC00 + + mov CTX, _CTX(%rsp) + +loop0: + lea K256(%rip), TBL + + ## Load first 16 dwords from two blocks + VMOVDQ 0*32(INP),XTMP0 + VMOVDQ 1*32(INP),XTMP1 + VMOVDQ 2*32(INP),XTMP2 + VMOVDQ 3*32(INP),XTMP3 + + ## byte swap data + vpshufb BYTE_FLIP_MASK, XTMP0, XTMP0 + vpshufb BYTE_FLIP_MASK, XTMP1, XTMP1 + vpshufb BYTE_FLIP_MASK, XTMP2, XTMP2 + vpshufb BYTE_FLIP_MASK, XTMP3, XTMP3 + + ## transpose data into high/low halves + vperm2i128 $0x20, XTMP2, XTMP0, X0 + vperm2i128 $0x31, XTMP2, XTMP0, X1 + vperm2i128 $0x20, XTMP3, XTMP1, X2 + vperm2i128 $0x31, XTMP3, XTMP1, X3 + +last_block_enter: + add $64, INP + mov INP, _INP(%rsp) + + ## schedule 48 input dwords, by doing 3 rounds of 12 each + xor SRND, SRND + +.align 16 +loop1: + vpaddd 0*32(TBL, SRND), X0, XFER + vmovdqa XFER, 0*32+_XFER(%rsp, SRND) + FOUR_ROUNDS_AND_SCHED _XFER + 0*32 + + vpaddd 1*32(TBL, SRND), X0, XFER + vmovdqa XFER, 1*32+_XFER(%rsp, SRND) + FOUR_ROUNDS_AND_SCHED _XFER + 1*32 + + vpaddd 2*32(TBL, SRND), X0, XFER + vmovdqa XFER, 2*32+_XFER(%rsp, SRND) + FOUR_ROUNDS_AND_SCHED _XFER + 2*32 + + vpaddd 3*32(TBL, SRND), X0, XFER + vmovdqa XFER, 3*32+_XFER(%rsp, SRND) + FOUR_ROUNDS_AND_SCHED _XFER + 3*32 + + add $4*32, SRND + cmp $3*4*32, SRND + jb loop1 + +loop2: + ## Do last 16 rounds with no scheduling + vpaddd 0*32(TBL, SRND), X0, XFER + vmovdqa XFER, 0*32+_XFER(%rsp, SRND) + DO_4ROUNDS _XFER + 0*32 + vpaddd 1*32(TBL, SRND), X1, XFER + vmovdqa XFER, 1*32+_XFER(%rsp, SRND) + DO_4ROUNDS _XFER + 1*32 + add $2*32, SRND + + vmovdqa X2, X0 + vmovdqa X3, X1 + + cmp $4*4*32, SRND + jb loop2 + + mov _CTX(%rsp), CTX + mov _INP(%rsp), INP + + addm (4*0)(CTX),a + addm (4*1)(CTX),b + addm (4*2)(CTX),c + addm (4*3)(CTX),d + addm (4*4)(CTX),e + addm (4*5)(CTX),f + addm (4*6)(CTX),g + addm (4*7)(CTX),h + + cmp _INP_END(%rsp), INP + ja done_hash + + #### Do second block using previously scheduled results + xor SRND, SRND +.align 16 +loop3: + DO_4ROUNDS _XFER + 0*32 + 16 + DO_4ROUNDS _XFER + 1*32 + 16 + add $2*32, SRND + cmp $4*4*32, SRND + jb loop3 + + mov _CTX(%rsp), CTX + mov _INP(%rsp), INP + add $64, INP + + addm (4*0)(CTX),a + addm (4*1)(CTX),b + addm (4*2)(CTX),c + addm (4*3)(CTX),d + addm (4*4)(CTX),e + addm (4*5)(CTX),f + addm (4*6)(CTX),g + addm (4*7)(CTX),h + + cmp _INP_END(%rsp), INP + jb loop0 + ja done_hash + +do_last_block: + #### do last block + lea K256(%rip), TBL + + VMOVDQ 0*16(INP),XWORD0 + VMOVDQ 1*16(INP),XWORD1 + VMOVDQ 2*16(INP),XWORD2 + VMOVDQ 3*16(INP),XWORD3 + + vpshufb X_BYTE_FLIP_MASK, XWORD0, XWORD0 + vpshufb X_BYTE_FLIP_MASK, XWORD1, XWORD1 + vpshufb X_BYTE_FLIP_MASK, XWORD2, XWORD2 + vpshufb X_BYTE_FLIP_MASK, XWORD3, XWORD3 + + jmp last_block_enter + +only_one_block: + + ## load initial digest + mov (4*0)(CTX),a + mov (4*1)(CTX),b + mov (4*2)(CTX),c + mov (4*3)(CTX),d + mov (4*4)(CTX),e + mov (4*5)(CTX),f + mov (4*6)(CTX),g + mov (4*7)(CTX),h + + vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK + vmovdqa _SHUF_00BA(%rip), SHUF_00BA + vmovdqa _SHUF_DC00(%rip), SHUF_DC00 + + mov CTX, _CTX(%rsp) + jmp do_last_block + +done_hash: + + mov _RSP(%rsp), %rsp + + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + ret +ENDPROC(sha256_transform_rorx) + +.data +.align 64 +K256: + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +PSHUFFLE_BYTE_FLIP_MASK: + .octa 0x0c0d0e0f08090a0b0405060700010203,0x0c0d0e0f08090a0b0405060700010203 + +# shuffle xBxA -> 00BA +_SHUF_00BA: + .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100,0xFFFFFFFFFFFFFFFF0b0a090803020100 + +# shuffle xDxC -> DC00 +_SHUF_DC00: + .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF,0x0b0a090803020100FFFFFFFFFFFFFFFF +#endif -- cgit v0.10.2 From 8275d1aa642295edd34a11a117080384bb9d65c2 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:59:17 -0700 Subject: crypto: sha256 - Create module providing optimized SHA256 routines using SSSE3, AVX or AVX2 instructions. We added glue code and config options to create crypto module that uses SSE/AVX/AVX2 optimized SHA256 x86_64 assembly routines. Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 83681a3..9414b91 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -25,6 +25,7 @@ obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o +obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o # These modules require assembler to support AVX. ifeq ($(avx_supported),yes) @@ -66,3 +67,4 @@ sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o crc32c-intel-y := crc32c-intel_glue.o crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o +sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c new file mode 100644 index 0000000..597d4da --- /dev/null +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -0,0 +1,275 @@ +/* + * Cryptographic API. + * + * Glue code for the SHA256 Secure Hash Algorithm assembler + * implementation using supplemental SSE3 / AVX / AVX2 instructions. + * + * This file is based on sha256_generic.c + * + * Copyright (C) 2013 Intel Corporation. + * + * Author: + * Tim Chen + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +asmlinkage void sha256_transform_ssse3(const char *data, u32 *digest, + u64 rounds); +#ifdef CONFIG_AS_AVX +asmlinkage void sha256_transform_avx(const char *data, u32 *digest, + u64 rounds); +#endif +#ifdef CONFIG_AS_AVX2 +asmlinkage void sha256_transform_rorx(const char *data, u32 *digest, + u64 rounds); +#endif + +static asmlinkage void (*sha256_transform_asm)(const char *, u32 *, u64); + + +static int sha256_ssse3_init(struct shash_desc *desc) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA256_H0; + sctx->state[1] = SHA256_H1; + sctx->state[2] = SHA256_H2; + sctx->state[3] = SHA256_H3; + sctx->state[4] = SHA256_H4; + sctx->state[5] = SHA256_H5; + sctx->state[6] = SHA256_H6; + sctx->state[7] = SHA256_H7; + sctx->count = 0; + + return 0; +} + +static int __sha256_ssse3_update(struct shash_desc *desc, const u8 *data, + unsigned int len, unsigned int partial) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + unsigned int done = 0; + + sctx->count += len; + + if (partial) { + done = SHA256_BLOCK_SIZE - partial; + memcpy(sctx->buf + partial, data, done); + sha256_transform_asm(sctx->buf, sctx->state, 1); + } + + if (len - done >= SHA256_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA256_BLOCK_SIZE; + + sha256_transform_asm(data + done, sctx->state, (u64) rounds); + + done += rounds * SHA256_BLOCK_SIZE; + } + + memcpy(sctx->buf, data + done, len - done); + + return 0; +} + +static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; + int res; + + /* Handle the fast case right here */ + if (partial + len < SHA256_BLOCK_SIZE) { + sctx->count += len; + memcpy(sctx->buf + partial, data, len); + + return 0; + } + + if (!irq_fpu_usable()) { + res = crypto_sha256_update(desc, data, len); + } else { + kernel_fpu_begin(); + res = __sha256_ssse3_update(desc, data, len, partial); + kernel_fpu_end(); + } + + return res; +} + + +/* Add padding and return the message digest. */ +static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be32 *dst = (__be32 *)out; + __be64 bits; + static const u8 padding[SHA256_BLOCK_SIZE] = { 0x80, }; + + bits = cpu_to_be64(sctx->count << 3); + + /* Pad out to 56 mod 64 and append length */ + index = sctx->count % SHA256_BLOCK_SIZE; + padlen = (index < 56) ? (56 - index) : ((SHA256_BLOCK_SIZE+56)-index); + + if (!irq_fpu_usable()) { + crypto_sha256_update(desc, padding, padlen); + crypto_sha256_update(desc, (const u8 *)&bits, sizeof(bits)); + } else { + kernel_fpu_begin(); + /* We need to fill a whole block for __sha256_ssse3_update() */ + if (padlen <= 56) { + sctx->count += padlen; + memcpy(sctx->buf + index, padding, padlen); + } else { + __sha256_ssse3_update(desc, padding, padlen, index); + } + __sha256_ssse3_update(desc, (const u8 *)&bits, + sizeof(bits), 56); + kernel_fpu_end(); + } + + /* Store state in digest */ + for (i = 0; i < 8; i++) + dst[i] = cpu_to_be32(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha256_ssse3_export(struct shash_desc *desc, void *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha256_ssse3_import(struct shash_desc *desc, const void *in) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static struct shash_alg alg = { + .digestsize = SHA256_DIGEST_SIZE, + .init = sha256_ssse3_init, + .update = sha256_ssse3_update, + .final = sha256_ssse3_final, + .export = sha256_ssse3_export, + .import = sha256_ssse3_import, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-ssse3", + .cra_priority = 150, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +#ifdef CONFIG_AS_AVX +static bool __init avx_usable(void) +{ + u64 xcr0; + + if (!cpu_has_avx || !cpu_has_osxsave) + return false; + + xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); + if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { + pr_info("AVX detected but unusable.\n"); + + return false; + } + + return true; +} +#endif + +static int __init sha256_ssse3_mod_init(void) +{ + /* test for SSE3 first */ + if (cpu_has_ssse3) + sha256_transform_asm = sha256_transform_ssse3; + +#ifdef CONFIG_AS_AVX + /* allow AVX to override SSSE3, it's a little faster */ + if (avx_usable()) { +#ifdef CONFIG_AS_AVX2 + if (boot_cpu_has(X86_FEATURE_AVX2)) + sha256_transform_asm = sha256_transform_rorx; + else +#endif + sha256_transform_asm = sha256_transform_avx; + } +#endif + + if (sha256_transform_asm) { +#ifdef CONFIG_AS_AVX + if (sha256_transform_asm == sha256_transform_avx) + pr_info("Using AVX optimized SHA-256 implementation\n"); +#ifdef CONFIG_AS_AVX2 + else if (sha256_transform_asm == sha256_transform_rorx) + pr_info("Using AVX2 optimized SHA-256 implementation\n"); +#endif + else +#endif + pr_info("Using SSSE3 optimized SHA-256 implementation\n"); + return crypto_register_shash(&alg); + } + pr_info("Neither AVX nor SSSE3 is available/usable.\n"); + + return -ENODEV; +} + +static void __exit sha256_ssse3_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(sha256_ssse3_mod_init); +module_exit(sha256_ssse3_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated"); + +MODULE_ALIAS("sha256"); diff --git a/crypto/Kconfig b/crypto/Kconfig index aed52b2..8064ef1 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -470,6 +470,17 @@ config CRYPTO_SHA1_SSSE3 using Supplemental SSE3 (SSSE3) instructions or Advanced Vector Extensions (AVX), when available. +config CRYPTO_SHA256_SSSE3 + tristate "SHA256 digest algorithm (SSSE3/AVX/AVX2)" + depends on X86 && 64BIT + select CRYPTO_SHA256 + select CRYPTO_HASH + help + SHA-256 secure hash standard (DFIPS 180-2) implemented + using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector + Extensions version 1 (AVX1), or Advanced Vector Extensions + version 2 (AVX2) instructions, when available. + config CRYPTO_SHA1_SPARC64 tristate "SHA1 digest algorithm (SPARC64)" depends on SPARC64 -- cgit v0.10.2 From bf70fa9d9ee07aa175453b19a39b2b9dab602d97 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:59:25 -0700 Subject: crypto: sha512 - Expose generic sha512 routine to be callable from other modules Other SHA512 routines may need to use the generic routine when FPU is not available. Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index 71fcf36..4c58620 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -163,8 +163,8 @@ sha384_init(struct shash_desc *desc) return 0; } -static int -sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) +int crypto_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { struct sha512_state *sctx = shash_desc_ctx(desc); @@ -197,6 +197,7 @@ sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) return 0; } +EXPORT_SYMBOL(crypto_sha512_update); static int sha512_final(struct shash_desc *desc, u8 *hash) @@ -215,10 +216,10 @@ sha512_final(struct shash_desc *desc, u8 *hash) /* Pad out to 112 mod 128. */ index = sctx->count[0] & 0x7f; pad_len = (index < 112) ? (112 - index) : ((128+112) - index); - sha512_update(desc, padding, pad_len); + crypto_sha512_update(desc, padding, pad_len); /* Append length (before padding) */ - sha512_update(desc, (const u8 *)bits, sizeof(bits)); + crypto_sha512_update(desc, (const u8 *)bits, sizeof(bits)); /* Store state in digest */ for (i = 0; i < 8; i++) @@ -245,7 +246,7 @@ static int sha384_final(struct shash_desc *desc, u8 *hash) static struct shash_alg sha512_algs[2] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_init, - .update = sha512_update, + .update = crypto_sha512_update, .final = sha512_final, .descsize = sizeof(struct sha512_state), .base = { @@ -257,7 +258,7 @@ static struct shash_alg sha512_algs[2] = { { }, { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_init, - .update = sha512_update, + .update = crypto_sha512_update, .final = sha384_final, .descsize = sizeof(struct sha512_state), .base = { diff --git a/include/crypto/sha.h b/include/crypto/sha.h index f46ff61..190f8a0 100644 --- a/include/crypto/sha.h +++ b/include/crypto/sha.h @@ -89,4 +89,7 @@ extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len); + +extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len); #endif -- cgit v0.10.2 From bf215cee23ad6e278bfba1291863718934de392a Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:59:37 -0700 Subject: crypto: sha512 - Optimized SHA512 x86_64 assembly routine using Supplemental SSE3 instructions. Provides SHA512 x86_64 assembly routine optimized with SSSE3 instructions. Speedup of 40% or more has been measured over the generic implementation. Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/arch/x86/crypto/sha512-ssse3-asm.S new file mode 100644 index 0000000..fb56855 --- /dev/null +++ b/arch/x86/crypto/sha512-ssse3-asm.S @@ -0,0 +1,421 @@ +######################################################################## +# Implement fast SHA-512 with SSSE3 instructions. (x86_64) +# +# Copyright (C) 2013 Intel Corporation. +# +# Authors: +# James Guilford +# Kirk Yap +# David Cote +# Tim Chen +# +# This software is available to you under a choice of one of two +# licenses. You may choose to be licensed under the terms of the GNU +# General Public License (GPL) Version 2, available from the file +# COPYING in the main directory of this source tree, or the +# OpenIB.org BSD license below: +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +######################################################################## +# +# This code is described in an Intel White-Paper: +# "Fast SHA-512 Implementations on Intel Architecture Processors" +# +# To find it, surf to http://www.intel.com/p/en_US/embedded +# and search for that title. +# +######################################################################## + +#include + +.text + +# Virtual Registers +# ARG1 +msg = %rdi +# ARG2 +digest = %rsi +# ARG3 +msglen = %rdx +T1 = %rcx +T2 = %r8 +a_64 = %r9 +b_64 = %r10 +c_64 = %r11 +d_64 = %r12 +e_64 = %r13 +f_64 = %r14 +g_64 = %r15 +h_64 = %rbx +tmp0 = %rax + +# Local variables (stack frame) + +W_SIZE = 80*8 +WK_SIZE = 2*8 +RSPSAVE_SIZE = 1*8 +GPRSAVE_SIZE = 5*8 + +frame_W = 0 +frame_WK = frame_W + W_SIZE +frame_RSPSAVE = frame_WK + WK_SIZE +frame_GPRSAVE = frame_RSPSAVE + RSPSAVE_SIZE +frame_size = frame_GPRSAVE + GPRSAVE_SIZE + +# Useful QWORD "arrays" for simpler memory references +# MSG, DIGEST, K_t, W_t are arrays +# WK_2(t) points to 1 of 2 qwords at frame.WK depdending on t being odd/even + +# Input message (arg1) +#define MSG(i) 8*i(msg) + +# Output Digest (arg2) +#define DIGEST(i) 8*i(digest) + +# SHA Constants (static mem) +#define K_t(i) 8*i+K512(%rip) + +# Message Schedule (stack frame) +#define W_t(i) 8*i+frame_W(%rsp) + +# W[t]+K[t] (stack frame) +#define WK_2(i) 8*((i%2))+frame_WK(%rsp) + +.macro RotateState + # Rotate symbols a..h right + TMP = h_64 + h_64 = g_64 + g_64 = f_64 + f_64 = e_64 + e_64 = d_64 + d_64 = c_64 + c_64 = b_64 + b_64 = a_64 + a_64 = TMP +.endm + +.macro SHA512_Round rnd + + # Compute Round %%t + mov f_64, T1 # T1 = f + mov e_64, tmp0 # tmp = e + xor g_64, T1 # T1 = f ^ g + ror $23, tmp0 # 41 # tmp = e ror 23 + and e_64, T1 # T1 = (f ^ g) & e + xor e_64, tmp0 # tmp = (e ror 23) ^ e + xor g_64, T1 # T1 = ((f ^ g) & e) ^ g = CH(e,f,g) + idx = \rnd + add WK_2(idx), T1 # W[t] + K[t] from message scheduler + ror $4, tmp0 # 18 # tmp = ((e ror 23) ^ e) ror 4 + xor e_64, tmp0 # tmp = (((e ror 23) ^ e) ror 4) ^ e + mov a_64, T2 # T2 = a + add h_64, T1 # T1 = CH(e,f,g) + W[t] + K[t] + h + ror $14, tmp0 # 14 # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e) + add tmp0, T1 # T1 = CH(e,f,g) + W[t] + K[t] + S1(e) + mov a_64, tmp0 # tmp = a + xor c_64, T2 # T2 = a ^ c + and c_64, tmp0 # tmp = a & c + and b_64, T2 # T2 = (a ^ c) & b + xor tmp0, T2 # T2 = ((a ^ c) & b) ^ (a & c) = Maj(a,b,c) + mov a_64, tmp0 # tmp = a + ror $5, tmp0 # 39 # tmp = a ror 5 + xor a_64, tmp0 # tmp = (a ror 5) ^ a + add T1, d_64 # e(next_state) = d + T1 + ror $6, tmp0 # 34 # tmp = ((a ror 5) ^ a) ror 6 + xor a_64, tmp0 # tmp = (((a ror 5) ^ a) ror 6) ^ a + lea (T1, T2), h_64 # a(next_state) = T1 + Maj(a,b,c) + ror $28, tmp0 # 28 # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a) + add tmp0, h_64 # a(next_state) = T1 + Maj(a,b,c) S0(a) + RotateState +.endm + +.macro SHA512_2Sched_2Round_sse rnd + + # Compute rounds t-2 and t-1 + # Compute message schedule QWORDS t and t+1 + + # Two rounds are computed based on the values for K[t-2]+W[t-2] and + # K[t-1]+W[t-1] which were previously stored at WK_2 by the message + # scheduler. + # The two new schedule QWORDS are stored at [W_t(%%t)] and [W_t(%%t+1)]. + # They are then added to their respective SHA512 constants at + # [K_t(%%t)] and [K_t(%%t+1)] and stored at dqword [WK_2(%%t)] + # For brievity, the comments following vectored instructions only refer to + # the first of a pair of QWORDS. + # Eg. XMM2=W[t-2] really means XMM2={W[t-2]|W[t-1]} + # The computation of the message schedule and the rounds are tightly + # stitched to take advantage of instruction-level parallelism. + # For clarity, integer instructions (for the rounds calculation) are indented + # by one tab. Vectored instructions (for the message scheduler) are indented + # by two tabs. + + mov f_64, T1 + idx = \rnd -2 + movdqa W_t(idx), %xmm2 # XMM2 = W[t-2] + xor g_64, T1 + and e_64, T1 + movdqa %xmm2, %xmm0 # XMM0 = W[t-2] + xor g_64, T1 + idx = \rnd + add WK_2(idx), T1 + idx = \rnd - 15 + movdqu W_t(idx), %xmm5 # XMM5 = W[t-15] + mov e_64, tmp0 + ror $23, tmp0 # 41 + movdqa %xmm5, %xmm3 # XMM3 = W[t-15] + xor e_64, tmp0 + ror $4, tmp0 # 18 + psrlq $61-19, %xmm0 # XMM0 = W[t-2] >> 42 + xor e_64, tmp0 + ror $14, tmp0 # 14 + psrlq $(8-7), %xmm3 # XMM3 = W[t-15] >> 1 + add tmp0, T1 + add h_64, T1 + pxor %xmm2, %xmm0 # XMM0 = (W[t-2] >> 42) ^ W[t-2] + mov a_64, T2 + xor c_64, T2 + pxor %xmm5, %xmm3 # XMM3 = (W[t-15] >> 1) ^ W[t-15] + and b_64, T2 + mov a_64, tmp0 + psrlq $(19-6), %xmm0 # XMM0 = ((W[t-2]>>42)^W[t-2])>>13 + and c_64, tmp0 + xor tmp0, T2 + psrlq $(7-1), %xmm3 # XMM3 = ((W[t-15]>>1)^W[t-15])>>6 + mov a_64, tmp0 + ror $5, tmp0 # 39 + pxor %xmm2, %xmm0 # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2] + xor a_64, tmp0 + ror $6, tmp0 # 34 + pxor %xmm5, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15] + xor a_64, tmp0 + ror $28, tmp0 # 28 + psrlq $6, %xmm0 # XMM0 = ((((W[t-2]>>42)^W[t-2])>>13)^W[t-2])>>6 + add tmp0, T2 + add T1, d_64 + psrlq $1, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]>>1 + lea (T1, T2), h_64 + RotateState + movdqa %xmm2, %xmm1 # XMM1 = W[t-2] + mov f_64, T1 + xor g_64, T1 + movdqa %xmm5, %xmm4 # XMM4 = W[t-15] + and e_64, T1 + xor g_64, T1 + psllq $(64-19)-(64-61) , %xmm1 # XMM1 = W[t-2] << 42 + idx = \rnd + 1 + add WK_2(idx), T1 + mov e_64, tmp0 + psllq $(64-1)-(64-8), %xmm4 # XMM4 = W[t-15] << 7 + ror $23, tmp0 # 41 + xor e_64, tmp0 + pxor %xmm2, %xmm1 # XMM1 = (W[t-2] << 42)^W[t-2] + ror $4, tmp0 # 18 + xor e_64, tmp0 + pxor %xmm5, %xmm4 # XMM4 = (W[t-15]<<7)^W[t-15] + ror $14, tmp0 # 14 + add tmp0, T1 + psllq $(64-61), %xmm1 # XMM1 = ((W[t-2] << 42)^W[t-2])<<3 + add h_64, T1 + mov a_64, T2 + psllq $(64-8), %xmm4 # XMM4 = ((W[t-15]<<7)^W[t-15])<<56 + xor c_64, T2 + and b_64, T2 + pxor %xmm1, %xmm0 # XMM0 = s1(W[t-2]) + mov a_64, tmp0 + and c_64, tmp0 + idx = \rnd - 7 + movdqu W_t(idx), %xmm1 # XMM1 = W[t-7] + xor tmp0, T2 + pxor %xmm4, %xmm3 # XMM3 = s0(W[t-15]) + mov a_64, tmp0 + paddq %xmm3, %xmm0 # XMM0 = s1(W[t-2]) + s0(W[t-15]) + ror $5, tmp0 # 39 + idx =\rnd-16 + paddq W_t(idx), %xmm0 # XMM0 = s1(W[t-2]) + s0(W[t-15]) + W[t-16] + xor a_64, tmp0 + paddq %xmm1, %xmm0 # XMM0 = s1(W[t-2]) + W[t-7] + s0(W[t-15]) + W[t-16] + ror $6, tmp0 # 34 + movdqa %xmm0, W_t(\rnd) # Store scheduled qwords + xor a_64, tmp0 + paddq K_t(\rnd), %xmm0 # Compute W[t]+K[t] + ror $28, tmp0 # 28 + idx = \rnd + movdqa %xmm0, WK_2(idx) # Store W[t]+K[t] for next rounds + add tmp0, T2 + add T1, d_64 + lea (T1, T2), h_64 + RotateState +.endm + +######################################################################## +# void sha512_transform_ssse3(const void* M, void* D, u64 L)# +# Purpose: Updates the SHA512 digest stored at D with the message stored in M. +# The size of the message pointed to by M must be an integer multiple of SHA512 +# message blocks. +# L is the message length in SHA512 blocks. +######################################################################## +ENTRY(sha512_transform_ssse3) + + cmp $0, msglen + je nowork + + # Allocate Stack Space + mov %rsp, %rax + sub $frame_size, %rsp + and $~(0x20 - 1), %rsp + mov %rax, frame_RSPSAVE(%rsp) + + # Save GPRs + mov %rbx, frame_GPRSAVE(%rsp) + mov %r12, frame_GPRSAVE +8*1(%rsp) + mov %r13, frame_GPRSAVE +8*2(%rsp) + mov %r14, frame_GPRSAVE +8*3(%rsp) + mov %r15, frame_GPRSAVE +8*4(%rsp) + +updateblock: + +# Load state variables + mov DIGEST(0), a_64 + mov DIGEST(1), b_64 + mov DIGEST(2), c_64 + mov DIGEST(3), d_64 + mov DIGEST(4), e_64 + mov DIGEST(5), f_64 + mov DIGEST(6), g_64 + mov DIGEST(7), h_64 + + t = 0 + .rept 80/2 + 1 + # (80 rounds) / (2 rounds/iteration) + (1 iteration) + # +1 iteration because the scheduler leads hashing by 1 iteration + .if t < 2 + # BSWAP 2 QWORDS + movdqa XMM_QWORD_BSWAP(%rip), %xmm1 + movdqu MSG(t), %xmm0 + pshufb %xmm1, %xmm0 # BSWAP + movdqa %xmm0, W_t(t) # Store Scheduled Pair + paddq K_t(t), %xmm0 # Compute W[t]+K[t] + movdqa %xmm0, WK_2(t) # Store into WK for rounds + .elseif t < 16 + # BSWAP 2 QWORDS# Compute 2 Rounds + movdqu MSG(t), %xmm0 + pshufb %xmm1, %xmm0 # BSWAP + SHA512_Round t-2 # Round t-2 + movdqa %xmm0, W_t(t) # Store Scheduled Pair + paddq K_t(t), %xmm0 # Compute W[t]+K[t] + SHA512_Round t-1 # Round t-1 + movdqa %xmm0, WK_2(t) # Store W[t]+K[t] into WK + .elseif t < 79 + # Schedule 2 QWORDS# Compute 2 Rounds + SHA512_2Sched_2Round_sse t + .else + # Compute 2 Rounds + SHA512_Round t-2 + SHA512_Round t-1 + .endif + t = t+2 + .endr + + # Update digest + add a_64, DIGEST(0) + add b_64, DIGEST(1) + add c_64, DIGEST(2) + add d_64, DIGEST(3) + add e_64, DIGEST(4) + add f_64, DIGEST(5) + add g_64, DIGEST(6) + add h_64, DIGEST(7) + + # Advance to next message block + add $16*8, msg + dec msglen + jnz updateblock + + # Restore GPRs + mov frame_GPRSAVE(%rsp), %rbx + mov frame_GPRSAVE +8*1(%rsp), %r12 + mov frame_GPRSAVE +8*2(%rsp), %r13 + mov frame_GPRSAVE +8*3(%rsp), %r14 + mov frame_GPRSAVE +8*4(%rsp), %r15 + + # Restore Stack Pointer + mov frame_RSPSAVE(%rsp), %rsp + +nowork: + ret +ENDPROC(sha512_transform_ssse3) + +######################################################################## +### Binary Data + +.data + +.align 16 + +# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. +XMM_QWORD_BSWAP: + .octa 0x08090a0b0c0d0e0f0001020304050607 + +# K[t] used in SHA512 hashing +K512: + .quad 0x428a2f98d728ae22,0x7137449123ef65cd + .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc + .quad 0x3956c25bf348b538,0x59f111f1b605d019 + .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 + .quad 0xd807aa98a3030242,0x12835b0145706fbe + .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 + .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 + .quad 0x9bdc06a725c71235,0xc19bf174cf692694 + .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 + .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65 + .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 + .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 + .quad 0x983e5152ee66dfab,0xa831c66d2db43210 + .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 + .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 + .quad 0x06ca6351e003826f,0x142929670a0e6e70 + .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 + .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df + .quad 0x650a73548baf63de,0x766a0abb3c77b2a8 + .quad 0x81c2c92e47edaee6,0x92722c851482353b + .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 + .quad 0xc24b8b70d0f89791,0xc76c51a30654be30 + .quad 0xd192e819d6ef5218,0xd69906245565a910 + .quad 0xf40e35855771202a,0x106aa07032bbd1b8 + .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 + .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 + .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb + .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 + .quad 0x748f82ee5defb2fc,0x78a5636f43172f60 + .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec + .quad 0x90befffa23631e28,0xa4506cebde82bde9 + .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b + .quad 0xca273eceea26619c,0xd186b8c721c0c207 + .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 + .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6 + .quad 0x113f9804bef90dae,0x1b710b35131c471b + .quad 0x28db77f523047d84,0x32caab7b40c72493 + .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c + .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a + .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 -- cgit v0.10.2 From e01d69cb01956e97b6880c1952e264b19473e7f3 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:59:46 -0700 Subject: crypto: sha512 - Optimized SHA512 x86_64 assembly routine using AVX instructions. Provides SHA512 x86_64 assembly routine optimized with SSE and AVX instructions. Speedup of 60% or more has been measured over the generic implementation. Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/sha512-avx-asm.S b/arch/x86/crypto/sha512-avx-asm.S new file mode 100644 index 0000000..974dde9 --- /dev/null +++ b/arch/x86/crypto/sha512-avx-asm.S @@ -0,0 +1,423 @@ +######################################################################## +# Implement fast SHA-512 with AVX instructions. (x86_64) +# +# Copyright (C) 2013 Intel Corporation. +# +# Authors: +# James Guilford +# Kirk Yap +# David Cote +# Tim Chen +# +# This software is available to you under a choice of one of two +# licenses. You may choose to be licensed under the terms of the GNU +# General Public License (GPL) Version 2, available from the file +# COPYING in the main directory of this source tree, or the +# OpenIB.org BSD license below: +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +######################################################################## +# +# This code is described in an Intel White-Paper: +# "Fast SHA-512 Implementations on Intel Architecture Processors" +# +# To find it, surf to http://www.intel.com/p/en_US/embedded +# and search for that title. +# +######################################################################## + +#ifdef CONFIG_AS_AVX +#include + +.text + +# Virtual Registers +# ARG1 +msg = %rdi +# ARG2 +digest = %rsi +# ARG3 +msglen = %rdx +T1 = %rcx +T2 = %r8 +a_64 = %r9 +b_64 = %r10 +c_64 = %r11 +d_64 = %r12 +e_64 = %r13 +f_64 = %r14 +g_64 = %r15 +h_64 = %rbx +tmp0 = %rax + +# Local variables (stack frame) + +# Message Schedule +W_SIZE = 80*8 +# W[t] + K[t] | W[t+1] + K[t+1] +WK_SIZE = 2*8 +RSPSAVE_SIZE = 1*8 +GPRSAVE_SIZE = 5*8 + +frame_W = 0 +frame_WK = frame_W + W_SIZE +frame_RSPSAVE = frame_WK + WK_SIZE +frame_GPRSAVE = frame_RSPSAVE + RSPSAVE_SIZE +frame_size = frame_GPRSAVE + GPRSAVE_SIZE + +# Useful QWORD "arrays" for simpler memory references +# MSG, DIGEST, K_t, W_t are arrays +# WK_2(t) points to 1 of 2 qwords at frame.WK depdending on t being odd/even + +# Input message (arg1) +#define MSG(i) 8*i(msg) + +# Output Digest (arg2) +#define DIGEST(i) 8*i(digest) + +# SHA Constants (static mem) +#define K_t(i) 8*i+K512(%rip) + +# Message Schedule (stack frame) +#define W_t(i) 8*i+frame_W(%rsp) + +# W[t]+K[t] (stack frame) +#define WK_2(i) 8*((i%2))+frame_WK(%rsp) + +.macro RotateState + # Rotate symbols a..h right + TMP = h_64 + h_64 = g_64 + g_64 = f_64 + f_64 = e_64 + e_64 = d_64 + d_64 = c_64 + c_64 = b_64 + b_64 = a_64 + a_64 = TMP +.endm + +.macro RORQ p1 p2 + # shld is faster than ror on Sandybridge + shld $(64-\p2), \p1, \p1 +.endm + +.macro SHA512_Round rnd + # Compute Round %%t + mov f_64, T1 # T1 = f + mov e_64, tmp0 # tmp = e + xor g_64, T1 # T1 = f ^ g + RORQ tmp0, 23 # 41 # tmp = e ror 23 + and e_64, T1 # T1 = (f ^ g) & e + xor e_64, tmp0 # tmp = (e ror 23) ^ e + xor g_64, T1 # T1 = ((f ^ g) & e) ^ g = CH(e,f,g) + idx = \rnd + add WK_2(idx), T1 # W[t] + K[t] from message scheduler + RORQ tmp0, 4 # 18 # tmp = ((e ror 23) ^ e) ror 4 + xor e_64, tmp0 # tmp = (((e ror 23) ^ e) ror 4) ^ e + mov a_64, T2 # T2 = a + add h_64, T1 # T1 = CH(e,f,g) + W[t] + K[t] + h + RORQ tmp0, 14 # 14 # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e) + add tmp0, T1 # T1 = CH(e,f,g) + W[t] + K[t] + S1(e) + mov a_64, tmp0 # tmp = a + xor c_64, T2 # T2 = a ^ c + and c_64, tmp0 # tmp = a & c + and b_64, T2 # T2 = (a ^ c) & b + xor tmp0, T2 # T2 = ((a ^ c) & b) ^ (a & c) = Maj(a,b,c) + mov a_64, tmp0 # tmp = a + RORQ tmp0, 5 # 39 # tmp = a ror 5 + xor a_64, tmp0 # tmp = (a ror 5) ^ a + add T1, d_64 # e(next_state) = d + T1 + RORQ tmp0, 6 # 34 # tmp = ((a ror 5) ^ a) ror 6 + xor a_64, tmp0 # tmp = (((a ror 5) ^ a) ror 6) ^ a + lea (T1, T2), h_64 # a(next_state) = T1 + Maj(a,b,c) + RORQ tmp0, 28 # 28 # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a) + add tmp0, h_64 # a(next_state) = T1 + Maj(a,b,c) S0(a) + RotateState +.endm + +.macro SHA512_2Sched_2Round_avx rnd + # Compute rounds t-2 and t-1 + # Compute message schedule QWORDS t and t+1 + + # Two rounds are computed based on the values for K[t-2]+W[t-2] and + # K[t-1]+W[t-1] which were previously stored at WK_2 by the message + # scheduler. + # The two new schedule QWORDS are stored at [W_t(t)] and [W_t(t+1)]. + # They are then added to their respective SHA512 constants at + # [K_t(t)] and [K_t(t+1)] and stored at dqword [WK_2(t)] + # For brievity, the comments following vectored instructions only refer to + # the first of a pair of QWORDS. + # Eg. XMM4=W[t-2] really means XMM4={W[t-2]|W[t-1]} + # The computation of the message schedule and the rounds are tightly + # stitched to take advantage of instruction-level parallelism. + + idx = \rnd - 2 + vmovdqa W_t(idx), %xmm4 # XMM4 = W[t-2] + idx = \rnd - 15 + vmovdqu W_t(idx), %xmm5 # XMM5 = W[t-15] + mov f_64, T1 + vpsrlq $61, %xmm4, %xmm0 # XMM0 = W[t-2]>>61 + mov e_64, tmp0 + vpsrlq $1, %xmm5, %xmm6 # XMM6 = W[t-15]>>1 + xor g_64, T1 + RORQ tmp0, 23 # 41 + vpsrlq $19, %xmm4, %xmm1 # XMM1 = W[t-2]>>19 + and e_64, T1 + xor e_64, tmp0 + vpxor %xmm1, %xmm0, %xmm0 # XMM0 = W[t-2]>>61 ^ W[t-2]>>19 + xor g_64, T1 + idx = \rnd + add WK_2(idx), T1# + vpsrlq $8, %xmm5, %xmm7 # XMM7 = W[t-15]>>8 + RORQ tmp0, 4 # 18 + vpsrlq $6, %xmm4, %xmm2 # XMM2 = W[t-2]>>6 + xor e_64, tmp0 + mov a_64, T2 + add h_64, T1 + vpxor %xmm7, %xmm6, %xmm6 # XMM6 = W[t-15]>>1 ^ W[t-15]>>8 + RORQ tmp0, 14 # 14 + add tmp0, T1 + vpsrlq $7, %xmm5, %xmm8 # XMM8 = W[t-15]>>7 + mov a_64, tmp0 + xor c_64, T2 + vpsllq $(64-61), %xmm4, %xmm3 # XMM3 = W[t-2]<<3 + and c_64, tmp0 + and b_64, T2 + vpxor %xmm3, %xmm2, %xmm2 # XMM2 = W[t-2]>>6 ^ W[t-2]<<3 + xor tmp0, T2 + mov a_64, tmp0 + vpsllq $(64-1), %xmm5, %xmm9 # XMM9 = W[t-15]<<63 + RORQ tmp0, 5 # 39 + vpxor %xmm9, %xmm8, %xmm8 # XMM8 = W[t-15]>>7 ^ W[t-15]<<63 + xor a_64, tmp0 + add T1, d_64 + RORQ tmp0, 6 # 34 + xor a_64, tmp0 + vpxor %xmm8, %xmm6, %xmm6 # XMM6 = W[t-15]>>1 ^ W[t-15]>>8 ^ + # W[t-15]>>7 ^ W[t-15]<<63 + lea (T1, T2), h_64 + RORQ tmp0, 28 # 28 + vpsllq $(64-19), %xmm4, %xmm4 # XMM4 = W[t-2]<<25 + add tmp0, h_64 + RotateState + vpxor %xmm4, %xmm0, %xmm0 # XMM0 = W[t-2]>>61 ^ W[t-2]>>19 ^ + # W[t-2]<<25 + mov f_64, T1 + vpxor %xmm2, %xmm0, %xmm0 # XMM0 = s1(W[t-2]) + mov e_64, tmp0 + xor g_64, T1 + idx = \rnd - 16 + vpaddq W_t(idx), %xmm0, %xmm0 # XMM0 = s1(W[t-2]) + W[t-16] + idx = \rnd - 7 + vmovdqu W_t(idx), %xmm1 # XMM1 = W[t-7] + RORQ tmp0, 23 # 41 + and e_64, T1 + xor e_64, tmp0 + xor g_64, T1 + vpsllq $(64-8), %xmm5, %xmm5 # XMM5 = W[t-15]<<56 + idx = \rnd + 1 + add WK_2(idx), T1 + vpxor %xmm5, %xmm6, %xmm6 # XMM6 = s0(W[t-15]) + RORQ tmp0, 4 # 18 + vpaddq %xmm6, %xmm0, %xmm0 # XMM0 = s1(W[t-2]) + W[t-16] + s0(W[t-15]) + xor e_64, tmp0 + vpaddq %xmm1, %xmm0, %xmm0 # XMM0 = W[t] = s1(W[t-2]) + W[t-7] + + # s0(W[t-15]) + W[t-16] + mov a_64, T2 + add h_64, T1 + RORQ tmp0, 14 # 14 + add tmp0, T1 + idx = \rnd + vmovdqa %xmm0, W_t(idx) # Store W[t] + vpaddq K_t(idx), %xmm0, %xmm0 # Compute W[t]+K[t] + vmovdqa %xmm0, WK_2(idx) # Store W[t]+K[t] for next rounds + mov a_64, tmp0 + xor c_64, T2 + and c_64, tmp0 + and b_64, T2 + xor tmp0, T2 + mov a_64, tmp0 + RORQ tmp0, 5 # 39 + xor a_64, tmp0 + add T1, d_64 + RORQ tmp0, 6 # 34 + xor a_64, tmp0 + lea (T1, T2), h_64 + RORQ tmp0, 28 # 28 + add tmp0, h_64 + RotateState +.endm + +######################################################################## +# void sha512_transform_avx(const void* M, void* D, u64 L) +# Purpose: Updates the SHA512 digest stored at D with the message stored in M. +# The size of the message pointed to by M must be an integer multiple of SHA512 +# message blocks. +# L is the message length in SHA512 blocks +######################################################################## +ENTRY(sha512_transform_avx) + cmp $0, msglen + je nowork + + # Allocate Stack Space + mov %rsp, %rax + sub $frame_size, %rsp + and $~(0x20 - 1), %rsp + mov %rax, frame_RSPSAVE(%rsp) + + # Save GPRs + mov %rbx, frame_GPRSAVE(%rsp) + mov %r12, frame_GPRSAVE +8*1(%rsp) + mov %r13, frame_GPRSAVE +8*2(%rsp) + mov %r14, frame_GPRSAVE +8*3(%rsp) + mov %r15, frame_GPRSAVE +8*4(%rsp) + +updateblock: + + # Load state variables + mov DIGEST(0), a_64 + mov DIGEST(1), b_64 + mov DIGEST(2), c_64 + mov DIGEST(3), d_64 + mov DIGEST(4), e_64 + mov DIGEST(5), f_64 + mov DIGEST(6), g_64 + mov DIGEST(7), h_64 + + t = 0 + .rept 80/2 + 1 + # (80 rounds) / (2 rounds/iteration) + (1 iteration) + # +1 iteration because the scheduler leads hashing by 1 iteration + .if t < 2 + # BSWAP 2 QWORDS + vmovdqa XMM_QWORD_BSWAP(%rip), %xmm1 + vmovdqu MSG(t), %xmm0 + vpshufb %xmm1, %xmm0, %xmm0 # BSWAP + vmovdqa %xmm0, W_t(t) # Store Scheduled Pair + vpaddq K_t(t), %xmm0, %xmm0 # Compute W[t]+K[t] + vmovdqa %xmm0, WK_2(t) # Store into WK for rounds + .elseif t < 16 + # BSWAP 2 QWORDS# Compute 2 Rounds + vmovdqu MSG(t), %xmm0 + vpshufb %xmm1, %xmm0, %xmm0 # BSWAP + SHA512_Round t-2 # Round t-2 + vmovdqa %xmm0, W_t(t) # Store Scheduled Pair + vpaddq K_t(t), %xmm0, %xmm0 # Compute W[t]+K[t] + SHA512_Round t-1 # Round t-1 + vmovdqa %xmm0, WK_2(t)# Store W[t]+K[t] into WK + .elseif t < 79 + # Schedule 2 QWORDS# Compute 2 Rounds + SHA512_2Sched_2Round_avx t + .else + # Compute 2 Rounds + SHA512_Round t-2 + SHA512_Round t-1 + .endif + t = t+2 + .endr + + # Update digest + add a_64, DIGEST(0) + add b_64, DIGEST(1) + add c_64, DIGEST(2) + add d_64, DIGEST(3) + add e_64, DIGEST(4) + add f_64, DIGEST(5) + add g_64, DIGEST(6) + add h_64, DIGEST(7) + + # Advance to next message block + add $16*8, msg + dec msglen + jnz updateblock + + # Restore GPRs + mov frame_GPRSAVE(%rsp), %rbx + mov frame_GPRSAVE +8*1(%rsp), %r12 + mov frame_GPRSAVE +8*2(%rsp), %r13 + mov frame_GPRSAVE +8*3(%rsp), %r14 + mov frame_GPRSAVE +8*4(%rsp), %r15 + + # Restore Stack Pointer + mov frame_RSPSAVE(%rsp), %rsp + +nowork: + ret +ENDPROC(sha512_transform_avx) + +######################################################################## +### Binary Data + +.data + +.align 16 + +# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. +XMM_QWORD_BSWAP: + .octa 0x08090a0b0c0d0e0f0001020304050607 + +# K[t] used in SHA512 hashing +K512: + .quad 0x428a2f98d728ae22,0x7137449123ef65cd + .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc + .quad 0x3956c25bf348b538,0x59f111f1b605d019 + .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 + .quad 0xd807aa98a3030242,0x12835b0145706fbe + .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 + .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 + .quad 0x9bdc06a725c71235,0xc19bf174cf692694 + .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 + .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65 + .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 + .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 + .quad 0x983e5152ee66dfab,0xa831c66d2db43210 + .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 + .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 + .quad 0x06ca6351e003826f,0x142929670a0e6e70 + .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 + .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df + .quad 0x650a73548baf63de,0x766a0abb3c77b2a8 + .quad 0x81c2c92e47edaee6,0x92722c851482353b + .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 + .quad 0xc24b8b70d0f89791,0xc76c51a30654be30 + .quad 0xd192e819d6ef5218,0xd69906245565a910 + .quad 0xf40e35855771202a,0x106aa07032bbd1b8 + .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 + .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 + .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb + .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 + .quad 0x748f82ee5defb2fc,0x78a5636f43172f60 + .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec + .quad 0x90befffa23631e28,0xa4506cebde82bde9 + .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b + .quad 0xca273eceea26619c,0xd186b8c721c0c207 + .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 + .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6 + .quad 0x113f9804bef90dae,0x1b710b35131c471b + .quad 0x28db77f523047d84,0x32caab7b40c72493 + .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c + .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a + .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 +#endif -- cgit v0.10.2 From 5663535b69eef3940dcdb3110f95651304fe41af Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:59:55 -0700 Subject: crypto: sha512 - Optimized SHA512 x86_64 assembly routine using AVX2 RORX instruction. Provides SHA512 x86_64 assembly routine optimized with SSE, AVX and AVX2's RORX instructions. Speedup of 70% or more has been measured over the generic implementation. Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/sha512-avx2-asm.S b/arch/x86/crypto/sha512-avx2-asm.S new file mode 100644 index 0000000..568b961 --- /dev/null +++ b/arch/x86/crypto/sha512-avx2-asm.S @@ -0,0 +1,743 @@ +######################################################################## +# Implement fast SHA-512 with AVX2 instructions. (x86_64) +# +# Copyright (C) 2013 Intel Corporation. +# +# Authors: +# James Guilford +# Kirk Yap +# David Cote +# Tim Chen +# +# This software is available to you under a choice of one of two +# licenses. You may choose to be licensed under the terms of the GNU +# General Public License (GPL) Version 2, available from the file +# COPYING in the main directory of this source tree, or the +# OpenIB.org BSD license below: +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +######################################################################## +# +# This code is described in an Intel White-Paper: +# "Fast SHA-512 Implementations on Intel Architecture Processors" +# +# To find it, surf to http://www.intel.com/p/en_US/embedded +# and search for that title. +# +######################################################################## +# This code schedules 1 blocks at a time, with 4 lanes per block +######################################################################## + +#ifdef CONFIG_AS_AVX2 +#include + +.text + +# Virtual Registers +Y_0 = %ymm4 +Y_1 = %ymm5 +Y_2 = %ymm6 +Y_3 = %ymm7 + +YTMP0 = %ymm0 +YTMP1 = %ymm1 +YTMP2 = %ymm2 +YTMP3 = %ymm3 +YTMP4 = %ymm8 +XFER = YTMP0 + +BYTE_FLIP_MASK = %ymm9 + +# 1st arg +INP = %rdi +# 2nd arg +CTX = %rsi +# 3rd arg +NUM_BLKS = %rdx + +c = %rcx +d = %r8 +e = %rdx +y3 = %rdi + +TBL = %rbp + +a = %rax +b = %rbx + +f = %r9 +g = %r10 +h = %r11 +old_h = %r11 + +T1 = %r12 +y0 = %r13 +y1 = %r14 +y2 = %r15 + +y4 = %r12 + +# Local variables (stack frame) +XFER_SIZE = 4*8 +SRND_SIZE = 1*8 +INP_SIZE = 1*8 +INPEND_SIZE = 1*8 +RSPSAVE_SIZE = 1*8 +GPRSAVE_SIZE = 6*8 + +frame_XFER = 0 +frame_SRND = frame_XFER + XFER_SIZE +frame_INP = frame_SRND + SRND_SIZE +frame_INPEND = frame_INP + INP_SIZE +frame_RSPSAVE = frame_INPEND + INPEND_SIZE +frame_GPRSAVE = frame_RSPSAVE + RSPSAVE_SIZE +frame_size = frame_GPRSAVE + GPRSAVE_SIZE + +## assume buffers not aligned +#define VMOVDQ vmovdqu + +# addm [mem], reg +# Add reg to mem using reg-mem add and store +.macro addm p1 p2 + add \p1, \p2 + mov \p2, \p1 +.endm + + +# COPY_YMM_AND_BSWAP ymm, [mem], byte_flip_mask +# Load ymm with mem and byte swap each dword +.macro COPY_YMM_AND_BSWAP p1 p2 p3 + VMOVDQ \p2, \p1 + vpshufb \p3, \p1, \p1 +.endm +# rotate_Ys +# Rotate values of symbols Y0...Y3 +.macro rotate_Ys + Y_ = Y_0 + Y_0 = Y_1 + Y_1 = Y_2 + Y_2 = Y_3 + Y_3 = Y_ +.endm + +# RotateState +.macro RotateState + # Rotate symbols a..h right + old_h = h + TMP_ = h + h = g + g = f + f = e + e = d + d = c + c = b + b = a + a = TMP_ +.endm + +# macro MY_VPALIGNR YDST, YSRC1, YSRC2, RVAL +# YDST = {YSRC1, YSRC2} >> RVAL*8 +.macro MY_VPALIGNR YDST YSRC1 YSRC2 RVAL + vperm2f128 $0x3, \YSRC2, \YSRC1, \YDST # YDST = {YS1_LO, YS2_HI} + vpalignr $\RVAL, \YSRC2, \YDST, \YDST # YDST = {YDS1, YS2} >> RVAL*8 +.endm + +.macro FOUR_ROUNDS_AND_SCHED +################################### RND N + 0 ######################################### + + # Extract w[t-7] + MY_VPALIGNR YTMP0, Y_3, Y_2, 8 # YTMP0 = W[-7] + # Calculate w[t-16] + w[t-7] + vpaddq Y_0, YTMP0, YTMP0 # YTMP0 = W[-7] + W[-16] + # Extract w[t-15] + MY_VPALIGNR YTMP1, Y_1, Y_0, 8 # YTMP1 = W[-15] + + # Calculate sigma0 + + # Calculate w[t-15] ror 1 + vpsrlq $1, YTMP1, YTMP2 + vpsllq $(64-1), YTMP1, YTMP3 + vpor YTMP2, YTMP3, YTMP3 # YTMP3 = W[-15] ror 1 + # Calculate w[t-15] shr 7 + vpsrlq $7, YTMP1, YTMP4 # YTMP4 = W[-15] >> 7 + + mov a, y3 # y3 = a # MAJA + rorx $41, e, y0 # y0 = e >> 41 # S1A + rorx $18, e, y1 # y1 = e >> 18 # S1B + add frame_XFER(%rsp),h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + mov f, y2 # y2 = f # CH + rorx $34, a, T1 # T1 = a >> 34 # S0B + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) # S1 + xor g, y2 # y2 = f^g # CH + rorx $14, e, y1 # y1 = (e >> 14) # S1 + + and e, y2 # y2 = (f^g)&e # CH + xor y1, y0 # y0 = (e>>41) ^ (e>>18) ^ (e>>14) # S1 + rorx $39, a, y1 # y1 = a >> 39 # S0A + add h, d # d = k + w + h + d # -- + + and b, y3 # y3 = (a|c)&b # MAJA + xor T1, y1 # y1 = (a>>39) ^ (a>>34) # S0 + rorx $28, a, T1 # T1 = (a >> 28) # S0 + + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + xor T1, y1 # y1 = (a>>39) ^ (a>>34) ^ (a>>28) # S0 + mov a, T1 # T1 = a # MAJB + and c, T1 # T1 = a&c # MAJB + + add y0, y2 # y2 = S1 + CH # -- + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + add y3, h # h = t1 + S0 + MAJ # -- + + RotateState + +################################### RND N + 1 ######################################### + + # Calculate w[t-15] ror 8 + vpsrlq $8, YTMP1, YTMP2 + vpsllq $(64-8), YTMP1, YTMP1 + vpor YTMP2, YTMP1, YTMP1 # YTMP1 = W[-15] ror 8 + # XOR the three components + vpxor YTMP4, YTMP3, YTMP3 # YTMP3 = W[-15] ror 1 ^ W[-15] >> 7 + vpxor YTMP1, YTMP3, YTMP1 # YTMP1 = s0 + + + # Add three components, w[t-16], w[t-7] and sigma0 + vpaddq YTMP1, YTMP0, YTMP0 # YTMP0 = W[-16] + W[-7] + s0 + # Move to appropriate lanes for calculating w[16] and w[17] + vperm2f128 $0x0, YTMP0, YTMP0, Y_0 # Y_0 = W[-16] + W[-7] + s0 {BABA} + # Move to appropriate lanes for calculating w[18] and w[19] + vpand MASK_YMM_LO(%rip), YTMP0, YTMP0 # YTMP0 = W[-16] + W[-7] + s0 {DC00} + + # Calculate w[16] and w[17] in both 128 bit lanes + + # Calculate sigma1 for w[16] and w[17] on both 128 bit lanes + vperm2f128 $0x11, Y_3, Y_3, YTMP2 # YTMP2 = W[-2] {BABA} + vpsrlq $6, YTMP2, YTMP4 # YTMP4 = W[-2] >> 6 {BABA} + + + mov a, y3 # y3 = a # MAJA + rorx $41, e, y0 # y0 = e >> 41 # S1A + rorx $18, e, y1 # y1 = e >> 18 # S1B + add 1*8+frame_XFER(%rsp), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + + mov f, y2 # y2 = f # CH + rorx $34, a, T1 # T1 = a >> 34 # S0B + xor y1, y0 # y0 = (e>>41) ^ (e>>18) # S1 + xor g, y2 # y2 = f^g # CH + + + rorx $14, e, y1 # y1 = (e >> 14) # S1 + xor y1, y0 # y0 = (e>>41) ^ (e>>18) ^ (e>>14) # S1 + rorx $39, a, y1 # y1 = a >> 39 # S0A + and e, y2 # y2 = (f^g)&e # CH + add h, d # d = k + w + h + d # -- + + and b, y3 # y3 = (a|c)&b # MAJA + xor T1, y1 # y1 = (a>>39) ^ (a>>34) # S0 + + rorx $28, a, T1 # T1 = (a >> 28) # S0 + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) ^ (a>>28) # S0 + mov a, T1 # T1 = a # MAJB + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + add y3, h # h = t1 + S0 + MAJ # -- + + RotateState + + +################################### RND N + 2 ######################################### + + vpsrlq $19, YTMP2, YTMP3 # YTMP3 = W[-2] >> 19 {BABA} + vpsllq $(64-19), YTMP2, YTMP1 # YTMP1 = W[-2] << 19 {BABA} + vpor YTMP1, YTMP3, YTMP3 # YTMP3 = W[-2] ror 19 {BABA} + vpxor YTMP3, YTMP4, YTMP4 # YTMP4 = W[-2] ror 19 ^ W[-2] >> 6 {BABA} + vpsrlq $61, YTMP2, YTMP3 # YTMP3 = W[-2] >> 61 {BABA} + vpsllq $(64-61), YTMP2, YTMP1 # YTMP1 = W[-2] << 61 {BABA} + vpor YTMP1, YTMP3, YTMP3 # YTMP3 = W[-2] ror 61 {BABA} + vpxor YTMP3, YTMP4, YTMP4 # YTMP4 = s1 = (W[-2] ror 19) ^ + # (W[-2] ror 61) ^ (W[-2] >> 6) {BABA} + + # Add sigma1 to the other compunents to get w[16] and w[17] + vpaddq YTMP4, Y_0, Y_0 # Y_0 = {W[1], W[0], W[1], W[0]} + + # Calculate sigma1 for w[18] and w[19] for upper 128 bit lane + vpsrlq $6, Y_0, YTMP4 # YTMP4 = W[-2] >> 6 {DC--} + + mov a, y3 # y3 = a # MAJA + rorx $41, e, y0 # y0 = e >> 41 # S1A + add 2*8+frame_XFER(%rsp), h # h = k + w + h # -- + + rorx $18, e, y1 # y1 = e >> 18 # S1B + or c, y3 # y3 = a|c # MAJA + mov f, y2 # y2 = f # CH + xor g, y2 # y2 = f^g # CH + + rorx $34, a, T1 # T1 = a >> 34 # S0B + xor y1, y0 # y0 = (e>>41) ^ (e>>18) # S1 + and e, y2 # y2 = (f^g)&e # CH + + rorx $14, e, y1 # y1 = (e >> 14) # S1 + add h, d # d = k + w + h + d # -- + and b, y3 # y3 = (a|c)&b # MAJA + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) ^ (e>>14) # S1 + rorx $39, a, y1 # y1 = a >> 39 # S0A + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) # S0 + rorx $28, a, T1 # T1 = (a >> 28) # S0 + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) ^ (a>>28) # S0 + mov a, T1 # T1 = a # MAJB + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + + add y3, h # h = t1 + S0 + MAJ # -- + + RotateState + +################################### RND N + 3 ######################################### + + vpsrlq $19, Y_0, YTMP3 # YTMP3 = W[-2] >> 19 {DC--} + vpsllq $(64-19), Y_0, YTMP1 # YTMP1 = W[-2] << 19 {DC--} + vpor YTMP1, YTMP3, YTMP3 # YTMP3 = W[-2] ror 19 {DC--} + vpxor YTMP3, YTMP4, YTMP4 # YTMP4 = W[-2] ror 19 ^ W[-2] >> 6 {DC--} + vpsrlq $61, Y_0, YTMP3 # YTMP3 = W[-2] >> 61 {DC--} + vpsllq $(64-61), Y_0, YTMP1 # YTMP1 = W[-2] << 61 {DC--} + vpor YTMP1, YTMP3, YTMP3 # YTMP3 = W[-2] ror 61 {DC--} + vpxor YTMP3, YTMP4, YTMP4 # YTMP4 = s1 = (W[-2] ror 19) ^ + # (W[-2] ror 61) ^ (W[-2] >> 6) {DC--} + + # Add the sigma0 + w[t-7] + w[t-16] for w[18] and w[19] + # to newly calculated sigma1 to get w[18] and w[19] + vpaddq YTMP4, YTMP0, YTMP2 # YTMP2 = {W[3], W[2], --, --} + + # Form w[19, w[18], w17], w[16] + vpblendd $0xF0, YTMP2, Y_0, Y_0 # Y_0 = {W[3], W[2], W[1], W[0]} + + mov a, y3 # y3 = a # MAJA + rorx $41, e, y0 # y0 = e >> 41 # S1A + rorx $18, e, y1 # y1 = e >> 18 # S1B + add 3*8+frame_XFER(%rsp), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + + mov f, y2 # y2 = f # CH + rorx $34, a, T1 # T1 = a >> 34 # S0B + xor y1, y0 # y0 = (e>>41) ^ (e>>18) # S1 + xor g, y2 # y2 = f^g # CH + + + rorx $14, e, y1 # y1 = (e >> 14) # S1 + and e, y2 # y2 = (f^g)&e # CH + add h, d # d = k + w + h + d # -- + and b, y3 # y3 = (a|c)&b # MAJA + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) ^ (e>>14) # S1 + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + + rorx $39, a, y1 # y1 = a >> 39 # S0A + add y0, y2 # y2 = S1 + CH # -- + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) # S0 + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + rorx $28, a, T1 # T1 = (a >> 28) # S0 + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) ^ (a>>28) # S0 + mov a, T1 # T1 = a # MAJB + and c, T1 # T1 = a&c # MAJB + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + + add y1, h # h = k + w + h + S0 # -- + add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + add y3, h # h = t1 + S0 + MAJ # -- + + RotateState + + rotate_Ys +.endm + +.macro DO_4ROUNDS + +################################### RND N + 0 ######################################### + + mov f, y2 # y2 = f # CH + rorx $41, e, y0 # y0 = e >> 41 # S1A + rorx $18, e, y1 # y1 = e >> 18 # S1B + xor g, y2 # y2 = f^g # CH + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) # S1 + rorx $14, e, y1 # y1 = (e >> 14) # S1 + and e, y2 # y2 = (f^g)&e # CH + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) ^ (e>>14) # S1 + rorx $34, a, T1 # T1 = a >> 34 # S0B + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + rorx $39, a, y1 # y1 = a >> 39 # S0A + mov a, y3 # y3 = a # MAJA + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) # S0 + rorx $28, a, T1 # T1 = (a >> 28) # S0 + add frame_XFER(%rsp), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) ^ (a>>28) # S0 + mov a, T1 # T1 = a # MAJB + and b, y3 # y3 = (a|c)&b # MAJA + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + add h, d # d = k + w + h + d # -- + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + RotateState + +################################### RND N + 1 ######################################### + + add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + mov f, y2 # y2 = f # CH + rorx $41, e, y0 # y0 = e >> 41 # S1A + rorx $18, e, y1 # y1 = e >> 18 # S1B + xor g, y2 # y2 = f^g # CH + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) # S1 + rorx $14, e, y1 # y1 = (e >> 14) # S1 + and e, y2 # y2 = (f^g)&e # CH + add y3, old_h # h = t1 + S0 + MAJ # -- + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) ^ (e>>14) # S1 + rorx $34, a, T1 # T1 = a >> 34 # S0B + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + rorx $39, a, y1 # y1 = a >> 39 # S0A + mov a, y3 # y3 = a # MAJA + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) # S0 + rorx $28, a, T1 # T1 = (a >> 28) # S0 + add 8*1+frame_XFER(%rsp), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) ^ (a>>28) # S0 + mov a, T1 # T1 = a # MAJB + and b, y3 # y3 = (a|c)&b # MAJA + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + add h, d # d = k + w + h + d # -- + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + RotateState + +################################### RND N + 2 ######################################### + + add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + mov f, y2 # y2 = f # CH + rorx $41, e, y0 # y0 = e >> 41 # S1A + rorx $18, e, y1 # y1 = e >> 18 # S1B + xor g, y2 # y2 = f^g # CH + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) # S1 + rorx $14, e, y1 # y1 = (e >> 14) # S1 + and e, y2 # y2 = (f^g)&e # CH + add y3, old_h # h = t1 + S0 + MAJ # -- + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) ^ (e>>14) # S1 + rorx $34, a, T1 # T1 = a >> 34 # S0B + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + rorx $39, a, y1 # y1 = a >> 39 # S0A + mov a, y3 # y3 = a # MAJA + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) # S0 + rorx $28, a, T1 # T1 = (a >> 28) # S0 + add 8*2+frame_XFER(%rsp), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) ^ (a>>28) # S0 + mov a, T1 # T1 = a # MAJB + and b, y3 # y3 = (a|c)&b # MAJA + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + add h, d # d = k + w + h + d # -- + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + RotateState + +################################### RND N + 3 ######################################### + + add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + mov f, y2 # y2 = f # CH + rorx $41, e, y0 # y0 = e >> 41 # S1A + rorx $18, e, y1 # y1 = e >> 18 # S1B + xor g, y2 # y2 = f^g # CH + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) # S1 + rorx $14, e, y1 # y1 = (e >> 14) # S1 + and e, y2 # y2 = (f^g)&e # CH + add y3, old_h # h = t1 + S0 + MAJ # -- + + xor y1, y0 # y0 = (e>>41) ^ (e>>18) ^ (e>>14) # S1 + rorx $34, a, T1 # T1 = a >> 34 # S0B + xor g, y2 # y2 = CH = ((f^g)&e)^g # CH + rorx $39, a, y1 # y1 = a >> 39 # S0A + mov a, y3 # y3 = a # MAJA + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) # S0 + rorx $28, a, T1 # T1 = (a >> 28) # S0 + add 8*3+frame_XFER(%rsp), h # h = k + w + h # -- + or c, y3 # y3 = a|c # MAJA + + xor T1, y1 # y1 = (a>>39) ^ (a>>34) ^ (a>>28) # S0 + mov a, T1 # T1 = a # MAJB + and b, y3 # y3 = (a|c)&b # MAJA + and c, T1 # T1 = a&c # MAJB + add y0, y2 # y2 = S1 + CH # -- + + + add h, d # d = k + w + h + d # -- + or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ + add y1, h # h = k + w + h + S0 # -- + + add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- + + add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- + + add y3, h # h = t1 + S0 + MAJ # -- + + RotateState + +.endm + +######################################################################## +# void sha512_transform_rorx(const void* M, void* D, uint64_t L)# +# Purpose: Updates the SHA512 digest stored at D with the message stored in M. +# The size of the message pointed to by M must be an integer multiple of SHA512 +# message blocks. +# L is the message length in SHA512 blocks +######################################################################## +ENTRY(sha512_transform_rorx) + # Allocate Stack Space + mov %rsp, %rax + sub $frame_size, %rsp + and $~(0x20 - 1), %rsp + mov %rax, frame_RSPSAVE(%rsp) + + # Save GPRs + mov %rbp, frame_GPRSAVE(%rsp) + mov %rbx, 8*1+frame_GPRSAVE(%rsp) + mov %r12, 8*2+frame_GPRSAVE(%rsp) + mov %r13, 8*3+frame_GPRSAVE(%rsp) + mov %r14, 8*4+frame_GPRSAVE(%rsp) + mov %r15, 8*5+frame_GPRSAVE(%rsp) + + shl $7, NUM_BLKS # convert to bytes + jz done_hash + add INP, NUM_BLKS # pointer to end of data + mov NUM_BLKS, frame_INPEND(%rsp) + + ## load initial digest + mov 8*0(CTX),a + mov 8*1(CTX),b + mov 8*2(CTX),c + mov 8*3(CTX),d + mov 8*4(CTX),e + mov 8*5(CTX),f + mov 8*6(CTX),g + mov 8*7(CTX),h + + vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK + +loop0: + lea K512(%rip), TBL + + ## byte swap first 16 dwords + COPY_YMM_AND_BSWAP Y_0, (INP), BYTE_FLIP_MASK + COPY_YMM_AND_BSWAP Y_1, 1*32(INP), BYTE_FLIP_MASK + COPY_YMM_AND_BSWAP Y_2, 2*32(INP), BYTE_FLIP_MASK + COPY_YMM_AND_BSWAP Y_3, 3*32(INP), BYTE_FLIP_MASK + + mov INP, frame_INP(%rsp) + + ## schedule 64 input dwords, by doing 12 rounds of 4 each + movq $4, frame_SRND(%rsp) + +.align 16 +loop1: + vpaddq (TBL), Y_0, XFER + vmovdqa XFER, frame_XFER(%rsp) + FOUR_ROUNDS_AND_SCHED + + vpaddq 1*32(TBL), Y_0, XFER + vmovdqa XFER, frame_XFER(%rsp) + FOUR_ROUNDS_AND_SCHED + + vpaddq 2*32(TBL), Y_0, XFER + vmovdqa XFER, frame_XFER(%rsp) + FOUR_ROUNDS_AND_SCHED + + vpaddq 3*32(TBL), Y_0, XFER + vmovdqa XFER, frame_XFER(%rsp) + add $(4*32), TBL + FOUR_ROUNDS_AND_SCHED + + subq $1, frame_SRND(%rsp) + jne loop1 + + movq $2, frame_SRND(%rsp) +loop2: + vpaddq (TBL), Y_0, XFER + vmovdqa XFER, frame_XFER(%rsp) + DO_4ROUNDS + vpaddq 1*32(TBL), Y_1, XFER + vmovdqa XFER, frame_XFER(%rsp) + add $(2*32), TBL + DO_4ROUNDS + + vmovdqa Y_2, Y_0 + vmovdqa Y_3, Y_1 + + subq $1, frame_SRND(%rsp) + jne loop2 + + addm 8*0(CTX),a + addm 8*1(CTX),b + addm 8*2(CTX),c + addm 8*3(CTX),d + addm 8*4(CTX),e + addm 8*5(CTX),f + addm 8*6(CTX),g + addm 8*7(CTX),h + + mov frame_INP(%rsp), INP + add $128, INP + cmp frame_INPEND(%rsp), INP + jne loop0 + +done_hash: + +# Restore GPRs + mov frame_GPRSAVE(%rsp) ,%rbp + mov 8*1+frame_GPRSAVE(%rsp) ,%rbx + mov 8*2+frame_GPRSAVE(%rsp) ,%r12 + mov 8*3+frame_GPRSAVE(%rsp) ,%r13 + mov 8*4+frame_GPRSAVE(%rsp) ,%r14 + mov 8*5+frame_GPRSAVE(%rsp) ,%r15 + + # Restore Stack Pointer + mov frame_RSPSAVE(%rsp), %rsp + ret +ENDPROC(sha512_transform_rorx) + +######################################################################## +### Binary Data + +.data + +.align 64 +# K[t] used in SHA512 hashing +K512: + .quad 0x428a2f98d728ae22,0x7137449123ef65cd + .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc + .quad 0x3956c25bf348b538,0x59f111f1b605d019 + .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 + .quad 0xd807aa98a3030242,0x12835b0145706fbe + .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 + .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 + .quad 0x9bdc06a725c71235,0xc19bf174cf692694 + .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 + .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65 + .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 + .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 + .quad 0x983e5152ee66dfab,0xa831c66d2db43210 + .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 + .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 + .quad 0x06ca6351e003826f,0x142929670a0e6e70 + .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 + .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df + .quad 0x650a73548baf63de,0x766a0abb3c77b2a8 + .quad 0x81c2c92e47edaee6,0x92722c851482353b + .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 + .quad 0xc24b8b70d0f89791,0xc76c51a30654be30 + .quad 0xd192e819d6ef5218,0xd69906245565a910 + .quad 0xf40e35855771202a,0x106aa07032bbd1b8 + .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 + .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 + .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb + .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 + .quad 0x748f82ee5defb2fc,0x78a5636f43172f60 + .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec + .quad 0x90befffa23631e28,0xa4506cebde82bde9 + .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b + .quad 0xca273eceea26619c,0xd186b8c721c0c207 + .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 + .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6 + .quad 0x113f9804bef90dae,0x1b710b35131c471b + .quad 0x28db77f523047d84,0x32caab7b40c72493 + .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c + .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a + .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 + +.align 32 + +# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. +PSHUFFLE_BYTE_FLIP_MASK: + .octa 0x08090a0b0c0d0e0f0001020304050607 + .octa 0x18191a1b1c1d1e1f1011121314151617 + +MASK_YMM_LO: + .octa 0x00000000000000000000000000000000 + .octa 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +#endif -- cgit v0.10.2 From 87de4579f92dbe50e92f33b94f8688793c894571 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 14:00:02 -0700 Subject: crypto: sha512 - Create module providing optimized SHA512 routines using SSSE3, AVX or AVX2 instructions. We added glue code and config options to create crypto module that uses SSE/AVX/AVX2 optimized SHA512 x86_64 assembly routines. Signed-off-by: Tim Chen Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 9414b91..03cd731 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o +obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o # These modules require assembler to support AVX. ifeq ($(avx_supported),yes) @@ -68,3 +69,4 @@ crc32c-intel-y := crc32c-intel_glue.o crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o +sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c new file mode 100644 index 0000000..6cbd8df --- /dev/null +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -0,0 +1,282 @@ +/* + * Cryptographic API. + * + * Glue code for the SHA512 Secure Hash Algorithm assembler + * implementation using supplemental SSE3 / AVX / AVX2 instructions. + * + * This file is based on sha512_generic.c + * + * Copyright (C) 2013 Intel Corporation + * Author: Tim Chen + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +asmlinkage void sha512_transform_ssse3(const char *data, u64 *digest, + u64 rounds); +#ifdef CONFIG_AS_AVX +asmlinkage void sha512_transform_avx(const char *data, u64 *digest, + u64 rounds); +#endif +#ifdef CONFIG_AS_AVX2 +asmlinkage void sha512_transform_rorx(const char *data, u64 *digest, + u64 rounds); +#endif + +static asmlinkage void (*sha512_transform_asm)(const char *, u64 *, u64); + + +static int sha512_ssse3_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA512_H0; + sctx->state[1] = SHA512_H1; + sctx->state[2] = SHA512_H2; + sctx->state[3] = SHA512_H3; + sctx->state[4] = SHA512_H4; + sctx->state[5] = SHA512_H5; + sctx->state[6] = SHA512_H6; + sctx->state[7] = SHA512_H7; + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static int __sha512_ssse3_update(struct shash_desc *desc, const u8 *data, + unsigned int len, unsigned int partial) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int done = 0; + + sctx->count[0] += len; + if (sctx->count[0] < len) + sctx->count[1]++; + + if (partial) { + done = SHA512_BLOCK_SIZE - partial; + memcpy(sctx->buf + partial, data, done); + sha512_transform_asm(sctx->buf, sctx->state, 1); + } + + if (len - done >= SHA512_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE; + + sha512_transform_asm(data + done, sctx->state, (u64) rounds); + + done += rounds * SHA512_BLOCK_SIZE; + } + + memcpy(sctx->buf, data + done, len - done); + + return 0; +} + +static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; + int res; + + /* Handle the fast case right here */ + if (partial + len < SHA512_BLOCK_SIZE) { + sctx->count[0] += len; + if (sctx->count[0] < len) + sctx->count[1]++; + memcpy(sctx->buf + partial, data, len); + + return 0; + } + + if (!irq_fpu_usable()) { + res = crypto_sha512_update(desc, data, len); + } else { + kernel_fpu_begin(); + res = __sha512_ssse3_update(desc, data, len, partial); + kernel_fpu_end(); + } + + return res; +} + + +/* Add padding and return the message digest. */ +static int sha512_ssse3_final(struct shash_desc *desc, u8 *out) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be64 *dst = (__be64 *)out; + __be64 bits[2]; + static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, }; + + /* save number of bits */ + bits[1] = cpu_to_be64(sctx->count[0] << 3); + bits[0] = cpu_to_be64(sctx->count[1] << 3) | sctx->count[0] >> 61; + + /* Pad out to 112 mod 128 and append length */ + index = sctx->count[0] & 0x7f; + padlen = (index < 112) ? (112 - index) : ((128+112) - index); + + if (!irq_fpu_usable()) { + crypto_sha512_update(desc, padding, padlen); + crypto_sha512_update(desc, (const u8 *)&bits, sizeof(bits)); + } else { + kernel_fpu_begin(); + /* We need to fill a whole block for __sha512_ssse3_update() */ + if (padlen <= 112) { + sctx->count[0] += padlen; + if (sctx->count[0] < padlen) + sctx->count[1]++; + memcpy(sctx->buf + index, padding, padlen); + } else { + __sha512_ssse3_update(desc, padding, padlen, index); + } + __sha512_ssse3_update(desc, (const u8 *)&bits, + sizeof(bits), 112); + kernel_fpu_end(); + } + + /* Store state in digest */ + for (i = 0; i < 8; i++) + dst[i] = cpu_to_be64(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha512_ssse3_export(struct shash_desc *desc, void *out) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha512_ssse3_import(struct shash_desc *desc, const void *in) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static struct shash_alg alg = { + .digestsize = SHA512_DIGEST_SIZE, + .init = sha512_ssse3_init, + .update = sha512_ssse3_update, + .final = sha512_ssse3_final, + .export = sha512_ssse3_export, + .import = sha512_ssse3_import, + .descsize = sizeof(struct sha512_state), + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-ssse3", + .cra_priority = 150, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +#ifdef CONFIG_AS_AVX +static bool __init avx_usable(void) +{ + u64 xcr0; + + if (!cpu_has_avx || !cpu_has_osxsave) + return false; + + xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); + if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { + pr_info("AVX detected but unusable.\n"); + + return false; + } + + return true; +} +#endif + +static int __init sha512_ssse3_mod_init(void) +{ + /* test for SSE3 first */ + if (cpu_has_ssse3) + sha512_transform_asm = sha512_transform_ssse3; + +#ifdef CONFIG_AS_AVX + /* allow AVX to override SSSE3, it's a little faster */ + if (avx_usable()) { +#ifdef CONFIG_AS_AVX2 + if (boot_cpu_has(X86_FEATURE_AVX2)) + sha512_transform_asm = sha512_transform_rorx; + else +#endif + sha512_transform_asm = sha512_transform_avx; + } +#endif + + if (sha512_transform_asm) { +#ifdef CONFIG_AS_AVX + if (sha512_transform_asm == sha512_transform_avx) + pr_info("Using AVX optimized SHA-512 implementation\n"); +#ifdef CONFIG_AS_AVX2 + else if (sha512_transform_asm == sha512_transform_rorx) + pr_info("Using AVX2 optimized SHA-512 implementation\n"); +#endif + else +#endif + pr_info("Using SSSE3 optimized SHA-512 implementation\n"); + return crypto_register_shash(&alg); + } + pr_info("Neither AVX nor SSSE3 is available/usable.\n"); + + return -ENODEV; +} + +static void __exit sha512_ssse3_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(sha512_ssse3_mod_init); +module_exit(sha512_ssse3_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated"); + +MODULE_ALIAS("sha512"); diff --git a/crypto/Kconfig b/crypto/Kconfig index 8064ef1..a654b13 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -481,6 +481,17 @@ config CRYPTO_SHA256_SSSE3 Extensions version 1 (AVX1), or Advanced Vector Extensions version 2 (AVX2) instructions, when available. +config CRYPTO_SHA512_SSSE3 + tristate "SHA512 digest algorithm (SSSE3/AVX/AVX2)" + depends on X86 && 64BIT + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-512 secure hash standard (DFIPS 180-2) implemented + using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector + Extensions version 1 (AVX1), or Advanced Vector Extensions + version 2 (AVX2) instructions, when available. + config CRYPTO_SHA1_SPARC64 tristate "SHA1 digest algorithm (SPARC64)" depends on SPARC64 -- cgit v0.10.2 From 66b3e8879f42a571e6779a6c4e5ed486aaaf6fef Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Tue, 26 Mar 2013 18:10:14 -0500 Subject: crypto: caam - change key gen functions to return signed int commit 2af8f4a "crypto: caam - coccicheck fixes" added error return values yet neglected to change the type from unsigned. Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 32aba7a..5996521 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -411,7 +411,7 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) return 0; } -static u32 gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in, +static int gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in, u32 keylen) { return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, @@ -420,7 +420,7 @@ static u32 gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in, } /* Digest hash size if it is too large */ -static u32 hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, +static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, u32 *keylen, u8 *key_out, u32 digestsize) { struct device *jrdev = ctx->jrdev; diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c index f6dba10..87138d2 100644 --- a/drivers/crypto/caam/key_gen.c +++ b/drivers/crypto/caam/key_gen.c @@ -44,7 +44,7 @@ Split key generation----------------------------------------------- [06] 0x64260028 fifostr: class2 mdsplit-jdk len=40 @0xffe04000 */ -u32 gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len, +int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len, int split_key_pad_len, const u8 *key_in, u32 keylen, u32 alg_op) { diff --git a/drivers/crypto/caam/key_gen.h b/drivers/crypto/caam/key_gen.h index d95d290..c5588f6 100644 --- a/drivers/crypto/caam/key_gen.h +++ b/drivers/crypto/caam/key_gen.h @@ -12,6 +12,6 @@ struct split_key_result { void split_key_done(struct device *dev, u32 *desc, u32 err, void *context); -u32 gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len, +int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len, int split_key_pad_len, const u8 *key_in, u32 keylen, u32 alg_op); -- cgit v0.10.2 From 96aef9a8ba37da48f2ae9612ccd0f6b7bc0542eb Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Tue, 26 Mar 2013 18:10:15 -0500 Subject: crypto: caam - static constify error data checkstack reports report_deco_status(), report_ccb_status() as particularly excessive stack users. Move their lookup tables off the stack and put them in .rodata. Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c index 30b8f74..9f25f52 100644 --- a/drivers/crypto/caam/error.c +++ b/drivers/crypto/caam/error.c @@ -36,7 +36,7 @@ static void report_jump_idx(u32 status, char *outstr) static void report_ccb_status(u32 status, char *outstr) { - char *cha_id_list[] = { + static const char * const cha_id_list[] = { "", "AES", "DES", @@ -51,7 +51,7 @@ static void report_ccb_status(u32 status, char *outstr) "ZUCE", "ZUCA", }; - char *err_id_list[] = { + static const char * const err_id_list[] = { "No error.", "Mode error.", "Data size error.", @@ -69,7 +69,7 @@ static void report_ccb_status(u32 status, char *outstr) "Invalid CHA combination was selected", "Invalid CHA selected.", }; - char *rng_err_id_list[] = { + static const char * const rng_err_id_list[] = { "", "", "", @@ -117,7 +117,7 @@ static void report_jump_status(u32 status, char *outstr) static void report_deco_status(u32 status, char *outstr) { - const struct { + static const struct { u8 value; char *error_text; } desc_error_list[] = { @@ -245,7 +245,7 @@ static void report_cond_code_status(u32 status, char *outstr) char *caam_jr_strstatus(char *outstr, u32 status) { - struct stat_src { + static const struct stat_src { void (*report_ssed)(u32 status, char *outstr); char *error; } status_src[] = { -- cgit v0.10.2 From 8c4196a2fd7c31acd6d02d9921f7896b8f160c92 Mon Sep 17 00:00:00 2001 From: Lubomir Rintel Date: Thu, 28 Mar 2013 07:19:38 +0100 Subject: hwrng: bcm2835 - Add Broadcom BCM2835 RNG driver This adds a driver for random number generator present on Broadcom BCM2835 SoC, used in Raspberry Pi and Roku 2 devices. Signed-off-by: Dom Cobley Signed-off-by: Lubomir Rintel Tested-by: Stephen Warren Cc: Herbert Xu Cc: Stephen Warren Cc: Matt Mackall Cc: linux-rpi-kernel@lists.infradead.org Signed-off-by: Herbert Xu diff --git a/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt b/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt new file mode 100644 index 0000000..07ccdaa --- /dev/null +++ b/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt @@ -0,0 +1,13 @@ +BCM2835 Random number generator + +Required properties: + +- compatible : should be "brcm,bcm2835-rng" +- reg : Specifies base physical address and size of the registers. + +Example: + +rng { + compatible = "brcm,bcm2835-rng"; + reg = <0x7e104000 0x10>; +}; diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index c5a0262..2f9dbf7 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -86,6 +86,18 @@ config HW_RANDOM_BCM63XX If unusure, say Y. +config HW_RANDOM_BCM2835 + tristate "Broadcom BCM2835 Random Number Generator support" + depends on HW_RANDOM && ARCH_BCM2835 + default HW_RANDOM + ---help--- + This driver provides kernel-side support for the Random Number + Generator hardware found on the Broadcom BCM2835 SoCs. + + To compile this driver as a module, choose M here: the + module will be called bcm2835-rng + + If unsure, say Y. config HW_RANDOM_GEODE tristate "AMD Geode HW Random Number Generator support" diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 1fd7eec..bed467c 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -26,3 +26,4 @@ obj-$(CONFIG_HW_RANDOM_PPC4XX) += ppc4xx-rng.o obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o obj-$(CONFIG_HW_RANDOM_EXYNOS) += exynos-rng.o obj-$(CONFIG_HW_RANDOM_TPM) += tpm-rng.o +obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c new file mode 100644 index 0000000..eb7f147 --- /dev/null +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -0,0 +1,113 @@ +/** + * Copyright (c) 2010-2012 Broadcom. All rights reserved. + * Copyright (c) 2013 Lubomir Rintel + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License ("GPL") + * version 2, as published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RNG_CTRL 0x0 +#define RNG_STATUS 0x4 +#define RNG_DATA 0x8 + +/* enable rng */ +#define RNG_RBGEN 0x1 + +/* the initial numbers generated are "less random" so will be discarded */ +#define RNG_WARMUP_COUNT 0x40000 + +static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, + bool wait) +{ + void __iomem *rng_base = (void __iomem *)rng->priv; + + while ((__raw_readl(rng_base + RNG_STATUS) >> 24) == 0) { + if (!wait) + return 0; + cpu_relax(); + } + + *(u32 *)buf = __raw_readl(rng_base + RNG_DATA); + return sizeof(u32); +} + +static struct hwrng bcm2835_rng_ops = { + .name = "bcm2835", + .read = bcm2835_rng_read, +}; + +static int bcm2835_rng_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + void __iomem *rng_base; + int err; + + /* map peripheral */ + rng_base = of_iomap(np, 0); + if (!rng_base) { + dev_err(dev, "failed to remap rng regs"); + return -ENODEV; + } + bcm2835_rng_ops.priv = (unsigned long)rng_base; + + /* register driver */ + err = hwrng_register(&bcm2835_rng_ops); + if (err) { + dev_err(dev, "hwrng registration failed\n"); + iounmap(rng_base); + } else { + dev_info(dev, "hwrng registered\n"); + + /* set warm-up count & enable */ + __raw_writel(RNG_WARMUP_COUNT, rng_base + RNG_STATUS); + __raw_writel(RNG_RBGEN, rng_base + RNG_CTRL); + } + return err; +} + +static int bcm2835_rng_remove(struct platform_device *pdev) +{ + void __iomem *rng_base = (void __iomem *)bcm2835_rng_ops.priv; + + /* disable rng hardware */ + __raw_writel(0, rng_base + RNG_CTRL); + + /* unregister driver */ + hwrng_unregister(&bcm2835_rng_ops); + iounmap(rng_base); + + return 0; +} + +static const struct of_device_id bcm2835_rng_of_match[] = { + { .compatible = "brcm,bcm2835-rng", }, + {}, +}; +MODULE_DEVICE_TABLE(of, bcm2835_rng_of_match); + +static struct platform_driver bcm2835_rng_driver = { + .driver = { + .name = "bcm2835-rng", + .owner = THIS_MODULE, + .of_match_table = bcm2835_rng_of_match, + }, + .probe = bcm2835_rng_probe, + .remove = bcm2835_rng_remove, +}; +module_platform_driver(bcm2835_rng_driver); + +MODULE_AUTHOR("Lubomir Rintel "); +MODULE_DESCRIPTION("BCM2835 Random Number Generator (RNG) driver"); +MODULE_LICENSE("GPLv2"); -- cgit v0.10.2 From 57ae1b0532977b30184aaba04b6cafe0a284c21f Mon Sep 17 00:00:00 2001 From: Sandy Wu Date: Thu, 28 Mar 2013 17:05:44 -0700 Subject: crypto: crc32-pclmul - Use gas macro for pclmulqdq Occurs when CONFIG_CRYPTO_CRC32C_INTEL=y and CONFIG_CRYPTO_CRC32C_INTEL=y. Older versions of bintuils do not support the pclmulqdq instruction. The PCLMULQDQ gas macro is used instead. Signed-off-by: Sandy Wu Cc: stable@vger.kernel.org # 3.8+ Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S index 19ec73f..dbc4339 100644 --- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S +++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S @@ -43,6 +43,7 @@ * SOFTWARE. */ +#include #include ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction @@ -226,10 +227,10 @@ LABEL crc_ %i movdqa (bufp), %xmm0 # 2 consts: K1:K2 movq crc_init, %xmm1 # CRC for block 1 - pclmulqdq $0x00,%xmm0,%xmm1 # Multiply by K2 + PCLMULQDQ 0x00,%xmm0,%xmm1 # Multiply by K2 movq crc1, %xmm2 # CRC for block 2 - pclmulqdq $0x10, %xmm0, %xmm2 # Multiply by K1 + PCLMULQDQ 0x10, %xmm0, %xmm2 # Multiply by K1 pxor %xmm2,%xmm1 movq %xmm1, %rax -- cgit v0.10.2 From 1907da78bf6a3f74e5d030a4a4f6700ba0d0a234 Mon Sep 17 00:00:00 2001 From: Alexander Clouter Date: Sun, 31 Mar 2013 17:34:50 +0100 Subject: hwrng: timeriomem - update to support more than one device timeriomem_rng only supports a single device instance. This patch enables multiple timeriomem_rng devices to coexist as well as adds some additional error checking. Signed-off-by: Alexander Clouter Signed-off-by: Herbert Xu diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index 849db19..9dd2cad 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -25,117 +25,179 @@ #include #include #include +#include #include #include #include #include #include -static struct timeriomem_rng_data *timeriomem_rng_data; +struct timeriomem_rng_private_data { + void __iomem *io_base; + unsigned int expires; + unsigned int period; + unsigned int present:1; -static void timeriomem_rng_trigger(unsigned long); -static DEFINE_TIMER(timeriomem_rng_timer, timeriomem_rng_trigger, 0, 0); + struct timer_list timer; + struct completion completion; + + struct hwrng timeriomem_rng_ops; +}; + +#define to_rng_priv(rng) \ + ((struct timeriomem_rng_private_data *)rng->priv) /* * have data return 1, however return 0 if we have nothing */ static int timeriomem_rng_data_present(struct hwrng *rng, int wait) { - if (rng->priv == 0) - return 1; + struct timeriomem_rng_private_data *priv = to_rng_priv(rng); - if (!wait || timeriomem_rng_data->present) - return timeriomem_rng_data->present; + if (!wait || priv->present) + return priv->present; - wait_for_completion(&timeriomem_rng_data->completion); + wait_for_completion(&priv->completion); return 1; } static int timeriomem_rng_data_read(struct hwrng *rng, u32 *data) { + struct timeriomem_rng_private_data *priv = to_rng_priv(rng); unsigned long cur; s32 delay; - *data = readl(timeriomem_rng_data->address); + *data = readl(priv->io_base); - if (rng->priv != 0) { - cur = jiffies; + cur = jiffies; - delay = cur - timeriomem_rng_timer.expires; - delay = rng->priv - (delay % rng->priv); + delay = cur - priv->expires; + delay = priv->period - (delay % priv->period); - timeriomem_rng_timer.expires = cur + delay; - timeriomem_rng_data->present = 0; + priv->expires = cur + delay; + priv->present = 0; - init_completion(&timeriomem_rng_data->completion); - add_timer(&timeriomem_rng_timer); - } + INIT_COMPLETION(priv->completion); + mod_timer(&priv->timer, priv->expires); return 4; } -static void timeriomem_rng_trigger(unsigned long dummy) +static void timeriomem_rng_trigger(unsigned long data) { - timeriomem_rng_data->present = 1; - complete(&timeriomem_rng_data->completion); -} + struct timeriomem_rng_private_data *priv + = (struct timeriomem_rng_private_data *)data; -static struct hwrng timeriomem_rng_ops = { - .name = "timeriomem", - .data_present = timeriomem_rng_data_present, - .data_read = timeriomem_rng_data_read, - .priv = 0, -}; + priv->present = 1; + complete(&priv->completion); +} static int timeriomem_rng_probe(struct platform_device *pdev) { + struct timeriomem_rng_data *pdata = pdev->dev.platform_data; + struct timeriomem_rng_private_data *priv; struct resource *res; - int ret; + int err = 0; + int period; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!pdata) { + dev_err(&pdev->dev, "timeriomem_rng_data is missing\n"); + return -EINVAL; + } + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) - return -ENOENT; + return -ENXIO; - timeriomem_rng_data = pdev->dev.platform_data; + if (res->start % 4 != 0 || resource_size(res) != 4) { + dev_err(&pdev->dev, + "address must be four bytes wide and aligned\n"); + return -EINVAL; + } - timeriomem_rng_data->address = ioremap(res->start, resource_size(res)); - if (!timeriomem_rng_data->address) - return -EIO; + /* Allocate memory for the device structure (and zero it) */ + priv = kzalloc(sizeof(struct timeriomem_rng_private_data), GFP_KERNEL); + if (!priv) { + dev_err(&pdev->dev, "failed to allocate device structure.\n"); + return -ENOMEM; + } + + platform_set_drvdata(pdev, priv); - if (timeriomem_rng_data->period != 0 - && usecs_to_jiffies(timeriomem_rng_data->period) > 0) { - timeriomem_rng_timer.expires = jiffies; + period = pdata->period; - timeriomem_rng_ops.priv = usecs_to_jiffies( - timeriomem_rng_data->period); + priv->period = usecs_to_jiffies(period); + if (priv->period < 1) { + dev_err(&pdev->dev, "period is less than one jiffy\n"); + err = -EINVAL; + goto out_free; } - timeriomem_rng_data->present = 1; - ret = hwrng_register(&timeriomem_rng_ops); - if (ret) - goto failed; + priv->expires = jiffies; + priv->present = 1; + + init_completion(&priv->completion); + complete(&priv->completion); + + setup_timer(&priv->timer, timeriomem_rng_trigger, (unsigned long)priv); + + priv->timeriomem_rng_ops.name = dev_name(&pdev->dev); + priv->timeriomem_rng_ops.data_present = timeriomem_rng_data_present; + priv->timeriomem_rng_ops.data_read = timeriomem_rng_data_read; + priv->timeriomem_rng_ops.priv = (unsigned long)priv; + + if (!request_mem_region(res->start, resource_size(res), + dev_name(&pdev->dev))) { + dev_err(&pdev->dev, "request_mem_region failed\n"); + err = -EBUSY; + goto out_timer; + } + + priv->io_base = ioremap(res->start, resource_size(res)); + if (priv->io_base == NULL) { + dev_err(&pdev->dev, "ioremap failed\n"); + err = -EIO; + goto out_release_io; + } + + err = hwrng_register(&priv->timeriomem_rng_ops); + if (err) { + dev_err(&pdev->dev, "problem registering\n"); + goto out; + } dev_info(&pdev->dev, "32bits from 0x%p @ %dus\n", - timeriomem_rng_data->address, - timeriomem_rng_data->period); + priv->io_base, period); return 0; -failed: - dev_err(&pdev->dev, "problem registering\n"); - iounmap(timeriomem_rng_data->address); - - return ret; +out: + iounmap(priv->io_base); +out_release_io: + release_mem_region(res->start, resource_size(res)); +out_timer: + del_timer_sync(&priv->timer); +out_free: + platform_set_drvdata(pdev, NULL); + kfree(priv); + return err; } static int timeriomem_rng_remove(struct platform_device *pdev) { - del_timer_sync(&timeriomem_rng_timer); - hwrng_unregister(&timeriomem_rng_ops); + struct timeriomem_rng_private_data *priv = platform_get_drvdata(pdev); + struct resource *res; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + + hwrng_unregister(&priv->timeriomem_rng_ops); - iounmap(timeriomem_rng_data->address); + del_timer_sync(&priv->timer); + iounmap(priv->io_base); + release_mem_region(res->start, resource_size(res)); + platform_set_drvdata(pdev, NULL); + kfree(priv); return 0; } diff --git a/include/linux/timeriomem-rng.h b/include/linux/timeriomem-rng.h index 3e08a1c..46eb27d 100644 --- a/include/linux/timeriomem-rng.h +++ b/include/linux/timeriomem-rng.h @@ -8,12 +8,7 @@ * published by the Free Software Foundation. */ -#include - struct timeriomem_rng_data { - struct completion completion; - unsigned int present:1; - void __iomem *address; /* measures in usecs */ -- cgit v0.10.2 From b149a30d87d165e1079163fdab6f14e48b2f57b2 Mon Sep 17 00:00:00 2001 From: Alexander Clouter Date: Sun, 31 Mar 2013 17:34:51 +0100 Subject: hwrng: timeriomem - added devicetree hooks This patch allows timeriomem_rng to be used via devicetree. Signed-off-by: Alexander Clouter Signed-off-by: Herbert Xu diff --git a/Documentation/devicetree/bindings/hwrng/timeriomem_rng.txt b/Documentation/devicetree/bindings/hwrng/timeriomem_rng.txt new file mode 100644 index 0000000..6616d15 --- /dev/null +++ b/Documentation/devicetree/bindings/hwrng/timeriomem_rng.txt @@ -0,0 +1,18 @@ +HWRNG support for the timeriomem_rng driver + +Required properties: +- compatible : "timeriomem_rng" +- reg : base address to sample from +- period : wait time in microseconds to use between samples + +N.B. currently 'reg' must be four bytes wide and aligned + +Example: + +hwrng@44 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "timeriomem_rng"; + reg = <0x44 0x04>; + period = <1000000>; +}; diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index 9dd2cad..3e75737 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -101,7 +102,7 @@ static int timeriomem_rng_probe(struct platform_device *pdev) int err = 0; int period; - if (!pdata) { + if (!pdev->dev.of_node && !pdata) { dev_err(&pdev->dev, "timeriomem_rng_data is missing\n"); return -EINVAL; } @@ -125,7 +126,19 @@ static int timeriomem_rng_probe(struct platform_device *pdev) platform_set_drvdata(pdev, priv); - period = pdata->period; + if (pdev->dev.of_node) { + int i; + + if (!of_property_read_u32(pdev->dev.of_node, + "period", &i)) + period = i; + else { + dev_err(&pdev->dev, "missing period\n"); + err = -EINVAL; + goto out_free; + } + } else + period = pdata->period; priv->period = usecs_to_jiffies(period); if (priv->period < 1) { @@ -202,10 +215,17 @@ static int timeriomem_rng_remove(struct platform_device *pdev) return 0; } +static const struct of_device_id timeriomem_rng_match[] = { + { .compatible = "timeriomem_rng" }, + {}, +}; +MODULE_DEVICE_TABLE(of, timeriomem_rng_match); + static struct platform_driver timeriomem_rng_driver = { .driver = { .name = "timeriomem_rng", .owner = THIS_MODULE, + .of_match_table = timeriomem_rng_match, }, .probe = timeriomem_rng_probe, .remove = timeriomem_rng_remove, -- cgit v0.10.2 From 9489667d3e3d39ba452037585e48a89ce44ccbfe Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sun, 7 Apr 2013 16:43:41 +0300 Subject: crypto: gcm - make GMAC work when dst and src are different The GMAC code assumes that dst==src, which causes problems when trying to add rfc4543(gcm(aes)) test vectors. So fix this code to work when source and destination buffer are different. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/crypto/Kconfig b/crypto/Kconfig index a654b13..6cc27f1 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -198,6 +198,7 @@ config CRYPTO_GCM select CRYPTO_CTR select CRYPTO_AEAD select CRYPTO_GHASH + select CRYPTO_NULL help Support for Galois/Counter Mode (GCM) and Galois Message Authentication Code (GMAC). Required for IPSec. diff --git a/crypto/gcm.c b/crypto/gcm.c index 137ad1e..4ff2139 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -37,8 +37,14 @@ struct crypto_rfc4106_ctx { u8 nonce[4]; }; +struct crypto_rfc4543_instance_ctx { + struct crypto_aead_spawn aead; + struct crypto_skcipher_spawn null; +}; + struct crypto_rfc4543_ctx { struct crypto_aead *child; + struct crypto_blkcipher *null; u8 nonce[4]; }; @@ -1094,20 +1100,20 @@ static int crypto_rfc4543_setauthsize(struct crypto_aead *parent, } static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req, - int enc) + bool enc) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req); struct aead_request *subreq = &rctx->subreq; - struct scatterlist *dst = req->dst; + struct scatterlist *src = req->src; struct scatterlist *cipher = rctx->cipher; struct scatterlist *payload = rctx->payload; struct scatterlist *assoc = rctx->assoc; unsigned int authsize = crypto_aead_authsize(aead); unsigned int assoclen = req->assoclen; - struct page *dstp; - u8 *vdst; + struct page *srcp; + u8 *vsrc; u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), crypto_aead_alignmask(ctx->child) + 1); @@ -1118,19 +1124,19 @@ static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req, if (enc) memset(rctx->auth_tag, 0, authsize); else - scatterwalk_map_and_copy(rctx->auth_tag, dst, + scatterwalk_map_and_copy(rctx->auth_tag, src, req->cryptlen - authsize, authsize, 0); sg_init_one(cipher, rctx->auth_tag, authsize); /* construct the aad */ - dstp = sg_page(dst); - vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset; + srcp = sg_page(src); + vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + src->offset; sg_init_table(payload, 2); sg_set_buf(payload, req->iv, 8); - scatterwalk_crypto_chain(payload, dst, vdst == req->iv + 8, 2); + scatterwalk_crypto_chain(payload, src, vsrc == req->iv + 8, 2); assoclen += 8 + req->cryptlen - (enc ? 0 : authsize); sg_init_table(assoc, 2); @@ -1147,6 +1153,19 @@ static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req, return subreq; } +static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); + unsigned int authsize = crypto_aead_authsize(aead); + unsigned int nbytes = req->cryptlen - (enc ? 0 : authsize); + struct blkcipher_desc desc = { + .tfm = ctx->null, + }; + + return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes); +} + static int crypto_rfc4543_encrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); @@ -1154,7 +1173,13 @@ static int crypto_rfc4543_encrypt(struct aead_request *req) struct aead_request *subreq; int err; - subreq = crypto_rfc4543_crypt(req, 1); + if (req->src != req->dst) { + err = crypto_rfc4543_copy_src_to_dst(req, true); + if (err) + return err; + } + + subreq = crypto_rfc4543_crypt(req, true); err = crypto_aead_encrypt(subreq); if (err) return err; @@ -1167,7 +1192,15 @@ static int crypto_rfc4543_encrypt(struct aead_request *req) static int crypto_rfc4543_decrypt(struct aead_request *req) { - req = crypto_rfc4543_crypt(req, 0); + int err; + + if (req->src != req->dst) { + err = crypto_rfc4543_copy_src_to_dst(req, false); + if (err) + return err; + } + + req = crypto_rfc4543_crypt(req, false); return crypto_aead_decrypt(req); } @@ -1175,16 +1208,25 @@ static int crypto_rfc4543_decrypt(struct aead_request *req) static int crypto_rfc4543_init_tfm(struct crypto_tfm *tfm) { struct crypto_instance *inst = (void *)tfm->__crt_alg; - struct crypto_aead_spawn *spawn = crypto_instance_ctx(inst); + struct crypto_rfc4543_instance_ctx *ictx = crypto_instance_ctx(inst); + struct crypto_aead_spawn *spawn = &ictx->aead; struct crypto_rfc4543_ctx *ctx = crypto_tfm_ctx(tfm); struct crypto_aead *aead; + struct crypto_blkcipher *null; unsigned long align; + int err = 0; aead = crypto_spawn_aead(spawn); if (IS_ERR(aead)) return PTR_ERR(aead); + null = crypto_spawn_blkcipher(&ictx->null.base); + err = PTR_ERR(null); + if (IS_ERR(null)) + goto err_free_aead; + ctx->child = aead; + ctx->null = null; align = crypto_aead_alignmask(aead); align &= ~(crypto_tfm_ctx_alignment() - 1); @@ -1194,6 +1236,10 @@ static int crypto_rfc4543_init_tfm(struct crypto_tfm *tfm) align + 16; return 0; + +err_free_aead: + crypto_free_aead(aead); + return err; } static void crypto_rfc4543_exit_tfm(struct crypto_tfm *tfm) @@ -1201,6 +1247,7 @@ static void crypto_rfc4543_exit_tfm(struct crypto_tfm *tfm) struct crypto_rfc4543_ctx *ctx = crypto_tfm_ctx(tfm); crypto_free_aead(ctx->child); + crypto_free_blkcipher(ctx->null); } static struct crypto_instance *crypto_rfc4543_alloc(struct rtattr **tb) @@ -1209,6 +1256,7 @@ static struct crypto_instance *crypto_rfc4543_alloc(struct rtattr **tb) struct crypto_instance *inst; struct crypto_aead_spawn *spawn; struct crypto_alg *alg; + struct crypto_rfc4543_instance_ctx *ctx; const char *ccm_name; int err; @@ -1223,11 +1271,12 @@ static struct crypto_instance *crypto_rfc4543_alloc(struct rtattr **tb) if (IS_ERR(ccm_name)) return ERR_CAST(ccm_name); - inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); + inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); if (!inst) return ERR_PTR(-ENOMEM); - spawn = crypto_instance_ctx(inst); + ctx = crypto_instance_ctx(inst); + spawn = &ctx->aead; crypto_set_aead_spawn(spawn, inst); err = crypto_grab_aead(spawn, ccm_name, 0, crypto_requires_sync(algt->type, algt->mask)); @@ -1236,15 +1285,23 @@ static struct crypto_instance *crypto_rfc4543_alloc(struct rtattr **tb) alg = crypto_aead_spawn_alg(spawn); + crypto_set_skcipher_spawn(&ctx->null, inst); + err = crypto_grab_skcipher(&ctx->null, "ecb(cipher_null)", 0, + CRYPTO_ALG_ASYNC); + if (err) + goto out_drop_alg; + + crypto_skcipher_spawn_alg(&ctx->null); + err = -EINVAL; /* We only support 16-byte blocks. */ if (alg->cra_aead.ivsize != 16) - goto out_drop_alg; + goto out_drop_ecbnull; /* Not a stream cipher? */ if (alg->cra_blocksize != 1) - goto out_drop_alg; + goto out_drop_ecbnull; err = -ENAMETOOLONG; if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, @@ -1252,7 +1309,7 @@ static struct crypto_instance *crypto_rfc4543_alloc(struct rtattr **tb) snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "rfc4543(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) - goto out_drop_alg; + goto out_drop_ecbnull; inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC; @@ -1279,6 +1336,8 @@ static struct crypto_instance *crypto_rfc4543_alloc(struct rtattr **tb) out: return inst; +out_drop_ecbnull: + crypto_drop_skcipher(&ctx->null); out_drop_alg: crypto_drop_aead(spawn); out_free_inst: @@ -1289,7 +1348,11 @@ out_free_inst: static void crypto_rfc4543_free(struct crypto_instance *inst) { - crypto_drop_spawn(crypto_instance_ctx(inst)); + struct crypto_rfc4543_instance_ctx *ctx = crypto_instance_ctx(inst); + + crypto_drop_aead(&ctx->aead); + crypto_drop_skcipher(&ctx->null); + kfree(inst); } -- cgit v0.10.2 From d733ac90f9fe8ac284e523f9920b507555b12f6d Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sun, 7 Apr 2013 16:43:46 +0300 Subject: crypto: gcm - fix rfc4543 to handle async crypto correctly If the gcm cipher used by rfc4543 does not complete request immediately, the authentication tag is not copied to destination buffer. Patch adds correct async logic for this case. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/crypto/gcm.c b/crypto/gcm.c index 4ff2139..b0d3cb1 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -1099,6 +1099,21 @@ static int crypto_rfc4543_setauthsize(struct crypto_aead *parent, return crypto_aead_setauthsize(ctx->child, authsize); } +static void crypto_rfc4543_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req); + + if (!err) { + scatterwalk_map_and_copy(rctx->auth_tag, req->dst, + req->cryptlen, + crypto_aead_authsize(aead), 1); + } + + aead_request_complete(req, err); +} + static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req, bool enc) { @@ -1145,8 +1160,8 @@ static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req, scatterwalk_crypto_chain(assoc, payload, 0, 2); aead_request_set_tfm(subreq, ctx->child); - aead_request_set_callback(subreq, req->base.flags, req->base.complete, - req->base.data); + aead_request_set_callback(subreq, req->base.flags, crypto_rfc4543_done, + req); aead_request_set_crypt(subreq, cipher, cipher, enc ? 0 : authsize, iv); aead_request_set_assoc(subreq, assoc, assoclen); -- cgit v0.10.2 From e9b7441a9926b8091c01431f10231c9ceac52c8f Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sun, 7 Apr 2013 16:43:51 +0300 Subject: crypto: testmgr - add AES GMAC test vectors Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 87ef7d6..6b911ef 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1225,6 +1225,10 @@ static int do_test(int m) ret += tcrypt_test("rfc4106(gcm(aes))"); break; + case 152: + ret += tcrypt_test("rfc4543(gcm(aes))"); + break; + case 200: test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index efd8b20..442ddb4 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -2696,8 +2696,6 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { - - .alg = "rfc4309(ccm(aes))", .test = alg_test_aead, .fips_allowed = 1, @@ -2714,6 +2712,21 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "rfc4543(gcm(aes))", + .test = alg_test_aead, + .suite = { + .aead = { + .enc = { + .vecs = aes_gcm_rfc4543_enc_tv_template, + .count = AES_GCM_4543_ENC_TEST_VECTORS + }, + .dec = { + .vecs = aes_gcm_rfc4543_dec_tv_template, + .count = AES_GCM_4543_DEC_TEST_VECTORS + }, + } + } + }, { .alg = "rmd128", .test = alg_test_hash, .suite = { diff --git a/crypto/testmgr.h b/crypto/testmgr.h index b5721e0..92db37d 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -12680,6 +12680,8 @@ static struct cipher_testvec cast6_xts_dec_tv_template[] = { #define AES_GCM_DEC_TEST_VECTORS 8 #define AES_GCM_4106_ENC_TEST_VECTORS 7 #define AES_GCM_4106_DEC_TEST_VECTORS 7 +#define AES_GCM_4543_ENC_TEST_VECTORS 1 +#define AES_GCM_4543_DEC_TEST_VECTORS 2 #define AES_CCM_ENC_TEST_VECTORS 7 #define AES_CCM_DEC_TEST_VECTORS 7 #define AES_CCM_4309_ENC_TEST_VECTORS 7 @@ -18193,6 +18195,93 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { } }; +static struct aead_testvec aes_gcm_rfc4543_enc_tv_template[] = { + { /* From draft-mcgrew-gcm-test-01 */ + .key = "\x4c\x80\xcd\xef\xbb\x5d\x10\xda" + "\x90\x6a\xc7\x3c\x36\x13\xa6\x34" + "\x22\x43\x3c\x64", + .klen = 20, + .iv = zeroed_string, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x07", + .alen = 8, + .input = "\x45\x00\x00\x30\xda\x3a\x00\x00" + "\x80\x01\xdf\x3b\xc0\xa8\x00\x05" + "\xc0\xa8\x00\x01\x08\x00\xc6\xcd" + "\x02\x00\x07\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x6f\x70\x71\x72\x73\x74" + "\x01\x02\x02\x01", + .ilen = 52, + .result = "\x45\x00\x00\x30\xda\x3a\x00\x00" + "\x80\x01\xdf\x3b\xc0\xa8\x00\x05" + "\xc0\xa8\x00\x01\x08\x00\xc6\xcd" + "\x02\x00\x07\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x6f\x70\x71\x72\x73\x74" + "\x01\x02\x02\x01\xf2\xa9\xa8\x36" + "\xe1\x55\x10\x6a\xa8\xdc\xd6\x18" + "\xe4\x09\x9a\xaa", + .rlen = 68, + } +}; + +static struct aead_testvec aes_gcm_rfc4543_dec_tv_template[] = { + { /* From draft-mcgrew-gcm-test-01 */ + .key = "\x4c\x80\xcd\xef\xbb\x5d\x10\xda" + "\x90\x6a\xc7\x3c\x36\x13\xa6\x34" + "\x22\x43\x3c\x64", + .klen = 20, + .iv = zeroed_string, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x07", + .alen = 8, + .input = "\x45\x00\x00\x30\xda\x3a\x00\x00" + "\x80\x01\xdf\x3b\xc0\xa8\x00\x05" + "\xc0\xa8\x00\x01\x08\x00\xc6\xcd" + "\x02\x00\x07\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x6f\x70\x71\x72\x73\x74" + "\x01\x02\x02\x01\xf2\xa9\xa8\x36" + "\xe1\x55\x10\x6a\xa8\xdc\xd6\x18" + "\xe4\x09\x9a\xaa", + .ilen = 68, + .result = "\x45\x00\x00\x30\xda\x3a\x00\x00" + "\x80\x01\xdf\x3b\xc0\xa8\x00\x05" + "\xc0\xa8\x00\x01\x08\x00\xc6\xcd" + "\x02\x00\x07\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x6f\x70\x71\x72\x73\x74" + "\x01\x02\x02\x01", + .rlen = 52, + }, { /* nearly same as previous, but should fail */ + .key = "\x4c\x80\xcd\xef\xbb\x5d\x10\xda" + "\x90\x6a\xc7\x3c\x36\x13\xa6\x34" + "\x22\x43\x3c\x64", + .klen = 20, + .iv = zeroed_string, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x07", + .alen = 8, + .input = "\x45\x00\x00\x30\xda\x3a\x00\x00" + "\x80\x01\xdf\x3b\xc0\xa8\x00\x05" + "\xc0\xa8\x00\x01\x08\x00\xc6\xcd" + "\x02\x00\x07\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x6f\x70\x71\x72\x73\x74" + "\x01\x02\x02\x01\xf2\xa9\xa8\x36" + "\xe1\x55\x10\x6a\xa8\xdc\xd6\x18" + "\x00\x00\x00\x00", + .ilen = 68, + .novrfy = 1, + .result = "\x45\x00\x00\x30\xda\x3a\x00\x00" + "\x80\x01\xdf\x3b\xc0\xa8\x00\x05" + "\xc0\xa8\x00\x01\x08\x00\xc6\xcd" + "\x02\x00\x07\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x6f\x70\x71\x72\x73\x74" + "\x01\x02\x02\x01", + .rlen = 52, + }, +}; + static struct aead_testvec aes_ccm_enc_tv_template[] = { { /* From RFC 3610 */ .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" -- cgit v0.10.2 From e448370d7377f064c2fef55f72e9b45184bf0926 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sun, 7 Apr 2013 16:43:56 +0300 Subject: crypto: testmgr - add empty test vectors for null ciphers Without these, kernel log shows: [ 5.984881] alg: No test for cipher_null (cipher_null-generic) [ 5.985096] alg: No test for ecb(cipher_null) (ecb-cipher_null) [ 5.985170] alg: No test for compress_null (compress_null-generic) [ 5.985297] alg: No test for digest_null (digest_null-generic) Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 442ddb4..f37e544 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1913,6 +1913,9 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "compress_null", + .test = alg_test_null, + }, { .alg = "crc32c", .test = alg_test_crc32c, .fips_allowed = 1, @@ -2127,6 +2130,9 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "digest_null", + .test = alg_test_null, + }, { .alg = "ecb(__aes-aesni)", .test = alg_test_null, .fips_allowed = 1, @@ -2237,6 +2243,9 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "ecb(cipher_null)", + .test = alg_test_null, + }, { .alg = "ecb(des)", .test = alg_test_skcipher, .fips_allowed = 1, -- cgit v0.10.2 From 93b5e86a6d13c5dec18c6611933fb38d7d80f0d2 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Mon, 8 Apr 2013 10:48:44 +0300 Subject: crypto: add CMAC support to CryptoAPI Patch adds support for NIST recommended block cipher mode CMAC to CryptoAPI. This work is based on Tom St Denis' earlier patch, http://marc.info/?l=linux-crypto-vger&m=135877306305466&w=2 Cc: Tom St Denis Signed-off-by: Jussi Kivilinna Acked-by: David S. Miller Signed-off-by: Herbert Xu diff --git a/crypto/Kconfig b/crypto/Kconfig index 6cc27f1..c1142f3 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -283,6 +283,17 @@ config CRYPTO_XTS comment "Hash modes" +config CRYPTO_CMAC + tristate "CMAC support" + select CRYPTO_HASH + select CRYPTO_MANAGER + help + Cipher-based Message Authentication Code (CMAC) specified by + The National Institute of Standards and Technology (NIST). + + https://tools.ietf.org/html/rfc4493 + http://csrc.nist.gov/publications/nistpubs/800-38B/SP_800-38B.pdf + config CRYPTO_HMAC tristate "HMAC support" select CRYPTO_HASH diff --git a/crypto/Makefile b/crypto/Makefile index be1a1be..a8e9b0fe 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -32,6 +32,7 @@ cryptomgr-y := algboss.o testmgr.o obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o obj-$(CONFIG_CRYPTO_USER) += crypto_user.o +obj-$(CONFIG_CRYPTO_CMAC) += cmac.o obj-$(CONFIG_CRYPTO_HMAC) += hmac.o obj-$(CONFIG_CRYPTO_VMAC) += vmac.o obj-$(CONFIG_CRYPTO_XCBC) += xcbc.o diff --git a/crypto/cmac.c b/crypto/cmac.c new file mode 100644 index 0000000..50880cf --- /dev/null +++ b/crypto/cmac.c @@ -0,0 +1,315 @@ +/* + * CMAC: Cipher Block Mode for Authentication + * + * Copyright © 2013 Jussi Kivilinna + * + * Based on work by: + * Copyright © 2013 Tom St Denis + * Based on crypto/xcbc.c: + * Copyright © 2006 USAGI/WIDE Project, + * Author: Kazunori Miyazawa + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#include +#include +#include +#include + +/* + * +------------------------ + * | + * +------------------------ + * | cmac_tfm_ctx + * +------------------------ + * | consts (block size * 2) + * +------------------------ + */ +struct cmac_tfm_ctx { + struct crypto_cipher *child; + u8 ctx[]; +}; + +/* + * +------------------------ + * | + * +------------------------ + * | cmac_desc_ctx + * +------------------------ + * | odds (block size) + * +------------------------ + * | prev (block size) + * +------------------------ + */ +struct cmac_desc_ctx { + unsigned int len; + u8 ctx[]; +}; + +static int crypto_cmac_digest_setkey(struct crypto_shash *parent, + const u8 *inkey, unsigned int keylen) +{ + unsigned long alignmask = crypto_shash_alignmask(parent); + struct cmac_tfm_ctx *ctx = crypto_shash_ctx(parent); + unsigned int bs = crypto_shash_blocksize(parent); + __be64 *consts = PTR_ALIGN((void *)ctx->ctx, alignmask + 1); + u64 _const[2]; + int i, err = 0; + u8 msb_mask, gfmask; + + err = crypto_cipher_setkey(ctx->child, inkey, keylen); + if (err) + return err; + + /* encrypt the zero block */ + memset(consts, 0, bs); + crypto_cipher_encrypt_one(ctx->child, (u8 *)consts, (u8 *)consts); + + switch (bs) { + case 16: + gfmask = 0x87; + _const[0] = be64_to_cpu(consts[1]); + _const[1] = be64_to_cpu(consts[0]); + + /* gf(2^128) multiply zero-ciphertext with u and u^2 */ + for (i = 0; i < 4; i += 2) { + msb_mask = ((s64)_const[1] >> 63) & gfmask; + _const[1] = (_const[1] << 1) | (_const[0] >> 63); + _const[0] = (_const[0] << 1) ^ msb_mask; + + consts[i + 0] = cpu_to_be64(_const[1]); + consts[i + 1] = cpu_to_be64(_const[0]); + } + + break; + case 8: + gfmask = 0x1B; + _const[0] = be64_to_cpu(consts[0]); + + /* gf(2^64) multiply zero-ciphertext with u and u^2 */ + for (i = 0; i < 2; i++) { + msb_mask = ((s64)_const[0] >> 63) & gfmask; + _const[0] = (_const[0] << 1) ^ msb_mask; + + consts[i] = cpu_to_be64(_const[0]); + } + + break; + } + + return 0; +} + +static int crypto_cmac_digest_init(struct shash_desc *pdesc) +{ + unsigned long alignmask = crypto_shash_alignmask(pdesc->tfm); + struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc); + int bs = crypto_shash_blocksize(pdesc->tfm); + u8 *prev = PTR_ALIGN((void *)ctx->ctx, alignmask + 1) + bs; + + ctx->len = 0; + memset(prev, 0, bs); + + return 0; +} + +static int crypto_cmac_digest_update(struct shash_desc *pdesc, const u8 *p, + unsigned int len) +{ + struct crypto_shash *parent = pdesc->tfm; + unsigned long alignmask = crypto_shash_alignmask(parent); + struct cmac_tfm_ctx *tctx = crypto_shash_ctx(parent); + struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc); + struct crypto_cipher *tfm = tctx->child; + int bs = crypto_shash_blocksize(parent); + u8 *odds = PTR_ALIGN((void *)ctx->ctx, alignmask + 1); + u8 *prev = odds + bs; + + /* checking the data can fill the block */ + if ((ctx->len + len) <= bs) { + memcpy(odds + ctx->len, p, len); + ctx->len += len; + return 0; + } + + /* filling odds with new data and encrypting it */ + memcpy(odds + ctx->len, p, bs - ctx->len); + len -= bs - ctx->len; + p += bs - ctx->len; + + crypto_xor(prev, odds, bs); + crypto_cipher_encrypt_one(tfm, prev, prev); + + /* clearing the length */ + ctx->len = 0; + + /* encrypting the rest of data */ + while (len > bs) { + crypto_xor(prev, p, bs); + crypto_cipher_encrypt_one(tfm, prev, prev); + p += bs; + len -= bs; + } + + /* keeping the surplus of blocksize */ + if (len) { + memcpy(odds, p, len); + ctx->len = len; + } + + return 0; +} + +static int crypto_cmac_digest_final(struct shash_desc *pdesc, u8 *out) +{ + struct crypto_shash *parent = pdesc->tfm; + unsigned long alignmask = crypto_shash_alignmask(parent); + struct cmac_tfm_ctx *tctx = crypto_shash_ctx(parent); + struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc); + struct crypto_cipher *tfm = tctx->child; + int bs = crypto_shash_blocksize(parent); + u8 *consts = PTR_ALIGN((void *)tctx->ctx, alignmask + 1); + u8 *odds = PTR_ALIGN((void *)ctx->ctx, alignmask + 1); + u8 *prev = odds + bs; + unsigned int offset = 0; + + if (ctx->len != bs) { + unsigned int rlen; + u8 *p = odds + ctx->len; + + *p = 0x80; + p++; + + rlen = bs - ctx->len - 1; + if (rlen) + memset(p, 0, rlen); + + offset += bs; + } + + crypto_xor(prev, odds, bs); + crypto_xor(prev, consts + offset, bs); + + crypto_cipher_encrypt_one(tfm, out, prev); + + return 0; +} + +static int cmac_init_tfm(struct crypto_tfm *tfm) +{ + struct crypto_cipher *cipher; + struct crypto_instance *inst = (void *)tfm->__crt_alg; + struct crypto_spawn *spawn = crypto_instance_ctx(inst); + struct cmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + cipher = crypto_spawn_cipher(spawn); + if (IS_ERR(cipher)) + return PTR_ERR(cipher); + + ctx->child = cipher; + + return 0; +}; + +static void cmac_exit_tfm(struct crypto_tfm *tfm) +{ + struct cmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + crypto_free_cipher(ctx->child); +} + +static int cmac_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct shash_instance *inst; + struct crypto_alg *alg; + unsigned long alignmask; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH); + if (err) + return err; + + alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, + CRYPTO_ALG_TYPE_MASK); + if (IS_ERR(alg)) + return PTR_ERR(alg); + + switch (alg->cra_blocksize) { + case 16: + case 8: + break; + default: + goto out_put_alg; + } + + inst = shash_alloc_instance("cmac", alg); + err = PTR_ERR(inst); + if (IS_ERR(inst)) + goto out_put_alg; + + err = crypto_init_spawn(shash_instance_ctx(inst), alg, + shash_crypto_instance(inst), + CRYPTO_ALG_TYPE_MASK); + if (err) + goto out_free_inst; + + alignmask = alg->cra_alignmask | (sizeof(long) - 1); + inst->alg.base.cra_alignmask = alignmask; + inst->alg.base.cra_priority = alg->cra_priority; + inst->alg.base.cra_blocksize = alg->cra_blocksize; + + inst->alg.digestsize = alg->cra_blocksize; + inst->alg.descsize = + ALIGN(sizeof(struct cmac_desc_ctx), crypto_tfm_ctx_alignment()) + + (alignmask & ~(crypto_tfm_ctx_alignment() - 1)) + + alg->cra_blocksize * 2; + + inst->alg.base.cra_ctxsize = + ALIGN(sizeof(struct cmac_tfm_ctx), alignmask + 1) + + alg->cra_blocksize * 2; + + inst->alg.base.cra_init = cmac_init_tfm; + inst->alg.base.cra_exit = cmac_exit_tfm; + + inst->alg.init = crypto_cmac_digest_init; + inst->alg.update = crypto_cmac_digest_update; + inst->alg.final = crypto_cmac_digest_final; + inst->alg.setkey = crypto_cmac_digest_setkey; + + err = shash_register_instance(tmpl, inst); + if (err) { +out_free_inst: + shash_free_instance(shash_crypto_instance(inst)); + } + +out_put_alg: + crypto_mod_put(alg); + return err; +} + +static struct crypto_template crypto_cmac_tmpl = { + .name = "cmac", + .create = cmac_create, + .free = shash_free_instance, + .module = THIS_MODULE, +}; + +static int __init crypto_cmac_module_init(void) +{ + return crypto_register_template(&crypto_cmac_tmpl); +} + +static void __exit crypto_cmac_module_exit(void) +{ + crypto_unregister_template(&crypto_cmac_tmpl); +} + +module_init(crypto_cmac_module_init); +module_exit(crypto_cmac_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("CMAC keyed hash algorithm"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 6b911ef..24ea7df 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1095,7 +1095,6 @@ static int do_test(int m) break; case 28: - ret += tcrypt_test("tgr160"); break; @@ -1118,6 +1117,7 @@ static int do_test(int m) ret += tcrypt_test("lrw(camellia)"); ret += tcrypt_test("xts(camellia)"); break; + case 33: ret += tcrypt_test("sha224"); break; @@ -1213,6 +1213,7 @@ static int do_test(int m) case 109: ret += tcrypt_test("vmac(aes)"); break; + case 110: ret += tcrypt_test("hmac(crc32)"); break; @@ -1229,6 +1230,14 @@ static int do_test(int m) ret += tcrypt_test("rfc4543(gcm(aes))"); break; + case 153: + ret += tcrypt_test("cmac(aes)"); + break; + + case 154: + ret += tcrypt_test("cmac(des3_ede)"); + break; + case 200: test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index f37e544..3807084 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1913,6 +1913,24 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "cmac(aes)", + .test = alg_test_hash, + .suite = { + .hash = { + .vecs = aes_cmac128_tv_template, + .count = CMAC_AES_TEST_VECTORS + } + } + }, { + .alg = "cmac(des3_ede)", + .test = alg_test_hash, + .suite = { + .hash = { + .vecs = des3_ede_cmac64_tv_template, + .count = CMAC_DES3_EDE_TEST_VECTORS + } + } + }, { .alg = "compress_null", .test = alg_test_null, }, { diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 92db37d..d503660 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -1639,6 +1639,131 @@ static struct hash_testvec hmac_sha256_tv_template[] = { }, }; +#define CMAC_AES_TEST_VECTORS 6 + +static struct hash_testvec aes_cmac128_tv_template[] = { + { /* From NIST Special Publication 800-38B, AES-128 */ + .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" + "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + .plaintext = zeroed_string, + .digest = "\xbb\x1d\x69\x29\xe9\x59\x37\x28" + "\x7f\xa3\x7d\x12\x9b\x75\x67\x46", + .psize = 0, + .ksize = 16, + }, { + .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" + "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a", + .digest = "\x07\x0a\x16\xb4\x6b\x4d\x41\x44" + "\xf7\x9b\xdd\x9d\xd0\x4a\x28\x7c", + .psize = 16, + .ksize = 16, + }, { + .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" + "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" + "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11", + .digest = "\xdf\xa6\x67\x47\xde\x9a\xe6\x30" + "\x30\xca\x32\x61\x14\x97\xc8\x27", + .psize = 40, + .ksize = 16, + }, { + .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" + "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" + "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" + "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" + "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" + "\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + .digest = "\x51\xf0\xbe\xbf\x7e\x3b\x9d\x92" + "\xfc\x49\x74\x17\x79\x36\x3c\xfe", + .psize = 64, + .ksize = 16, + }, { /* From NIST Special Publication 800-38B, AES-256 */ + .key = "\x60\x3d\xeb\x10\x15\xca\x71\xbe" + "\x2b\x73\xae\xf0\x85\x7d\x77\x81" + "\x1f\x35\x2c\x07\x3b\x61\x08\xd7" + "\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + .plaintext = zeroed_string, + .digest = "\x02\x89\x62\xf6\x1b\x7b\xf8\x9e" + "\xfc\x6b\x55\x1f\x46\x67\xd9\x83", + .psize = 0, + .ksize = 32, + }, { + .key = "\x60\x3d\xeb\x10\x15\xca\x71\xbe" + "\x2b\x73\xae\xf0\x85\x7d\x77\x81" + "\x1f\x35\x2c\x07\x3b\x61\x08\xd7" + "\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" + "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" + "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" + "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" + "\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + .digest = "\xe1\x99\x21\x90\x54\x9f\x6e\xd5" + "\x69\x6a\x2c\x05\x6c\x31\x54\x10", + .psize = 64, + .ksize = 32, + } +}; + +#define CMAC_DES3_EDE_TEST_VECTORS 4 + +static struct hash_testvec des3_ede_cmac64_tv_template[] = { +/* + * From NIST Special Publication 800-38B, Three Key TDEA + * Corrected test vectors from: + * http://csrc.nist.gov/publications/nistpubs/800-38B/Updated_CMAC_Examples.pdf + */ + { + .key = "\x8a\xa8\x3b\xf8\xcb\xda\x10\x62" + "\x0b\xc1\xbf\x19\xfb\xb6\xcd\x58" + "\xbc\x31\x3d\x4a\x37\x1c\xa8\xb5", + .plaintext = zeroed_string, + .digest = "\xb7\xa6\x88\xe1\x22\xff\xaf\x95", + .psize = 0, + .ksize = 24, + }, { + .key = "\x8a\xa8\x3b\xf8\xcb\xda\x10\x62" + "\x0b\xc1\xbf\x19\xfb\xb6\xcd\x58" + "\xbc\x31\x3d\x4a\x37\x1c\xa8\xb5", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96", + .digest = "\x8e\x8f\x29\x31\x36\x28\x37\x97", + .psize = 8, + .ksize = 24, + }, { + .key = "\x8a\xa8\x3b\xf8\xcb\xda\x10\x62" + "\x0b\xc1\xbf\x19\xfb\xb6\xcd\x58" + "\xbc\x31\x3d\x4a\x37\x1c\xa8\xb5", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" + "\xae\x2d\x8a\x57", + .digest = "\x74\x3d\xdb\xe0\xce\x2d\xc2\xed", + .psize = 20, + .ksize = 24, + }, { + .key = "\x8a\xa8\x3b\xf8\xcb\xda\x10\x62" + "\x0b\xc1\xbf\x19\xfb\xb6\xcd\x58" + "\xbc\x31\x3d\x4a\x37\x1c\xa8\xb5", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51", + .digest = "\x33\xe6\xb1\x09\x24\x00\xea\xe5", + .psize = 32, + .ksize = 24, + } +}; + #define XCBC_AES_TEST_VECTORS 6 static struct hash_testvec aes_xcbc128_tv_template[] = { -- cgit v0.10.2 From d2049d8566bf74723f0c353621174b37ff3d75ec Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Mon, 8 Apr 2013 10:48:49 +0300 Subject: xfrm: add rfc4494 AES-CMAC-96 support Now that CryptoAPI has support for CMAC, we can add support for AES-CMAC-96 (rfc4494). Cc: Tom St Denis Signed-off-by: Jussi Kivilinna Acked-by: David S. Miller Signed-off-by: Herbert Xu diff --git a/net/xfrm/xfrm_algo.c b/net/xfrm/xfrm_algo.c index 6fb9d00..ab4ef72 100644 --- a/net/xfrm/xfrm_algo.c +++ b/net/xfrm/xfrm_algo.c @@ -311,6 +311,19 @@ static struct xfrm_algo_desc aalg_list[] = { .sadb_alg_maxbits = 128 } }, +{ + /* rfc4494 */ + .name = "cmac(aes)", + + .uinfo = { + .auth = { + .icv_truncbits = 96, + .icv_fullbits = 128, + } + }, + + .pfkey_supported = 0, +}, }; static struct xfrm_algo_desc ealg_list[] = { -- cgit v0.10.2 From a05248ed2d9a83ae7c3e6db7c4ef9331c3dedc81 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Mon, 8 Apr 2013 21:50:55 +0300 Subject: crypto: x86 - add more optimized XTS-mode for serpent-avx This patch adds AVX optimized XTS-mode helper functions/macros and converts serpent-avx to use the new facilities. Benefits are slightly improved speed and reduced stack usage as use of temporary IV-array is avoided. tcrypt results, with Intel i5-2450M: enc dec 16B 1.00x 1.00x 64B 1.00x 1.00x 256B 1.04x 1.06x 1024B 1.09x 1.09x 8192B 1.10x 1.09x Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/glue_helper-asm-avx.S b/arch/x86/crypto/glue_helper-asm-avx.S index f7b6ea2..02ee230 100644 --- a/arch/x86/crypto/glue_helper-asm-avx.S +++ b/arch/x86/crypto/glue_helper-asm-avx.S @@ -1,7 +1,7 @@ /* * Shared glue code for 128bit block ciphers, AVX assembler macros * - * Copyright (c) 2012 Jussi Kivilinna + * Copyright © 2012-2013 Jussi Kivilinna * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -89,3 +89,62 @@ vpxor (6*16)(src), x6, x6; \ vpxor (7*16)(src), x7, x7; \ store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7); + +#define gf128mul_x_ble(iv, mask, tmp) \ + vpsrad $31, iv, tmp; \ + vpaddq iv, iv, iv; \ + vpshufd $0x13, tmp, tmp; \ + vpand mask, tmp, tmp; \ + vpxor tmp, iv, iv; + +#define load_xts_8way(iv, src, dst, x0, x1, x2, x3, x4, x5, x6, x7, tiv, t0, \ + t1, xts_gf128mul_and_shl1_mask) \ + vmovdqa xts_gf128mul_and_shl1_mask, t0; \ + \ + /* load IV */ \ + vmovdqu (iv), tiv; \ + vpxor (0*16)(src), tiv, x0; \ + vmovdqu tiv, (0*16)(dst); \ + \ + /* construct and store IVs, also xor with source */ \ + gf128mul_x_ble(tiv, t0, t1); \ + vpxor (1*16)(src), tiv, x1; \ + vmovdqu tiv, (1*16)(dst); \ + \ + gf128mul_x_ble(tiv, t0, t1); \ + vpxor (2*16)(src), tiv, x2; \ + vmovdqu tiv, (2*16)(dst); \ + \ + gf128mul_x_ble(tiv, t0, t1); \ + vpxor (3*16)(src), tiv, x3; \ + vmovdqu tiv, (3*16)(dst); \ + \ + gf128mul_x_ble(tiv, t0, t1); \ + vpxor (4*16)(src), tiv, x4; \ + vmovdqu tiv, (4*16)(dst); \ + \ + gf128mul_x_ble(tiv, t0, t1); \ + vpxor (5*16)(src), tiv, x5; \ + vmovdqu tiv, (5*16)(dst); \ + \ + gf128mul_x_ble(tiv, t0, t1); \ + vpxor (6*16)(src), tiv, x6; \ + vmovdqu tiv, (6*16)(dst); \ + \ + gf128mul_x_ble(tiv, t0, t1); \ + vpxor (7*16)(src), tiv, x7; \ + vmovdqu tiv, (7*16)(dst); \ + \ + gf128mul_x_ble(tiv, t0, t1); \ + vmovdqu tiv, (iv); + +#define store_xts_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7) \ + vpxor (0*16)(dst), x0, x0; \ + vpxor (1*16)(dst), x1, x1; \ + vpxor (2*16)(dst), x2, x2; \ + vpxor (3*16)(dst), x3, x3; \ + vpxor (4*16)(dst), x4, x4; \ + vpxor (5*16)(dst), x5, x5; \ + vpxor (6*16)(dst), x6, x6; \ + vpxor (7*16)(dst), x7, x7; \ + store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7); diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c index 22ce4f6..432f1d76 100644 --- a/arch/x86/crypto/glue_helper.c +++ b/arch/x86/crypto/glue_helper.c @@ -1,7 +1,7 @@ /* * Shared glue code for 128bit block ciphers * - * Copyright (c) 2012 Jussi Kivilinna + * Copyright © 2012-2013 Jussi Kivilinna * * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: * Copyright (c) 2006 Herbert Xu @@ -304,4 +304,99 @@ int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx, } EXPORT_SYMBOL_GPL(glue_ctr_crypt_128bit); +static unsigned int __glue_xts_crypt_128bit(const struct common_glue_ctx *gctx, + void *ctx, + struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + const unsigned int bsize = 128 / 8; + unsigned int nbytes = walk->nbytes; + u128 *src = (u128 *)walk->src.virt.addr; + u128 *dst = (u128 *)walk->dst.virt.addr; + unsigned int num_blocks, func_bytes; + unsigned int i; + + /* Process multi-block batch */ + for (i = 0; i < gctx->num_funcs; i++) { + num_blocks = gctx->funcs[i].num_blocks; + func_bytes = bsize * num_blocks; + + if (nbytes >= func_bytes) { + do { + gctx->funcs[i].fn_u.xts(ctx, dst, src, + (le128 *)walk->iv); + + src += num_blocks; + dst += num_blocks; + nbytes -= func_bytes; + } while (nbytes >= func_bytes); + + if (nbytes < bsize) + goto done; + } + } + +done: + return nbytes; +} + +/* for implementations implementing faster XTS IV generator */ +int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes, + void (*tweak_fn)(void *ctx, u8 *dst, const u8 *src), + void *tweak_ctx, void *crypt_ctx) +{ + const unsigned int bsize = 128 / 8; + bool fpu_enabled = false; + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + + err = blkcipher_walk_virt(desc, &walk); + nbytes = walk.nbytes; + if (!nbytes) + return err; + + /* set minimum length to bsize, for tweak_fn */ + fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, + desc, fpu_enabled, + nbytes < bsize ? bsize : nbytes); + + /* calculate first value of T */ + tweak_fn(tweak_ctx, walk.iv, walk.iv); + + while (nbytes) { + nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk); + + err = blkcipher_walk_done(desc, &walk, nbytes); + nbytes = walk.nbytes; + } + + glue_fpu_end(fpu_enabled); + + return err; +} +EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit); + +void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv, + common_glue_func_t fn) +{ + le128 ivblk = *iv; + + /* generate next IV */ + le128_gf128mul_x_ble(iv, &ivblk); + + /* CC <- T xor C */ + u128_xor(dst, src, (u128 *)&ivblk); + + /* PP <- D(Key2,CC) */ + fn(ctx, (u8 *)dst, (u8 *)dst); + + /* P <- T xor PP */ + u128_xor(dst, dst, (u128 *)&ivblk); +} +EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one); + MODULE_LICENSE("GPL"); diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S index 43c9386..2f202f4 100644 --- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S @@ -4,8 +4,7 @@ * Copyright (C) 2012 Johannes Goetzfried * * - * Based on arch/x86/crypto/serpent-sse2-x86_64-asm_64.S by - * Copyright (C) 2011 Jussi Kivilinna + * Copyright © 2011-2013 Jussi Kivilinna * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -34,6 +33,8 @@ .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +.Lxts_gf128mul_and_shl1_mask: + .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 .text @@ -739,3 +740,43 @@ ENTRY(serpent_ctr_8way_avx) ret; ENDPROC(serpent_ctr_8way_avx) + +ENTRY(serpent_xts_enc_8way_avx) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + /* regs <= src, dst <= IVs, regs <= regs xor IVs */ + load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, + RK0, RK1, RK2, .Lxts_gf128mul_and_shl1_mask); + + call __serpent_enc_blk8_avx; + + /* dst <= regs xor IVs(in dst) */ + store_xts_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); + + ret; +ENDPROC(serpent_xts_enc_8way_avx) + +ENTRY(serpent_xts_dec_8way_avx) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + /* regs <= src, dst <= IVs, regs <= regs xor IVs */ + load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, + RK0, RK1, RK2, .Lxts_gf128mul_and_shl1_mask); + + call __serpent_dec_blk8_avx; + + /* dst <= regs xor IVs(in dst) */ + store_xts_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); + + ret; +ENDPROC(serpent_xts_dec_8way_avx) diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c index 52abaaf..0f8519c 100644 --- a/arch/x86/crypto/serpent_avx_glue.c +++ b/arch/x86/crypto/serpent_avx_glue.c @@ -4,8 +4,7 @@ * Copyright (C) 2012 Johannes Goetzfried * * - * Glue code based on serpent_sse2_glue.c by: - * Copyright (C) 2011 Jussi Kivilinna + * Copyright © 2011-2013 Jussi Kivilinna * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -53,6 +52,18 @@ static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv) u128_xor(dst, src, (u128 *)&ctrblk); } +static void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + glue_xts_crypt_128bit_one(ctx, dst, src, iv, + GLUE_FUNC_CAST(__serpent_encrypt)); +} + +static void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + glue_xts_crypt_128bit_one(ctx, dst, src, iv, + GLUE_FUNC_CAST(__serpent_decrypt)); +} + static const struct common_glue_ctx serpent_enc = { .num_funcs = 2, .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, @@ -79,6 +90,19 @@ static const struct common_glue_ctx serpent_ctr = { } } }; +static const struct common_glue_ctx serpent_enc_xts = { + .num_funcs = 2, + .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = SERPENT_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) } + } } +}; + static const struct common_glue_ctx serpent_dec = { .num_funcs = 2, .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, @@ -105,6 +129,19 @@ static const struct common_glue_ctx serpent_dec_cbc = { } } }; +static const struct common_glue_ctx serpent_dec_xts = { + .num_funcs = 2, + .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = SERPENT_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) } + } } +}; + static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { @@ -299,54 +336,20 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[SERPENT_PARALLEL_BLOCKS]; - struct crypt_priv crypt_ctx = { - .ctx = &ctx->crypt_ctx, - .fpu_enabled = false, - }; - struct xts_crypt_req req = { - .tbuf = buf, - .tbuflen = sizeof(buf), - - .tweak_ctx = &ctx->tweak_ctx, - .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt), - .crypt_ctx = &crypt_ctx, - .crypt_fn = encrypt_callback, - }; - int ret; - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - ret = xts_crypt(desc, dst, src, nbytes, &req); - serpent_fpu_end(crypt_ctx.fpu_enabled); - - return ret; + return glue_xts_crypt_128bit(&serpent_enc_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(__serpent_encrypt), + &ctx->tweak_ctx, &ctx->crypt_ctx); } static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[SERPENT_PARALLEL_BLOCKS]; - struct crypt_priv crypt_ctx = { - .ctx = &ctx->crypt_ctx, - .fpu_enabled = false, - }; - struct xts_crypt_req req = { - .tbuf = buf, - .tbuflen = sizeof(buf), - - .tweak_ctx = &ctx->tweak_ctx, - .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt), - .crypt_ctx = &crypt_ctx, - .crypt_fn = decrypt_callback, - }; - int ret; - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - ret = xts_crypt(desc, dst, src, nbytes, &req); - serpent_fpu_end(crypt_ctx.fpu_enabled); - - return ret; + return glue_xts_crypt_128bit(&serpent_dec_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(__serpent_encrypt), + &ctx->tweak_ctx, &ctx->crypt_ctx); } static struct crypto_alg serpent_algs[10] = { { diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h index e2d65b0..1eef555 100644 --- a/arch/x86/include/asm/crypto/glue_helper.h +++ b/arch/x86/include/asm/crypto/glue_helper.h @@ -14,10 +14,13 @@ typedef void (*common_glue_func_t)(void *ctx, u8 *dst, const u8 *src); typedef void (*common_glue_cbc_func_t)(void *ctx, u128 *dst, const u128 *src); typedef void (*common_glue_ctr_func_t)(void *ctx, u128 *dst, const u128 *src, le128 *iv); +typedef void (*common_glue_xts_func_t)(void *ctx, u128 *dst, const u128 *src, + le128 *iv); #define GLUE_FUNC_CAST(fn) ((common_glue_func_t)(fn)) #define GLUE_CBC_FUNC_CAST(fn) ((common_glue_cbc_func_t)(fn)) #define GLUE_CTR_FUNC_CAST(fn) ((common_glue_ctr_func_t)(fn)) +#define GLUE_XTS_FUNC_CAST(fn) ((common_glue_xts_func_t)(fn)) struct common_glue_func_entry { unsigned int num_blocks; /* number of blocks that @fn will process */ @@ -25,6 +28,7 @@ struct common_glue_func_entry { common_glue_func_t ecb; common_glue_cbc_func_t cbc; common_glue_ctr_func_t ctr; + common_glue_xts_func_t xts; } fn_u; }; @@ -96,6 +100,16 @@ static inline void le128_inc(le128 *i) i->b = cpu_to_le64(b); } +static inline void le128_gf128mul_x_ble(le128 *dst, const le128 *src) +{ + u64 a = le64_to_cpu(src->a); + u64 b = le64_to_cpu(src->b); + u64 _tt = ((s64)a >> 63) & 0x87; + + dst->a = cpu_to_le64((a << 1) ^ (b >> 63)); + dst->b = cpu_to_le64((b << 1) ^ _tt); +} + extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx, struct blkcipher_desc *desc, struct scatterlist *dst, @@ -118,4 +132,14 @@ extern int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes); +extern int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes, + common_glue_func_t tweak_fn, void *tweak_ctx, + void *crypt_ctx); + +extern void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, + le128 *iv, common_glue_func_t fn); + #endif /* _CRYPTO_GLUE_HELPER_H */ diff --git a/arch/x86/include/asm/crypto/serpent-avx.h b/arch/x86/include/asm/crypto/serpent-avx.h index 0da1d3e..56e79cc 100644 --- a/arch/x86/include/asm/crypto/serpent-avx.h +++ b/arch/x86/include/asm/crypto/serpent-avx.h @@ -16,4 +16,9 @@ asmlinkage void serpent_cbc_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst, asmlinkage void serpent_ctr_8way_avx(struct serpent_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +asmlinkage void serpent_xts_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +asmlinkage void serpent_xts_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + #endif -- cgit v0.10.2 From 18be45270a80ab489d9402b63e1f103428f0afde Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Mon, 8 Apr 2013 21:51:00 +0300 Subject: crypto: x86/twofish-avx - use optimized XTS code Change twofish-avx to use the new XTS code, for smaller stack usage and small boost to performance. tcrypt results, with Intel i5-2450M: enc dec 16B 1.03x 1.02x 64B 0.91x 0.91x 256B 1.10x 1.09x 1024B 1.12x 1.11x 8192B 1.12x 1.11x Since XTS is practically always used with data blocks of size 512 bytes or more, I chose to not make use of twofish-3way for block sized smaller than 128 bytes. This causes slower result in tcrypt for 64 bytes. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S index 8d3e113..0505813 100644 --- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S @@ -4,7 +4,7 @@ * Copyright (C) 2012 Johannes Goetzfried * * - * Copyright © 2012 Jussi Kivilinna + * Copyright © 2012-2013 Jussi Kivilinna * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -33,6 +33,8 @@ .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +.Lxts_gf128mul_and_shl1_mask: + .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 .text @@ -408,3 +410,47 @@ ENTRY(twofish_ctr_8way) ret; ENDPROC(twofish_ctr_8way) + +ENTRY(twofish_xts_enc_8way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + movq %rsi, %r11; + + /* regs <= src, dst <= IVs, regs <= regs xor IVs */ + load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, + RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask); + + call __twofish_enc_blk8; + + /* dst <= regs xor IVs(in dst) */ + store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); + + ret; +ENDPROC(twofish_xts_enc_8way) + +ENTRY(twofish_xts_dec_8way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + movq %rsi, %r11; + + /* regs <= src, dst <= IVs, regs <= regs xor IVs */ + load_xts_8way(%rcx, %rdx, %rsi, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2, + RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask); + + call __twofish_dec_blk8; + + /* dst <= regs xor IVs(in dst) */ + store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); + + ret; +ENDPROC(twofish_xts_dec_8way) diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c index 94ac91d..a62ba54 100644 --- a/arch/x86/crypto/twofish_avx_glue.c +++ b/arch/x86/crypto/twofish_avx_glue.c @@ -4,6 +4,8 @@ * Copyright (C) 2012 Johannes Goetzfried * * + * Copyright © 2013 Jussi Kivilinna + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -56,12 +58,29 @@ asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst, asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, const u8 *src) { __twofish_enc_blk_3way(ctx, dst, src, false); } +static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + glue_xts_crypt_128bit_one(ctx, dst, src, iv, + GLUE_FUNC_CAST(twofish_enc_blk)); +} + +static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + glue_xts_crypt_128bit_one(ctx, dst, src, iv, + GLUE_FUNC_CAST(twofish_dec_blk)); +} + static const struct common_glue_ctx twofish_enc = { .num_funcs = 3, @@ -95,6 +114,19 @@ static const struct common_glue_ctx twofish_ctr = { } } }; +static const struct common_glue_ctx twofish_enc_xts = { + .num_funcs = 2, + .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = TWOFISH_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) } + } } +}; + static const struct common_glue_ctx twofish_dec = { .num_funcs = 3, .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, @@ -127,6 +159,19 @@ static const struct common_glue_ctx twofish_dec_cbc = { } } }; +static const struct common_glue_ctx twofish_dec_xts = { + .num_funcs = 2, + .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = TWOFISH_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) } + } } +}; + static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { @@ -275,54 +320,20 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[TWOFISH_PARALLEL_BLOCKS]; - struct crypt_priv crypt_ctx = { - .ctx = &ctx->crypt_ctx, - .fpu_enabled = false, - }; - struct xts_crypt_req req = { - .tbuf = buf, - .tbuflen = sizeof(buf), - - .tweak_ctx = &ctx->tweak_ctx, - .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), - .crypt_ctx = &crypt_ctx, - .crypt_fn = encrypt_callback, - }; - int ret; - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - ret = xts_crypt(desc, dst, src, nbytes, &req); - twofish_fpu_end(crypt_ctx.fpu_enabled); - - return ret; + return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(twofish_enc_blk), + &ctx->tweak_ctx, &ctx->crypt_ctx); } static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[TWOFISH_PARALLEL_BLOCKS]; - struct crypt_priv crypt_ctx = { - .ctx = &ctx->crypt_ctx, - .fpu_enabled = false, - }; - struct xts_crypt_req req = { - .tbuf = buf, - .tbuflen = sizeof(buf), - - .tweak_ctx = &ctx->tweak_ctx, - .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), - .crypt_ctx = &crypt_ctx, - .crypt_fn = decrypt_callback, - }; - int ret; - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - ret = xts_crypt(desc, dst, src, nbytes, &req); - twofish_fpu_end(crypt_ctx.fpu_enabled); - - return ret; + return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(twofish_enc_blk), + &ctx->tweak_ctx, &ctx->crypt_ctx); } static struct crypto_alg twofish_algs[10] = { { -- cgit v0.10.2 From 70177286e1d49dfa2ce565af10d1f63d9b769d77 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Mon, 8 Apr 2013 21:51:05 +0300 Subject: crypto: cast6-avx: use new optimized XTS code Change cast6-avx to use the new XTS code, for smaller stack usage and small boost to performance. tcrypt results, with Intel i5-2450M: enc dec 16B 1.01x 1.01x 64B 1.01x 1.00x 256B 1.09x 1.02x 1024B 1.08x 1.06x 8192B 1.08x 1.07x Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S index f93b610..e3531f8 100644 --- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S @@ -4,7 +4,7 @@ * Copyright (C) 2012 Johannes Goetzfried * * - * Copyright © 2012 Jussi Kivilinna + * Copyright © 2012-2013 Jussi Kivilinna * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -227,6 +227,8 @@ .data .align 16 +.Lxts_gf128mul_and_shl1_mask: + .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 .Lbswap_mask: .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 .Lbswap128_mask: @@ -424,3 +426,47 @@ ENTRY(cast6_ctr_8way) ret; ENDPROC(cast6_ctr_8way) + +ENTRY(cast6_xts_enc_8way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + movq %rsi, %r11; + + /* regs <= src, dst <= IVs, regs <= regs xor IVs */ + load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, + RX, RKR, RKM, .Lxts_gf128mul_and_shl1_mask); + + call __cast6_enc_blk8; + + /* dst <= regs xor IVs(in dst) */ + store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); + + ret; +ENDPROC(cast6_xts_enc_8way) + +ENTRY(cast6_xts_dec_8way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + movq %rsi, %r11; + + /* regs <= src, dst <= IVs, regs <= regs xor IVs */ + load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, + RX, RKR, RKM, .Lxts_gf128mul_and_shl1_mask); + + call __cast6_dec_blk8; + + /* dst <= regs xor IVs(in dst) */ + store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); + + ret; +ENDPROC(cast6_xts_dec_8way) diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c index 92f7ca2..8d0dfb8 100644 --- a/arch/x86/crypto/cast6_avx_glue.c +++ b/arch/x86/crypto/cast6_avx_glue.c @@ -4,6 +4,8 @@ * Copyright (C) 2012 Johannes Goetzfried * * + * Copyright © 2013 Jussi Kivilinna + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -50,6 +52,23 @@ asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx *ctx, u8 *dst, asmlinkage void cast6_ctr_8way(struct cast6_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +asmlinkage void cast6_xts_enc_8way(struct cast6_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +asmlinkage void cast6_xts_dec_8way(struct cast6_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + +static void cast6_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + glue_xts_crypt_128bit_one(ctx, dst, src, iv, + GLUE_FUNC_CAST(__cast6_encrypt)); +} + +static void cast6_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + glue_xts_crypt_128bit_one(ctx, dst, src, iv, + GLUE_FUNC_CAST(__cast6_decrypt)); +} + static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv) { be128 ctrblk; @@ -87,6 +106,19 @@ static const struct common_glue_ctx cast6_ctr = { } } }; +static const struct common_glue_ctx cast6_enc_xts = { + .num_funcs = 2, + .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = CAST6_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc_8way) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc) } + } } +}; + static const struct common_glue_ctx cast6_dec = { .num_funcs = 2, .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, @@ -113,6 +145,19 @@ static const struct common_glue_ctx cast6_dec_cbc = { } } }; +static const struct common_glue_ctx cast6_dec_xts = { + .num_funcs = 2, + .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = CAST6_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec_8way) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec) } + } } +}; + static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { @@ -307,54 +352,20 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[CAST6_PARALLEL_BLOCKS]; - struct crypt_priv crypt_ctx = { - .ctx = &ctx->crypt_ctx, - .fpu_enabled = false, - }; - struct xts_crypt_req req = { - .tbuf = buf, - .tbuflen = sizeof(buf), - .tweak_ctx = &ctx->tweak_ctx, - .tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt), - .crypt_ctx = &crypt_ctx, - .crypt_fn = encrypt_callback, - }; - int ret; - - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - ret = xts_crypt(desc, dst, src, nbytes, &req); - cast6_fpu_end(crypt_ctx.fpu_enabled); - - return ret; + return glue_xts_crypt_128bit(&cast6_enc_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(__cast6_encrypt), + &ctx->tweak_ctx, &ctx->crypt_ctx); } static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[CAST6_PARALLEL_BLOCKS]; - struct crypt_priv crypt_ctx = { - .ctx = &ctx->crypt_ctx, - .fpu_enabled = false, - }; - struct xts_crypt_req req = { - .tbuf = buf, - .tbuflen = sizeof(buf), - - .tweak_ctx = &ctx->tweak_ctx, - .tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt), - .crypt_ctx = &crypt_ctx, - .crypt_fn = decrypt_callback, - }; - int ret; - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - ret = xts_crypt(desc, dst, src, nbytes, &req); - cast6_fpu_end(crypt_ctx.fpu_enabled); - - return ret; + return glue_xts_crypt_128bit(&cast6_dec_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(__cast6_encrypt), + &ctx->tweak_ctx, &ctx->crypt_ctx); } static struct crypto_alg cast6_algs[10] = { { -- cgit v0.10.2 From b5c5b072dc2f35d45d3404b957e264a3e8e71069 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Mon, 8 Apr 2013 21:51:11 +0300 Subject: crypto: x86/camellia-aesni-avx - add more optimized XTS code Add more optimized XTS code for camellia-aesni-avx, for smaller stack usage and small boost for speed. tcrypt results, with Intel i5-2450M: enc dec 16B 1.10x 1.01x 64B 0.82x 0.77x 256B 1.14x 1.10x 1024B 1.17x 1.16x 8192B 1.10x 1.11x Since XTS is practically always used with data blocks of size 512 bytes or more, I chose to not make use of camellia-2way for block sized smaller than 256 bytes. This causes slower result in tcrypt for 64 bytes. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S index cfc1634..ce71f92 100644 --- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S +++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S @@ -1,7 +1,7 @@ /* * x86_64/AVX/AES-NI assembler implementation of Camellia * - * Copyright © 2012 Jussi Kivilinna + * Copyright © 2012-2013 Jussi Kivilinna * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -589,6 +589,10 @@ ENDPROC(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +/* For XTS mode IV generation */ +.Lxts_gf128mul_and_shl1_mask: + .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 + /* * pre-SubByte transform * @@ -1090,3 +1094,177 @@ ENTRY(camellia_ctr_16way) ret; ENDPROC(camellia_ctr_16way) + +#define gf128mul_x_ble(iv, mask, tmp) \ + vpsrad $31, iv, tmp; \ + vpaddq iv, iv, iv; \ + vpshufd $0x13, tmp, tmp; \ + vpand mask, tmp, tmp; \ + vpxor tmp, iv, iv; + +.align 8 +camellia_xts_crypt_16way: + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + * %r8: index for input whitening key + * %r9: pointer to __camellia_enc_blk16 or __camellia_dec_blk16 + */ + + subq $(16 * 16), %rsp; + movq %rsp, %rax; + + vmovdqa .Lxts_gf128mul_and_shl1_mask, %xmm14; + + /* load IV */ + vmovdqu (%rcx), %xmm0; + vpxor 0 * 16(%rdx), %xmm0, %xmm15; + vmovdqu %xmm15, 15 * 16(%rax); + vmovdqu %xmm0, 0 * 16(%rsi); + + /* construct IVs */ + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 1 * 16(%rdx), %xmm0, %xmm15; + vmovdqu %xmm15, 14 * 16(%rax); + vmovdqu %xmm0, 1 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 2 * 16(%rdx), %xmm0, %xmm13; + vmovdqu %xmm0, 2 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 3 * 16(%rdx), %xmm0, %xmm12; + vmovdqu %xmm0, 3 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 4 * 16(%rdx), %xmm0, %xmm11; + vmovdqu %xmm0, 4 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 5 * 16(%rdx), %xmm0, %xmm10; + vmovdqu %xmm0, 5 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 6 * 16(%rdx), %xmm0, %xmm9; + vmovdqu %xmm0, 6 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 7 * 16(%rdx), %xmm0, %xmm8; + vmovdqu %xmm0, 7 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 8 * 16(%rdx), %xmm0, %xmm7; + vmovdqu %xmm0, 8 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 9 * 16(%rdx), %xmm0, %xmm6; + vmovdqu %xmm0, 9 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 10 * 16(%rdx), %xmm0, %xmm5; + vmovdqu %xmm0, 10 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 11 * 16(%rdx), %xmm0, %xmm4; + vmovdqu %xmm0, 11 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 12 * 16(%rdx), %xmm0, %xmm3; + vmovdqu %xmm0, 12 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 13 * 16(%rdx), %xmm0, %xmm2; + vmovdqu %xmm0, 13 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 14 * 16(%rdx), %xmm0, %xmm1; + vmovdqu %xmm0, 14 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vpxor 15 * 16(%rdx), %xmm0, %xmm15; + vmovdqu %xmm15, 0 * 16(%rax); + vmovdqu %xmm0, 15 * 16(%rsi); + + gf128mul_x_ble(%xmm0, %xmm14, %xmm15); + vmovdqu %xmm0, (%rcx); + + /* inpack16_pre: */ + vmovq (key_table)(CTX, %r8, 8), %xmm15; + vpshufb .Lpack_bswap, %xmm15, %xmm15; + vpxor 0 * 16(%rax), %xmm15, %xmm0; + vpxor %xmm1, %xmm15, %xmm1; + vpxor %xmm2, %xmm15, %xmm2; + vpxor %xmm3, %xmm15, %xmm3; + vpxor %xmm4, %xmm15, %xmm4; + vpxor %xmm5, %xmm15, %xmm5; + vpxor %xmm6, %xmm15, %xmm6; + vpxor %xmm7, %xmm15, %xmm7; + vpxor %xmm8, %xmm15, %xmm8; + vpxor %xmm9, %xmm15, %xmm9; + vpxor %xmm10, %xmm15, %xmm10; + vpxor %xmm11, %xmm15, %xmm11; + vpxor %xmm12, %xmm15, %xmm12; + vpxor %xmm13, %xmm15, %xmm13; + vpxor 14 * 16(%rax), %xmm15, %xmm14; + vpxor 15 * 16(%rax), %xmm15, %xmm15; + + call *%r9; + + addq $(16 * 16), %rsp; + + vpxor 0 * 16(%rsi), %xmm7, %xmm7; + vpxor 1 * 16(%rsi), %xmm6, %xmm6; + vpxor 2 * 16(%rsi), %xmm5, %xmm5; + vpxor 3 * 16(%rsi), %xmm4, %xmm4; + vpxor 4 * 16(%rsi), %xmm3, %xmm3; + vpxor 5 * 16(%rsi), %xmm2, %xmm2; + vpxor 6 * 16(%rsi), %xmm1, %xmm1; + vpxor 7 * 16(%rsi), %xmm0, %xmm0; + vpxor 8 * 16(%rsi), %xmm15, %xmm15; + vpxor 9 * 16(%rsi), %xmm14, %xmm14; + vpxor 10 * 16(%rsi), %xmm13, %xmm13; + vpxor 11 * 16(%rsi), %xmm12, %xmm12; + vpxor 12 * 16(%rsi), %xmm11, %xmm11; + vpxor 13 * 16(%rsi), %xmm10, %xmm10; + vpxor 14 * 16(%rsi), %xmm9, %xmm9; + vpxor 15 * 16(%rsi), %xmm8, %xmm8; + write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0, + %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9, + %xmm8, %rsi); + + ret; +ENDPROC(camellia_xts_crypt_16way) + +ENTRY(camellia_xts_enc_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + xorl %r8d, %r8d; /* input whitening key, 0 for enc */ + + leaq __camellia_enc_blk16, %r9; + + jmp camellia_xts_crypt_16way; +ENDPROC(camellia_xts_enc_16way) + +ENTRY(camellia_xts_dec_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + cmpl $16, key_length(CTX); + movl $32, %r8d; + movl $24, %eax; + cmovel %eax, %r8d; /* input whitening key, last for dec */ + + leaq __camellia_dec_blk16, %r9; + + jmp camellia_xts_crypt_16way; +ENDPROC(camellia_xts_dec_16way) diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c index 96cbb60..4ff7ed4 100644 --- a/arch/x86/crypto/camellia_aesni_avx_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx_glue.c @@ -1,7 +1,7 @@ /* * Glue Code for x86_64/AVX/AES-NI assembler optimized version of Camellia * - * Copyright © 2012 Jussi Kivilinna + * Copyright © 2012-2013 Jussi Kivilinna * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -37,6 +37,23 @@ asmlinkage void camellia_cbc_dec_16way(struct camellia_ctx *ctx, u8 *dst, asmlinkage void camellia_ctr_16way(struct camellia_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +asmlinkage void camellia_xts_enc_16way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +asmlinkage void camellia_xts_dec_16way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + +static void camellia_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + glue_xts_crypt_128bit_one(ctx, dst, src, iv, + GLUE_FUNC_CAST(camellia_enc_blk)); +} + +static void camellia_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + glue_xts_crypt_128bit_one(ctx, dst, src, iv, + GLUE_FUNC_CAST(camellia_dec_blk)); +} + static const struct common_glue_ctx camellia_enc = { .num_funcs = 3, .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, @@ -69,6 +86,19 @@ static const struct common_glue_ctx camellia_ctr = { } } }; +static const struct common_glue_ctx camellia_enc_xts = { + .num_funcs = 2, + .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) } + } } +}; + static const struct common_glue_ctx camellia_dec = { .num_funcs = 3, .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, @@ -101,6 +131,19 @@ static const struct common_glue_ctx camellia_dec_cbc = { } } }; +static const struct common_glue_ctx camellia_dec_xts = { + .num_funcs = 2, + .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) } + } } +}; + static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { @@ -261,54 +304,20 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[CAMELLIA_AESNI_PARALLEL_BLOCKS]; - struct crypt_priv crypt_ctx = { - .ctx = &ctx->crypt_ctx, - .fpu_enabled = false, - }; - struct xts_crypt_req req = { - .tbuf = buf, - .tbuflen = sizeof(buf), - - .tweak_ctx = &ctx->tweak_ctx, - .tweak_fn = XTS_TWEAK_CAST(camellia_enc_blk), - .crypt_ctx = &crypt_ctx, - .crypt_fn = encrypt_callback, - }; - int ret; - - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - ret = xts_crypt(desc, dst, src, nbytes, &req); - camellia_fpu_end(crypt_ctx.fpu_enabled); - return ret; + return glue_xts_crypt_128bit(&camellia_enc_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(camellia_enc_blk), + &ctx->tweak_ctx, &ctx->crypt_ctx); } static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[CAMELLIA_AESNI_PARALLEL_BLOCKS]; - struct crypt_priv crypt_ctx = { - .ctx = &ctx->crypt_ctx, - .fpu_enabled = false, - }; - struct xts_crypt_req req = { - .tbuf = buf, - .tbuflen = sizeof(buf), - .tweak_ctx = &ctx->tweak_ctx, - .tweak_fn = XTS_TWEAK_CAST(camellia_enc_blk), - .crypt_ctx = &crypt_ctx, - .crypt_fn = decrypt_callback, - }; - int ret; - - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - ret = xts_crypt(desc, dst, src, nbytes, &req); - camellia_fpu_end(crypt_ctx.fpu_enabled); - - return ret; + return glue_xts_crypt_128bit(&camellia_dec_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(camellia_enc_blk), + &ctx->tweak_ctx, &ctx->crypt_ctx); } static struct crypto_alg cmll_algs[10] = { { -- cgit v0.10.2 From c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Mon, 8 Apr 2013 21:51:16 +0300 Subject: crypto: aesni_intel - add more optimized XTS mode for x86-64 Add more optimized XTS code for aesni_intel in 64-bit mode, for smaller stack usage and boost for speed. tcrypt results, with Intel i5-2450M: 256-bit key enc dec 16B 0.98x 0.99x 64B 0.64x 0.63x 256B 1.29x 1.32x 1024B 1.54x 1.58x 8192B 1.57x 1.60x 512-bit key enc dec 16B 0.98x 0.99x 64B 0.60x 0.59x 256B 1.24x 1.25x 1024B 1.39x 1.42x 8192B 1.38x 1.42x I chose not to optimize smaller than block size of 256 bytes, since XTS is practically always used with data blocks of size 512 bytes. This is why performance is reduced in tcrypt for 64 byte long blocks. Cc: Huang Ying Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 04b7977..62fe22c 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -34,6 +34,10 @@ #ifdef __x86_64__ .data +.align 16 +.Lgf128mul_x_ble_mask: + .octa 0x00000000000000010000000000000087 + POLY: .octa 0xC2000000000000000000000000000001 TWOONE: .octa 0x00000001000000000000000000000001 @@ -105,6 +109,8 @@ enc: .octa 0x2 #define CTR %xmm11 #define INC %xmm12 +#define GF128MUL_MASK %xmm10 + #ifdef __x86_64__ #define AREG %rax #define KEYP %rdi @@ -2636,4 +2642,115 @@ ENTRY(aesni_ctr_enc) .Lctr_enc_just_ret: ret ENDPROC(aesni_ctr_enc) + +/* + * _aesni_gf128mul_x_ble: internal ABI + * Multiply in GF(2^128) for XTS IVs + * input: + * IV: current IV + * GF128MUL_MASK == mask with 0x87 and 0x01 + * output: + * IV: next IV + * changed: + * CTR: == temporary value + */ +#define _aesni_gf128mul_x_ble() \ + pshufd $0x13, IV, CTR; \ + paddq IV, IV; \ + psrad $31, CTR; \ + pand GF128MUL_MASK, CTR; \ + pxor CTR, IV; + +/* + * void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, + * bool enc, u8 *iv) + */ +ENTRY(aesni_xts_crypt8) + cmpb $0, %cl + movl $0, %ecx + movl $240, %r10d + leaq _aesni_enc4, %r11 + leaq _aesni_dec4, %rax + cmovel %r10d, %ecx + cmoveq %rax, %r11 + + movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK + movups (IVP), IV + + mov 480(KEYP), KLEN + addq %rcx, KEYP + + movdqa IV, STATE1 + pxor 0x00(INP), STATE1 + movdqu IV, 0x00(OUTP) + + _aesni_gf128mul_x_ble() + movdqa IV, STATE2 + pxor 0x10(INP), STATE2 + movdqu IV, 0x10(OUTP) + + _aesni_gf128mul_x_ble() + movdqa IV, STATE3 + pxor 0x20(INP), STATE3 + movdqu IV, 0x20(OUTP) + + _aesni_gf128mul_x_ble() + movdqa IV, STATE4 + pxor 0x30(INP), STATE4 + movdqu IV, 0x30(OUTP) + + call *%r11 + + pxor 0x00(OUTP), STATE1 + movdqu STATE1, 0x00(OUTP) + + _aesni_gf128mul_x_ble() + movdqa IV, STATE1 + pxor 0x40(INP), STATE1 + movdqu IV, 0x40(OUTP) + + pxor 0x10(OUTP), STATE2 + movdqu STATE2, 0x10(OUTP) + + _aesni_gf128mul_x_ble() + movdqa IV, STATE2 + pxor 0x50(INP), STATE2 + movdqu IV, 0x50(OUTP) + + pxor 0x20(OUTP), STATE3 + movdqu STATE3, 0x20(OUTP) + + _aesni_gf128mul_x_ble() + movdqa IV, STATE3 + pxor 0x60(INP), STATE3 + movdqu IV, 0x60(OUTP) + + pxor 0x30(OUTP), STATE4 + movdqu STATE4, 0x30(OUTP) + + _aesni_gf128mul_x_ble() + movdqa IV, STATE4 + pxor 0x70(INP), STATE4 + movdqu IV, 0x70(OUTP) + + _aesni_gf128mul_x_ble() + movups IV, (IVP) + + call *%r11 + + pxor 0x40(OUTP), STATE1 + movdqu STATE1, 0x40(OUTP) + + pxor 0x50(OUTP), STATE2 + movdqu STATE2, 0x50(OUTP) + + pxor 0x60(OUTP), STATE3 + movdqu STATE3, 0x60(OUTP) + + pxor 0x70(OUTP), STATE4 + movdqu STATE4, 0x70(OUTP) + + ret +ENDPROC(aesni_xts_crypt8) + #endif diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index a0795da..f80e668 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -39,6 +39,9 @@ #include #include #include +#ifdef CONFIG_X86_64 +#include +#endif #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) #define HAS_PCBC @@ -102,6 +105,9 @@ void crypto_fpu_exit(void); asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); +asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out, + const u8 *in, bool enc, u8 *iv); + /* asmlinkage void aesni_gcm_enc() * void *ctx, AES Key schedule. Starts on a 16 byte boundary. * u8 *out, Ciphertext output. Encrypt in-place is allowed. @@ -510,6 +516,78 @@ static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in) aesni_enc(ctx, out, in); } +#ifdef CONFIG_X86_64 + +static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc)); +} + +static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec)); +} + +static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv); +} + +static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv) +{ + aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv); +} + +static const struct common_glue_ctx aesni_enc_xts = { + .num_funcs = 2, + .fpu_blocks_limit = 1, + + .funcs = { { + .num_blocks = 8, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) } + } } +}; + +static const struct common_glue_ctx aesni_dec_xts = { + .num_funcs = 2, + .fpu_blocks_limit = 1, + + .funcs = { { + .num_blocks = 8, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) } + } } +}; + +static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + + return glue_xts_crypt_128bit(&aesni_enc_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(aesni_xts_tweak), + aes_ctx(ctx->raw_tweak_ctx), + aes_ctx(ctx->raw_crypt_ctx)); +} + +static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + + return glue_xts_crypt_128bit(&aesni_dec_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(aesni_xts_tweak), + aes_ctx(ctx->raw_tweak_ctx), + aes_ctx(ctx->raw_crypt_ctx)); +} + +#else + static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { @@ -560,6 +638,8 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return ret; } +#endif + #ifdef CONFIG_X86_64 static int rfc4106_init(struct crypto_tfm *tfm) { diff --git a/crypto/Kconfig b/crypto/Kconfig index c1142f3..808ac37 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -678,6 +678,7 @@ config CRYPTO_AES_NI_INTEL select CRYPTO_CRYPTD select CRYPTO_ABLK_HELPER_X86 select CRYPTO_ALGAPI + select CRYPTO_GLUE_HELPER if 64BIT select CRYPTO_LRW select CRYPTO_XTS help -- cgit v0.10.2 From 7643a11a35c178c01083829d1a5b43e22b688751 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Wed, 10 Apr 2013 18:39:20 +0300 Subject: crypto: aesni_intel - fix Kconfig problem with CRYPTO_GLUE_HELPER_X86 The Kconfig setting for glue helper module is CRYPTO_GLUE_HELPER_X86, but recent change for aesni_intel used CRYPTO_GLUE_HELPER instead. Patch corrects this issue. Cc: kbuild-all@01.org Reported-by: kbuild test robot Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/crypto/Kconfig b/crypto/Kconfig index 808ac37..0e7a237 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -678,7 +678,7 @@ config CRYPTO_AES_NI_INTEL select CRYPTO_CRYPTD select CRYPTO_ABLK_HELPER_X86 select CRYPTO_ALGAPI - select CRYPTO_GLUE_HELPER if 64BIT + select CRYPTO_GLUE_HELPER_X86 if 64BIT select CRYPTO_LRW select CRYPTO_XTS help -- cgit v0.10.2 From 23a836e87d6cd85dc456dde103dfc69c1743f95a Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sat, 13 Apr 2013 13:46:35 +0300 Subject: crypto: testmgr - extend camellia test-vectors for camellia-aesni/avx2 Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/crypto/testmgr.h b/crypto/testmgr.h index d503660..dc2c054 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -20997,8 +20997,72 @@ static struct cipher_testvec camellia_enc_tv_template[] = { "\x86\x1D\xB4\x28\xBF\x56\xED\x61" "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3" "\x6A\x01\x75\x0C\xA3\x17\xAE\x45" - "\xDC\x50\xE7\x7E\x15\x89\x20\xB7", - .ilen = 496, + "\xDC\x50\xE7\x7E\x15\x89\x20\xB7" + "\x2B\xC2\x59\xF0\x64\xFB\x92\x06" + "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78" + "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA" + "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C" + "\xF3\x67\xFE\x95\x09\xA0\x37\xCE" + "\x42\xD9\x70\x07\x7B\x12\xA9\x1D" + "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F" + "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01" + "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73" + "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5" + "\x59\xF0\x87\x1E\x92\x29\xC0\x34" + "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6" + "\x3D\xD4\x48\xDF\x76\x0D\x81\x18" + "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A" + "\x21\x95\x2C\xC3\x37\xCE\x65\xFC" + "\x70\x07\x9E\x12\xA9\x40\xD7\x4B" + "\xE2\x79\x10\x84\x1B\xB2\x26\xBD" + "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F" + "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1" + "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13" + "\x87\x1E\xB5\x29\xC0\x57\xEE\x62" + "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4" + "\x6B\x02\x76\x0D\xA4\x18\xAF\x46" + "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8" + "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07" + "\x9E\x35\xCC\x40\xD7\x6E\x05\x79" + "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB" + "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D" + "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF" + "\x43\xDA\x71\x08\x7C\x13\xAA\x1E" + "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90" + "\x27\xBE\x32\xC9\x60\xF7\x6B\x02" + "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74" + "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6" + "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35" + "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7" + "\x3E\xD5\x49\xE0\x77\x0E\x82\x19" + "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B" + "\x22\x96\x2D\xC4\x38\xCF\x66\xFD" + "\x71\x08\x9F\x13\xAA\x41\xD8\x4C" + "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE" + "\x55\xEC\x60\xF7\x8E\x02\x99\x30" + "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2" + "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14" + "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63" + "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5" + "\x6C\x03\x77\x0E\xA5\x19\xB0\x47" + "\xDE\x52\xE9\x80\x17\x8B\x22\xB9" + "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08" + "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A" + "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC" + "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E" + "\xF5\x69\x00\x97\x0B\xA2\x39\xD0" + "\x44\xDB\x72\x09\x7D\x14\xAB\x1F" + "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91" + "\x28\xBF\x33\xCA\x61\xF8\x6C\x03" + "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75" + "\x0C\x80\x17\xAE\x22\xB9\x50\xE7" + "\x5B\xF2\x89\x20\x94\x2B\xC2\x36" + "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8" + "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A" + "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C" + "\x00\x97\x2E\xC5\x39\xD0\x67\xFE" + "\x72\x09\xA0\x14\xAB\x42\xD9\x4D", + .ilen = 1008, .result = "\xED\xCD\xDB\xB8\x68\xCE\xBD\xEA" "\x9D\x9D\xCD\x9F\x4F\xFC\x4D\xB7" "\xA5\xFF\x6F\x43\x0F\xBA\x32\x04" @@ -21060,11 +21124,75 @@ static struct cipher_testvec camellia_enc_tv_template[] = { "\x2C\x35\x1B\x38\x85\x7D\xE8\xF3" "\x87\x4F\xDA\xD8\x5F\xFC\xB6\x44" "\xD0\xE3\x9B\x8B\xBF\xD6\xB8\xC4" - "\x73\xAE\x1D\x8B\x5B\x74\x8B\xCB", - .rlen = 496, + "\x73\xAE\x1D\x8B\x5B\x74\x8B\xCB" + "\xA4\xAD\xCF\x5D\xD4\x58\xC9\xCD" + "\xF7\x90\x68\xCF\xC9\x11\x52\x3E" + "\xE8\xA1\xA3\x78\x8B\xD0\xAC\x0A" + "\xD4\xC9\xA3\xA5\x55\x30\xC8\x3E" + "\xED\x28\x39\xE9\x63\xED\x41\x70" + "\x51\xE3\xC4\xA0\xFC\xD5\x43\xCB" + "\x4D\x65\xC8\xFD\x3A\x91\x8F\x60" + "\x8A\xA6\x6D\x9D\x3E\x01\x23\x4B" + "\x50\x47\xC9\xDC\x9B\xDE\x37\xC5" + "\xBF\x67\xB1\x6B\x78\x38\xD5\x7E" + "\xB6\xFF\x67\x83\x3B\x6E\xBE\x23" + "\x45\xFA\x1D\x69\x44\xFD\xC6\xB9" + "\xD0\x4A\x92\xD1\xBE\xF6\x4A\xB7" + "\xCA\xA8\xA2\x9E\x13\x87\x57\x92" + "\x64\x7C\x85\x0B\xB3\x29\x37\xD8" + "\xE6\xAA\xAF\xC4\x03\x67\xA3\xBF" + "\x2E\x45\x83\xB6\xD8\x54\x00\x89" + "\xF6\xBC\x3A\x7A\x88\x58\x51\xED" + "\xF4\x4E\x01\xA5\xC3\x2E\xD9\x42" + "\xBD\x6E\x0D\x0B\x21\xB0\x1A\xCC" + "\xA4\xD3\x3F\xDC\x9B\x81\xD8\xF1" + "\xEA\x7A\x6A\xB7\x07\xC9\x6D\x91" + "\x6D\x3A\xF5\x5F\xA6\xFF\x87\x1E" + "\x3F\xDD\xC0\x72\xEA\xAC\x08\x15" + "\x21\xE6\xC6\xB6\x0D\xD8\x51\x86" + "\x2A\x03\x73\xF7\x29\xD4\xC4\xE4" + "\x7F\x95\x10\xF7\xAB\x3F\x92\x23" + "\xD3\xCE\x9C\x2E\x46\x3B\x63\x43" + "\xBB\xC2\x82\x7A\x83\xD5\x55\xE2" + "\xE7\x9B\x2F\x92\xAF\xFD\x81\x56" + "\x79\xFD\x3E\xF9\x46\xE0\x25\xD4" + "\x38\xDE\xBC\x2C\xC4\x7A\x2A\x8F" + "\x94\x4F\xD0\xAD\x9B\x37\x18\xD4" + "\x0E\x4D\x0F\x02\x3A\xDC\x5A\xA2" + "\x39\x25\x55\x20\x5A\xA6\x02\x9F" + "\xE6\x77\x21\x77\xE5\x4B\x7B\x0B" + "\x30\xF8\x5F\x33\x0F\x49\xCD\xFF" + "\xF2\xE4\x35\xF9\xF0\x63\xC3\x7E" + "\xF1\xA6\x73\xB4\xDF\xE7\xBB\x78" + "\xFF\x21\xA9\xF3\xF3\xCF\x5D\xBA" + "\xED\x87\x98\xAC\xFE\x48\x97\x6D" + "\xA6\x7F\x69\x31\xB1\xC4\xFF\x14" + "\xC6\x76\xD4\x10\xDD\xF6\x49\x2C" + "\x9C\xC8\x6D\x76\xC0\x8F\x5F\x55" + "\x2F\x3C\x8A\x30\xAA\xC3\x16\x55" + "\xC6\xFC\x8D\x8B\xB9\xE5\x80\x6C" + "\xC8\x7E\xBD\x65\x58\x36\xD5\xBC" + "\xF0\x33\x52\x29\x70\xF9\x5C\xE9" + "\xAC\x1F\xB5\x73\x56\x66\x54\xAF" + "\x1B\x8F\x7D\xED\xAB\x03\xCE\xE3" + "\xAE\x47\xB6\x69\x86\xE9\x01\x31" + "\x83\x18\x3D\xF4\x74\x7B\xF9\x42" + "\x4C\xFD\x75\x4A\x6D\xF0\x03\xA6" + "\x2B\x20\x63\xDA\x49\x65\x5E\x8B" + "\xC0\x19\xE3\x8D\xD9\xF3\xB0\x34" + "\xD3\x52\xFC\x68\x00\x43\x1B\x37" + "\x31\x93\x51\x1C\x63\x97\x70\xB0" + "\x99\x78\x83\x13\xFD\xCF\x53\x81" + "\x36\x46\xB5\x42\x52\x2F\x32\xEB" + "\x4A\x3D\xF1\x8F\x1C\x54\x2E\xFC" + "\x41\x75\x5A\x8C\x8E\x6F\xE7\x1A" + "\xAE\xEF\x3E\x82\x12\x0B\x74\x72" + "\xF8\xB2\xAA\x7A\xD6\xFF\xFA\x55" + "\x33\x1A\xBB\xD3\xA2\x7E\x97\x66", + .rlen = 1008, .also_non_np = 1, .np = 2, - .tap = { 496 - 16, 16 }, + .tap = { 1008 - 16, 16 }, }, }; @@ -21169,8 +21297,72 @@ static struct cipher_testvec camellia_dec_tv_template[] = { "\x2C\x35\x1B\x38\x85\x7D\xE8\xF3" "\x87\x4F\xDA\xD8\x5F\xFC\xB6\x44" "\xD0\xE3\x9B\x8B\xBF\xD6\xB8\xC4" - "\x73\xAE\x1D\x8B\x5B\x74\x8B\xCB", - .ilen = 496, + "\x73\xAE\x1D\x8B\x5B\x74\x8B\xCB" + "\xA4\xAD\xCF\x5D\xD4\x58\xC9\xCD" + "\xF7\x90\x68\xCF\xC9\x11\x52\x3E" + "\xE8\xA1\xA3\x78\x8B\xD0\xAC\x0A" + "\xD4\xC9\xA3\xA5\x55\x30\xC8\x3E" + "\xED\x28\x39\xE9\x63\xED\x41\x70" + "\x51\xE3\xC4\xA0\xFC\xD5\x43\xCB" + "\x4D\x65\xC8\xFD\x3A\x91\x8F\x60" + "\x8A\xA6\x6D\x9D\x3E\x01\x23\x4B" + "\x50\x47\xC9\xDC\x9B\xDE\x37\xC5" + "\xBF\x67\xB1\x6B\x78\x38\xD5\x7E" + "\xB6\xFF\x67\x83\x3B\x6E\xBE\x23" + "\x45\xFA\x1D\x69\x44\xFD\xC6\xB9" + "\xD0\x4A\x92\xD1\xBE\xF6\x4A\xB7" + "\xCA\xA8\xA2\x9E\x13\x87\x57\x92" + "\x64\x7C\x85\x0B\xB3\x29\x37\xD8" + "\xE6\xAA\xAF\xC4\x03\x67\xA3\xBF" + "\x2E\x45\x83\xB6\xD8\x54\x00\x89" + "\xF6\xBC\x3A\x7A\x88\x58\x51\xED" + "\xF4\x4E\x01\xA5\xC3\x2E\xD9\x42" + "\xBD\x6E\x0D\x0B\x21\xB0\x1A\xCC" + "\xA4\xD3\x3F\xDC\x9B\x81\xD8\xF1" + "\xEA\x7A\x6A\xB7\x07\xC9\x6D\x91" + "\x6D\x3A\xF5\x5F\xA6\xFF\x87\x1E" + "\x3F\xDD\xC0\x72\xEA\xAC\x08\x15" + "\x21\xE6\xC6\xB6\x0D\xD8\x51\x86" + "\x2A\x03\x73\xF7\x29\xD4\xC4\xE4" + "\x7F\x95\x10\xF7\xAB\x3F\x92\x23" + "\xD3\xCE\x9C\x2E\x46\x3B\x63\x43" + "\xBB\xC2\x82\x7A\x83\xD5\x55\xE2" + "\xE7\x9B\x2F\x92\xAF\xFD\x81\x56" + "\x79\xFD\x3E\xF9\x46\xE0\x25\xD4" + "\x38\xDE\xBC\x2C\xC4\x7A\x2A\x8F" + "\x94\x4F\xD0\xAD\x9B\x37\x18\xD4" + "\x0E\x4D\x0F\x02\x3A\xDC\x5A\xA2" + "\x39\x25\x55\x20\x5A\xA6\x02\x9F" + "\xE6\x77\x21\x77\xE5\x4B\x7B\x0B" + "\x30\xF8\x5F\x33\x0F\x49\xCD\xFF" + "\xF2\xE4\x35\xF9\xF0\x63\xC3\x7E" + "\xF1\xA6\x73\xB4\xDF\xE7\xBB\x78" + "\xFF\x21\xA9\xF3\xF3\xCF\x5D\xBA" + "\xED\x87\x98\xAC\xFE\x48\x97\x6D" + "\xA6\x7F\x69\x31\xB1\xC4\xFF\x14" + "\xC6\x76\xD4\x10\xDD\xF6\x49\x2C" + "\x9C\xC8\x6D\x76\xC0\x8F\x5F\x55" + "\x2F\x3C\x8A\x30\xAA\xC3\x16\x55" + "\xC6\xFC\x8D\x8B\xB9\xE5\x80\x6C" + "\xC8\x7E\xBD\x65\x58\x36\xD5\xBC" + "\xF0\x33\x52\x29\x70\xF9\x5C\xE9" + "\xAC\x1F\xB5\x73\x56\x66\x54\xAF" + "\x1B\x8F\x7D\xED\xAB\x03\xCE\xE3" + "\xAE\x47\xB6\x69\x86\xE9\x01\x31" + "\x83\x18\x3D\xF4\x74\x7B\xF9\x42" + "\x4C\xFD\x75\x4A\x6D\xF0\x03\xA6" + "\x2B\x20\x63\xDA\x49\x65\x5E\x8B" + "\xC0\x19\xE3\x8D\xD9\xF3\xB0\x34" + "\xD3\x52\xFC\x68\x00\x43\x1B\x37" + "\x31\x93\x51\x1C\x63\x97\x70\xB0" + "\x99\x78\x83\x13\xFD\xCF\x53\x81" + "\x36\x46\xB5\x42\x52\x2F\x32\xEB" + "\x4A\x3D\xF1\x8F\x1C\x54\x2E\xFC" + "\x41\x75\x5A\x8C\x8E\x6F\xE7\x1A" + "\xAE\xEF\x3E\x82\x12\x0B\x74\x72" + "\xF8\xB2\xAA\x7A\xD6\xFF\xFA\x55" + "\x33\x1A\xBB\xD3\xA2\x7E\x97\x66", + .ilen = 1008, .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31" "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3" "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15" @@ -21232,11 +21424,75 @@ static struct cipher_testvec camellia_dec_tv_template[] = { "\x86\x1D\xB4\x28\xBF\x56\xED\x61" "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3" "\x6A\x01\x75\x0C\xA3\x17\xAE\x45" - "\xDC\x50\xE7\x7E\x15\x89\x20\xB7", - .rlen = 496, + "\xDC\x50\xE7\x7E\x15\x89\x20\xB7" + "\x2B\xC2\x59\xF0\x64\xFB\x92\x06" + "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78" + "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA" + "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C" + "\xF3\x67\xFE\x95\x09\xA0\x37\xCE" + "\x42\xD9\x70\x07\x7B\x12\xA9\x1D" + "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F" + "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01" + "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73" + "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5" + "\x59\xF0\x87\x1E\x92\x29\xC0\x34" + "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6" + "\x3D\xD4\x48\xDF\x76\x0D\x81\x18" + "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A" + "\x21\x95\x2C\xC3\x37\xCE\x65\xFC" + "\x70\x07\x9E\x12\xA9\x40\xD7\x4B" + "\xE2\x79\x10\x84\x1B\xB2\x26\xBD" + "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F" + "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1" + "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13" + "\x87\x1E\xB5\x29\xC0\x57\xEE\x62" + "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4" + "\x6B\x02\x76\x0D\xA4\x18\xAF\x46" + "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8" + "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07" + "\x9E\x35\xCC\x40\xD7\x6E\x05\x79" + "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB" + "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D" + "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF" + "\x43\xDA\x71\x08\x7C\x13\xAA\x1E" + "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90" + "\x27\xBE\x32\xC9\x60\xF7\x6B\x02" + "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74" + "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6" + "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35" + "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7" + "\x3E\xD5\x49\xE0\x77\x0E\x82\x19" + "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B" + "\x22\x96\x2D\xC4\x38\xCF\x66\xFD" + "\x71\x08\x9F\x13\xAA\x41\xD8\x4C" + "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE" + "\x55\xEC\x60\xF7\x8E\x02\x99\x30" + "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2" + "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14" + "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63" + "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5" + "\x6C\x03\x77\x0E\xA5\x19\xB0\x47" + "\xDE\x52\xE9\x80\x17\x8B\x22\xB9" + "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08" + "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A" + "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC" + "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E" + "\xF5\x69\x00\x97\x0B\xA2\x39\xD0" + "\x44\xDB\x72\x09\x7D\x14\xAB\x1F" + "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91" + "\x28\xBF\x33\xCA\x61\xF8\x6C\x03" + "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75" + "\x0C\x80\x17\xAE\x22\xB9\x50\xE7" + "\x5B\xF2\x89\x20\x94\x2B\xC2\x36" + "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8" + "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A" + "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C" + "\x00\x97\x2E\xC5\x39\xD0\x67\xFE" + "\x72\x09\xA0\x14\xAB\x42\xD9\x4D", + .rlen = 1008, .also_non_np = 1, .np = 2, - .tap = { 496 - 16, 16 }, + .tap = { 1008 - 16, 16 }, }, }; @@ -21337,8 +21593,72 @@ static struct cipher_testvec camellia_cbc_enc_tv_template[] = { "\x86\x1D\xB4\x28\xBF\x56\xED\x61" "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3" "\x6A\x01\x75\x0C\xA3\x17\xAE\x45" - "\xDC\x50\xE7\x7E\x15\x89\x20\xB7", - .ilen = 496, + "\xDC\x50\xE7\x7E\x15\x89\x20\xB7" + "\x2B\xC2\x59\xF0\x64\xFB\x92\x06" + "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78" + "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA" + "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C" + "\xF3\x67\xFE\x95\x09\xA0\x37\xCE" + "\x42\xD9\x70\x07\x7B\x12\xA9\x1D" + "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F" + "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01" + "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73" + "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5" + "\x59\xF0\x87\x1E\x92\x29\xC0\x34" + "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6" + "\x3D\xD4\x48\xDF\x76\x0D\x81\x18" + "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A" + "\x21\x95\x2C\xC3\x37\xCE\x65\xFC" + "\x70\x07\x9E\x12\xA9\x40\xD7\x4B" + "\xE2\x79\x10\x84\x1B\xB2\x26\xBD" + "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F" + "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1" + "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13" + "\x87\x1E\xB5\x29\xC0\x57\xEE\x62" + "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4" + "\x6B\x02\x76\x0D\xA4\x18\xAF\x46" + "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8" + "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07" + "\x9E\x35\xCC\x40\xD7\x6E\x05\x79" + "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB" + "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D" + "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF" + "\x43\xDA\x71\x08\x7C\x13\xAA\x1E" + "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90" + "\x27\xBE\x32\xC9\x60\xF7\x6B\x02" + "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74" + "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6" + "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35" + "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7" + "\x3E\xD5\x49\xE0\x77\x0E\x82\x19" + "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B" + "\x22\x96\x2D\xC4\x38\xCF\x66\xFD" + "\x71\x08\x9F\x13\xAA\x41\xD8\x4C" + "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE" + "\x55\xEC\x60\xF7\x8E\x02\x99\x30" + "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2" + "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14" + "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63" + "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5" + "\x6C\x03\x77\x0E\xA5\x19\xB0\x47" + "\xDE\x52\xE9\x80\x17\x8B\x22\xB9" + "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08" + "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A" + "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC" + "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E" + "\xF5\x69\x00\x97\x0B\xA2\x39\xD0" + "\x44\xDB\x72\x09\x7D\x14\xAB\x1F" + "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91" + "\x28\xBF\x33\xCA\x61\xF8\x6C\x03" + "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75" + "\x0C\x80\x17\xAE\x22\xB9\x50\xE7" + "\x5B\xF2\x89\x20\x94\x2B\xC2\x36" + "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8" + "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A" + "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C" + "\x00\x97\x2E\xC5\x39\xD0\x67\xFE" + "\x72\x09\xA0\x14\xAB\x42\xD9\x4D", + .ilen = 1008, .result = "\xCD\x3E\x2A\x3B\x3E\x94\xC5\x77" "\xBA\xBB\x5B\xB1\xDE\x7B\xA4\x40" "\x88\x39\xE3\xFD\x94\x4B\x25\x58" @@ -21400,11 +21720,75 @@ static struct cipher_testvec camellia_cbc_enc_tv_template[] = { "\x2D\x1A\x68\xFE\xEC\x92\x94\xDA" "\x94\x2A\x6F\xD6\xFE\xE5\x76\x97" "\xF4\x6E\xEE\xCB\x2B\x95\x4E\x36" - "\x5F\x74\x8C\x86\x5B\x71\xD0\x20", - .rlen = 496, + "\x5F\x74\x8C\x86\x5B\x71\xD0\x20" + "\x78\x1A\x7F\x18\x8C\xD9\xCD\xF5" + "\x21\x41\x56\x72\x13\xE1\x86\x07" + "\x07\x26\xF3\x4F\x7B\xEA\xB5\x18" + "\xFE\x94\x2D\x9F\xE0\x72\x18\x65" + "\xB2\xA5\x63\x48\xB4\x13\x22\xF7" + "\x25\xF1\x80\xA8\x7F\x54\x86\x7B" + "\x39\xAE\x95\x0C\x09\x32\x22\x2D" + "\x4D\x73\x39\x0C\x09\x2C\x7C\x10" + "\xD0\x4B\x53\xF6\x90\xC5\x99\x2F" + "\x15\xE1\x7F\xC6\xC5\x7A\x52\x14" + "\x65\xEE\x93\x54\xD0\x66\x15\x3C" + "\x4C\x68\xFD\x64\x0F\xF9\x10\x39" + "\x46\x7A\xDD\x97\x20\xEE\xC7\xD2" + "\x98\x4A\xB6\xE6\xF5\xA8\x1F\x4F" + "\xDB\xAB\x6D\xD5\x9B\x34\x16\x97" + "\x2F\x64\xE5\x37\xEF\x0E\xA1\xE9" + "\xBE\x31\x31\x96\x8B\x40\x18\x75" + "\x11\x75\x14\x32\xA5\x2D\x1B\x6B" + "\xDB\x59\xEB\xFA\x3D\x8E\x7C\xC4" + "\xDE\x68\xC8\x9F\xC9\x99\xE3\xC6" + "\x71\xB0\x12\x57\x89\x0D\xC0\x2B" + "\x9F\x12\x6A\x04\x67\xF1\x95\x31" + "\x59\xFD\x84\x95\x2C\x9C\x5B\xEC" + "\x09\xB0\x43\x96\x4A\x64\x80\x40" + "\xB9\x72\x19\xDD\x70\x42\xFA\xB1" + "\x4A\x2C\x0C\x0A\x60\x6E\xE3\x7C" + "\x37\x5A\xBE\xA4\x62\xCF\x29\xAB" + "\x7F\x4D\xA6\xB3\xE2\xB6\x64\xC6" + "\x33\x0B\xF3\xD5\x01\x38\x74\xA4" + "\x67\x1E\x75\x68\xC3\xAD\x76\xE9" + "\xE9\xBC\xF0\xEB\xD8\xFD\x31\x8A" + "\x5F\xC9\x18\x94\x4B\x86\x66\xFC" + "\xBD\x0B\x3D\xB3\x9F\xFA\x1F\xD9" + "\x78\xC4\xE3\x24\x1C\x67\xA2\xF8" + "\x43\xBC\x76\x75\xBF\x6C\x05\xB3" + "\x32\xE8\x7C\x80\xDB\xC7\xB6\x61" + "\x1A\x3E\x2B\xA7\x25\xED\x8F\xA0" + "\x00\x4B\xF8\x90\xCA\xD8\xFB\x12" + "\xAC\x1F\x18\xE9\xD2\x5E\xA2\x8E" + "\xE4\x84\x6B\x9D\xEB\x1E\x6B\xA3" + "\x7B\xDC\xCE\x15\x97\x27\xB2\x65" + "\xBC\x0E\x47\xAB\x55\x13\x53\xAB" + "\x0E\x34\x55\x02\x5F\x27\xC5\x89" + "\xDF\xC5\x70\xC4\xDD\x76\x82\xEE" + "\x68\xA6\x09\xB0\xE5\x5E\xF1\x0C" + "\xE3\xF3\x09\x9B\xFE\x65\x4B\xB8" + "\x30\xEC\xD5\x7C\x6A\xEC\x1D\xD2" + "\x93\xB7\xA1\x1A\x02\xD4\xC0\xD6" + "\x8D\x4D\x83\x9A\xED\x29\x4E\x14" + "\x86\xD5\x3C\x1A\xD5\xB9\x0A\x6A" + "\x72\x22\xD5\x92\x38\xF1\xA1\x86" + "\xB2\x41\x51\xCA\x4E\xAB\x8F\xD3" + "\x80\x56\xC3\xD7\x65\xE1\xB3\x86" + "\xCB\xCE\x98\xA1\xD4\x59\x1C\x06" + "\x01\xED\xF8\x29\x91\x19\x5C\x9A" + "\xEE\x28\x1B\x48\xD7\x32\xEF\x9F" + "\x6C\x2B\x66\x4E\x78\xD5\x8B\x72" + "\x80\xE7\x29\xDC\x23\x55\x98\x54" + "\xB1\xFF\x3E\x95\x56\xA8\x78\x78" + "\xEF\xC4\xA5\x11\x2D\x2B\xD8\x93" + "\x30\x6E\x7E\x51\xBB\x42\x5F\x03" + "\x43\x94\x23\x7E\xEE\xF0\xA5\x79" + "\x55\x01\xD4\x58\xB2\xF2\x85\x49" + "\x70\xC5\xB9\x0B\x3B\x7A\x6E\x6C", + .rlen = 1008, .also_non_np = 1, .np = 2, - .tap = { 496 - 16, 16 }, + .tap = { 1008 - 16, 16 }, }, }; @@ -21505,8 +21889,72 @@ static struct cipher_testvec camellia_cbc_dec_tv_template[] = { "\x2D\x1A\x68\xFE\xEC\x92\x94\xDA" "\x94\x2A\x6F\xD6\xFE\xE5\x76\x97" "\xF4\x6E\xEE\xCB\x2B\x95\x4E\x36" - "\x5F\x74\x8C\x86\x5B\x71\xD0\x20", - .ilen = 496, + "\x5F\x74\x8C\x86\x5B\x71\xD0\x20" + "\x78\x1A\x7F\x18\x8C\xD9\xCD\xF5" + "\x21\x41\x56\x72\x13\xE1\x86\x07" + "\x07\x26\xF3\x4F\x7B\xEA\xB5\x18" + "\xFE\x94\x2D\x9F\xE0\x72\x18\x65" + "\xB2\xA5\x63\x48\xB4\x13\x22\xF7" + "\x25\xF1\x80\xA8\x7F\x54\x86\x7B" + "\x39\xAE\x95\x0C\x09\x32\x22\x2D" + "\x4D\x73\x39\x0C\x09\x2C\x7C\x10" + "\xD0\x4B\x53\xF6\x90\xC5\x99\x2F" + "\x15\xE1\x7F\xC6\xC5\x7A\x52\x14" + "\x65\xEE\x93\x54\xD0\x66\x15\x3C" + "\x4C\x68\xFD\x64\x0F\xF9\x10\x39" + "\x46\x7A\xDD\x97\x20\xEE\xC7\xD2" + "\x98\x4A\xB6\xE6\xF5\xA8\x1F\x4F" + "\xDB\xAB\x6D\xD5\x9B\x34\x16\x97" + "\x2F\x64\xE5\x37\xEF\x0E\xA1\xE9" + "\xBE\x31\x31\x96\x8B\x40\x18\x75" + "\x11\x75\x14\x32\xA5\x2D\x1B\x6B" + "\xDB\x59\xEB\xFA\x3D\x8E\x7C\xC4" + "\xDE\x68\xC8\x9F\xC9\x99\xE3\xC6" + "\x71\xB0\x12\x57\x89\x0D\xC0\x2B" + "\x9F\x12\x6A\x04\x67\xF1\x95\x31" + "\x59\xFD\x84\x95\x2C\x9C\x5B\xEC" + "\x09\xB0\x43\x96\x4A\x64\x80\x40" + "\xB9\x72\x19\xDD\x70\x42\xFA\xB1" + "\x4A\x2C\x0C\x0A\x60\x6E\xE3\x7C" + "\x37\x5A\xBE\xA4\x62\xCF\x29\xAB" + "\x7F\x4D\xA6\xB3\xE2\xB6\x64\xC6" + "\x33\x0B\xF3\xD5\x01\x38\x74\xA4" + "\x67\x1E\x75\x68\xC3\xAD\x76\xE9" + "\xE9\xBC\xF0\xEB\xD8\xFD\x31\x8A" + "\x5F\xC9\x18\x94\x4B\x86\x66\xFC" + "\xBD\x0B\x3D\xB3\x9F\xFA\x1F\xD9" + "\x78\xC4\xE3\x24\x1C\x67\xA2\xF8" + "\x43\xBC\x76\x75\xBF\x6C\x05\xB3" + "\x32\xE8\x7C\x80\xDB\xC7\xB6\x61" + "\x1A\x3E\x2B\xA7\x25\xED\x8F\xA0" + "\x00\x4B\xF8\x90\xCA\xD8\xFB\x12" + "\xAC\x1F\x18\xE9\xD2\x5E\xA2\x8E" + "\xE4\x84\x6B\x9D\xEB\x1E\x6B\xA3" + "\x7B\xDC\xCE\x15\x97\x27\xB2\x65" + "\xBC\x0E\x47\xAB\x55\x13\x53\xAB" + "\x0E\x34\x55\x02\x5F\x27\xC5\x89" + "\xDF\xC5\x70\xC4\xDD\x76\x82\xEE" + "\x68\xA6\x09\xB0\xE5\x5E\xF1\x0C" + "\xE3\xF3\x09\x9B\xFE\x65\x4B\xB8" + "\x30\xEC\xD5\x7C\x6A\xEC\x1D\xD2" + "\x93\xB7\xA1\x1A\x02\xD4\xC0\xD6" + "\x8D\x4D\x83\x9A\xED\x29\x4E\x14" + "\x86\xD5\x3C\x1A\xD5\xB9\x0A\x6A" + "\x72\x22\xD5\x92\x38\xF1\xA1\x86" + "\xB2\x41\x51\xCA\x4E\xAB\x8F\xD3" + "\x80\x56\xC3\xD7\x65\xE1\xB3\x86" + "\xCB\xCE\x98\xA1\xD4\x59\x1C\x06" + "\x01\xED\xF8\x29\x91\x19\x5C\x9A" + "\xEE\x28\x1B\x48\xD7\x32\xEF\x9F" + "\x6C\x2B\x66\x4E\x78\xD5\x8B\x72" + "\x80\xE7\x29\xDC\x23\x55\x98\x54" + "\xB1\xFF\x3E\x95\x56\xA8\x78\x78" + "\xEF\xC4\xA5\x11\x2D\x2B\xD8\x93" + "\x30\x6E\x7E\x51\xBB\x42\x5F\x03" + "\x43\x94\x23\x7E\xEE\xF0\xA5\x79" + "\x55\x01\xD4\x58\xB2\xF2\x85\x49" + "\x70\xC5\xB9\x0B\x3B\x7A\x6E\x6C", + .ilen = 1008, .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31" "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3" "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15" @@ -21568,11 +22016,75 @@ static struct cipher_testvec camellia_cbc_dec_tv_template[] = { "\x86\x1D\xB4\x28\xBF\x56\xED\x61" "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3" "\x6A\x01\x75\x0C\xA3\x17\xAE\x45" - "\xDC\x50\xE7\x7E\x15\x89\x20\xB7", - .rlen = 496, + "\xDC\x50\xE7\x7E\x15\x89\x20\xB7" + "\x2B\xC2\x59\xF0\x64\xFB\x92\x06" + "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78" + "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA" + "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C" + "\xF3\x67\xFE\x95\x09\xA0\x37\xCE" + "\x42\xD9\x70\x07\x7B\x12\xA9\x1D" + "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F" + "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01" + "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73" + "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5" + "\x59\xF0\x87\x1E\x92\x29\xC0\x34" + "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6" + "\x3D\xD4\x48\xDF\x76\x0D\x81\x18" + "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A" + "\x21\x95\x2C\xC3\x37\xCE\x65\xFC" + "\x70\x07\x9E\x12\xA9\x40\xD7\x4B" + "\xE2\x79\x10\x84\x1B\xB2\x26\xBD" + "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F" + "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1" + "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13" + "\x87\x1E\xB5\x29\xC0\x57\xEE\x62" + "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4" + "\x6B\x02\x76\x0D\xA4\x18\xAF\x46" + "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8" + "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07" + "\x9E\x35\xCC\x40\xD7\x6E\x05\x79" + "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB" + "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D" + "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF" + "\x43\xDA\x71\x08\x7C\x13\xAA\x1E" + "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90" + "\x27\xBE\x32\xC9\x60\xF7\x6B\x02" + "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74" + "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6" + "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35" + "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7" + "\x3E\xD5\x49\xE0\x77\x0E\x82\x19" + "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B" + "\x22\x96\x2D\xC4\x38\xCF\x66\xFD" + "\x71\x08\x9F\x13\xAA\x41\xD8\x4C" + "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE" + "\x55\xEC\x60\xF7\x8E\x02\x99\x30" + "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2" + "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14" + "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63" + "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5" + "\x6C\x03\x77\x0E\xA5\x19\xB0\x47" + "\xDE\x52\xE9\x80\x17\x8B\x22\xB9" + "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08" + "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A" + "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC" + "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E" + "\xF5\x69\x00\x97\x0B\xA2\x39\xD0" + "\x44\xDB\x72\x09\x7D\x14\xAB\x1F" + "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91" + "\x28\xBF\x33\xCA\x61\xF8\x6C\x03" + "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75" + "\x0C\x80\x17\xAE\x22\xB9\x50\xE7" + "\x5B\xF2\x89\x20\x94\x2B\xC2\x36" + "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8" + "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A" + "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C" + "\x00\x97\x2E\xC5\x39\xD0\x67\xFE" + "\x72\x09\xA0\x14\xAB\x42\xD9\x4D", + .rlen = 1008, .also_non_np = 1, .np = 2, - .tap = { 496 - 16, 16 }, + .tap = { 1008 - 16, 16 }, }, }; @@ -21781,8 +22293,72 @@ static struct cipher_testvec camellia_ctr_enc_tv_template[] = { "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3" "\x6A\x01\x75\x0C\xA3\x17\xAE\x45" "\xDC\x50\xE7\x7E\x15\x89\x20\xB7" - "\x2B\xC2\x59", - .ilen = 499, + "\x2B\xC2\x59\xF0\x64\xFB\x92\x06" + "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78" + "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA" + "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C" + "\xF3\x67\xFE\x95\x09\xA0\x37\xCE" + "\x42\xD9\x70\x07\x7B\x12\xA9\x1D" + "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F" + "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01" + "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73" + "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5" + "\x59\xF0\x87\x1E\x92\x29\xC0\x34" + "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6" + "\x3D\xD4\x48\xDF\x76\x0D\x81\x18" + "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A" + "\x21\x95\x2C\xC3\x37\xCE\x65\xFC" + "\x70\x07\x9E\x12\xA9\x40\xD7\x4B" + "\xE2\x79\x10\x84\x1B\xB2\x26\xBD" + "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F" + "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1" + "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13" + "\x87\x1E\xB5\x29\xC0\x57\xEE\x62" + "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4" + "\x6B\x02\x76\x0D\xA4\x18\xAF\x46" + "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8" + "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07" + "\x9E\x35\xCC\x40\xD7\x6E\x05\x79" + "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB" + "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D" + "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF" + "\x43\xDA\x71\x08\x7C\x13\xAA\x1E" + "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90" + "\x27\xBE\x32\xC9\x60\xF7\x6B\x02" + "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74" + "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6" + "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35" + "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7" + "\x3E\xD5\x49\xE0\x77\x0E\x82\x19" + "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B" + "\x22\x96\x2D\xC4\x38\xCF\x66\xFD" + "\x71\x08\x9F\x13\xAA\x41\xD8\x4C" + "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE" + "\x55\xEC\x60\xF7\x8E\x02\x99\x30" + "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2" + "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14" + "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63" + "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5" + "\x6C\x03\x77\x0E\xA5\x19\xB0\x47" + "\xDE\x52\xE9\x80\x17\x8B\x22\xB9" + "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08" + "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A" + "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC" + "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E" + "\xF5\x69\x00\x97\x0B\xA2\x39\xD0" + "\x44\xDB\x72\x09\x7D\x14\xAB\x1F" + "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91" + "\x28\xBF\x33\xCA\x61\xF8\x6C\x03" + "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75" + "\x0C\x80\x17\xAE\x22\xB9\x50\xE7" + "\x5B\xF2\x89\x20\x94\x2B\xC2\x36" + "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8" + "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A" + "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C" + "\x00\x97\x2E\xC5\x39\xD0\x67\xFE" + "\x72\x09\xA0\x14\xAB\x42\xD9\x4D" + "\xE4\x7B\x12", + .ilen = 1011, .result = "\xF3\x06\x3A\x84\xCD\xBA\x8E\x11" "\xB7\x74\x6F\x5C\x97\xFB\x36\xFE" "\xDE\x71\x58\xD4\x15\xD1\xC1\xA4" @@ -21845,11 +22421,75 @@ static struct cipher_testvec camellia_ctr_enc_tv_template[] = { "\x7E\x42\xEC\xB6\x6F\x4D\x6B\x48" "\xE6\xA6\x50\x80\x78\x9E\xF1\xB0" "\x4D\xB2\x0D\x3D\xFC\x40\x25\x4D" - "\x93\x11\x1C", - .rlen = 499, + "\x93\x11\x1C\xE9\xD2\x9F\x6E\x90" + "\xE5\x41\x4A\xE2\x3C\x45\x29\x35" + "\xEC\xD6\x47\x50\xCB\x7B\xA2\x32" + "\xF7\x8B\x62\xF1\xE3\x9A\xFE\xC7" + "\x1D\x8C\x02\x72\x68\x09\xE9\xB6" + "\x4A\x80\xE6\xB1\x56\xDF\x90\xD4" + "\x93\x74\xA4\xCE\x20\x23\xBF\x48" + "\xA5\xDE\x1B\xFA\x40\x69\x31\x98" + "\x62\x6E\xA5\xC7\xBF\x0C\x62\xE5" + "\x6D\xE1\x93\xF1\x83\x10\x1C\xCA" + "\xF6\x5C\x19\xF8\x90\x78\xCB\xE4" + "\x0B\x3A\xB5\xF8\x43\x86\xD3\x3F" + "\xBA\x83\x34\x3C\x42\xCC\x7D\x28" + "\x29\x63\x4F\xD8\x02\x17\xC5\x07" + "\x2C\xA4\xAC\x79\xCB\xC3\xA9\x09" + "\x81\x45\x18\xED\xE4\xCB\x42\x3B" + "\x87\x2D\x23\xDC\xC5\xBA\x45\xBD" + "\x92\xE5\x02\x97\x96\xCE\xAD\xEC" + "\xBA\xD8\x76\xF8\xCA\xC1\x31\xEC" + "\x1E\x4F\x3F\x83\xF8\x33\xE8\x6E" + "\xCC\xF8\x5F\xDD\x65\x50\x99\x69" + "\xAF\x48\xCE\xA5\xBA\xB6\x14\x9F" + "\x05\x93\xB2\xE6\x59\xC8\x28\xFE" + "\x8F\x37\xF9\x64\xB9\xA5\x56\x8F" + "\xF1\x1B\x90\xEF\xAE\xEB\xFC\x09" + "\x11\x7A\xF2\x19\x0A\x0A\x9A\x3C" + "\xE2\x5E\x29\xFA\x31\x9B\xC1\x74" + "\x1E\x10\x3E\x07\xA9\x31\x6D\xF8" + "\x81\xF5\xD5\x8A\x04\x23\x51\xAC" + "\xA2\xE2\x63\xFD\x27\x1F\x79\x5B" + "\x1F\xE8\xDA\x11\x49\x4D\x1C\xBA" + "\x54\xCC\x0F\xBA\x92\x69\xE5\xCB" + "\x41\x1A\x67\xA6\x40\x82\x70\x8C" + "\x19\x79\x08\xA4\x51\x20\x7D\xC9" + "\x12\x27\xAE\x20\x0D\x2C\xA1\x6D" + "\xF4\x55\xD4\xE7\xE6\xD4\x28\x08" + "\x00\x70\x12\x56\x56\x50\xAD\x14" + "\x5C\x3E\xA2\xD1\x36\x3F\x36\x48" + "\xED\xB1\x57\x3E\x5D\x15\xF6\x1E" + "\x53\xE9\xA4\x3E\xED\x7D\xCF\x7D" + "\x29\xAF\xF3\x1E\x51\xA8\x9F\x85" + "\x8B\xF0\xBB\xCE\xCC\x39\xC3\x64" + "\x4B\xF2\xAD\x70\x19\xD4\x44\x8F" + "\x91\x76\xE8\x15\x66\x34\x9F\xF6" + "\x0F\x15\xA4\xA8\x24\xF8\x58\xB1" + "\x38\x46\x47\xC7\x9B\xCA\xE9\x42" + "\x44\xAA\xE6\xB5\x9C\x91\xA4\xD3" + "\x16\xA0\xED\x42\xBE\xB5\x06\x19" + "\xBE\x67\xE8\xBC\x22\x32\xA4\x1E" + "\x93\xEB\xBE\xE9\xE1\x93\xE5\x31" + "\x3A\xA2\x75\xDF\xE3\x6B\xE7\xCC" + "\xB4\x70\x20\xE0\x6D\x82\x7C\xC8" + "\x94\x5C\x5E\x37\x18\xAD\xED\x8B" + "\x44\x86\xCA\x5E\x07\xB7\x70\x8D" + "\x40\x48\x19\x73\x7C\x78\x64\x0B" + "\xDB\x01\xCA\xAE\x63\x19\xE9\xD1" + "\x6B\x2C\x84\x10\x45\x42\x2E\xC3" + "\xDF\x7F\xAA\xE8\x87\x1B\x63\x46" + "\x74\x28\x9D\x05\x30\x20\x62\x41" + "\xC0\x9F\x2C\x36\x2B\x78\xD7\x26" + "\xDF\x58\x51\xED\xFA\xDC\x87\x79" + "\xBF\x8C\xBF\xC4\x0F\xE5\x05\xDA" + "\x45\xE3\x35\x0D\x69\x91\x54\x1C" + "\xE7\x2C\x49\x08\x8B\x72\xFA\x5C" + "\xF1\x6B\xD9", + .rlen = 1011, .also_non_np = 1, .np = 2, - .tap = { 499 - 16, 16 }, + .tap = { 1011 - 16, 16 }, }, { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -21919,8 +22559,72 @@ static struct cipher_testvec camellia_ctr_enc_tv_template[] = { "\x86\x1D\xB4\x28\xBF\x56\xED\x61" "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3" "\x6A\x01\x75\x0C\xA3\x17\xAE\x45" - "\xDC\x50\xE7\x7E\x15\x89\x20\xB7", - .ilen = 496, + "\xDC\x50\xE7\x7E\x15\x89\x20\xB7" + "\x2B\xC2\x59\xF0\x64\xFB\x92\x06" + "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78" + "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA" + "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C" + "\xF3\x67\xFE\x95\x09\xA0\x37\xCE" + "\x42\xD9\x70\x07\x7B\x12\xA9\x1D" + "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F" + "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01" + "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73" + "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5" + "\x59\xF0\x87\x1E\x92\x29\xC0\x34" + "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6" + "\x3D\xD4\x48\xDF\x76\x0D\x81\x18" + "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A" + "\x21\x95\x2C\xC3\x37\xCE\x65\xFC" + "\x70\x07\x9E\x12\xA9\x40\xD7\x4B" + "\xE2\x79\x10\x84\x1B\xB2\x26\xBD" + "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F" + "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1" + "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13" + "\x87\x1E\xB5\x29\xC0\x57\xEE\x62" + "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4" + "\x6B\x02\x76\x0D\xA4\x18\xAF\x46" + "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8" + "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07" + "\x9E\x35\xCC\x40\xD7\x6E\x05\x79" + "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB" + "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D" + "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF" + "\x43\xDA\x71\x08\x7C\x13\xAA\x1E" + "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90" + "\x27\xBE\x32\xC9\x60\xF7\x6B\x02" + "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74" + "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6" + "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35" + "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7" + "\x3E\xD5\x49\xE0\x77\x0E\x82\x19" + "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B" + "\x22\x96\x2D\xC4\x38\xCF\x66\xFD" + "\x71\x08\x9F\x13\xAA\x41\xD8\x4C" + "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE" + "\x55\xEC\x60\xF7\x8E\x02\x99\x30" + "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2" + "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14" + "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63" + "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5" + "\x6C\x03\x77\x0E\xA5\x19\xB0\x47" + "\xDE\x52\xE9\x80\x17\x8B\x22\xB9" + "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08" + "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A" + "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC" + "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E" + "\xF5\x69\x00\x97\x0B\xA2\x39\xD0" + "\x44\xDB\x72\x09\x7D\x14\xAB\x1F" + "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91" + "\x28\xBF\x33\xCA\x61\xF8\x6C\x03" + "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75" + "\x0C\x80\x17\xAE\x22\xB9\x50\xE7" + "\x5B\xF2\x89\x20\x94\x2B\xC2\x36" + "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8" + "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A" + "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C" + "\x00\x97\x2E\xC5\x39\xD0\x67\xFE" + "\x72\x09\xA0\x14\xAB\x42\xD9\x4D", + .ilen = 1008, .result = "\x85\x79\x6C\x8B\x2B\x6D\x14\xF9" "\xA6\x83\xB6\x80\x5B\x3A\xF3\x7E" "\x30\x29\xEB\x1F\xDC\x19\x5F\xEB" @@ -21982,8 +22686,72 @@ static struct cipher_testvec camellia_ctr_enc_tv_template[] = { "\xB4\x3A\x5F\x19\xCF\x42\x1B\x22" "\x0B\x2D\x7B\xF1\xC5\x43\xF7\x5E" "\x12\xA8\x01\x64\x16\x0B\x26\x5A" - "\x0C\x95\x0F\x40\xC5\x5A\x06\x7C", - .rlen = 496, + "\x0C\x95\x0F\x40\xC5\x5A\x06\x7C" + "\xCF\xF5\xD5\xB7\x7A\x34\x23\xB6" + "\xAA\x9E\xA8\x98\xA2\xF8\x3D\xD3" + "\x3F\x23\x69\x63\x56\x96\x45\xD6" + "\x74\x23\x1D\x5C\x63\xCC\xD8\x78" + "\x16\xE2\x9C\xD2\x80\x02\xF2\x28" + "\x69\x2F\xC4\xA8\x15\x15\x24\x3B" + "\xCB\xF0\x14\xE4\x62\xC8\xF3\xD1" + "\x03\x58\x1B\x33\x77\x74\x1F\xB4" + "\x07\x86\xF2\x21\xB7\x41\xAE\xBF" + "\x25\xC2\xFF\x51\xEF\xEA\xCE\xC4" + "\x5F\xD9\xB8\x18\x6A\xF0\x0F\x0D" + "\xF8\x04\xBB\x6D\x62\x33\x87\x26" + "\x4F\x2F\x14\x6E\xDC\xDB\x66\x09" + "\x2A\xEF\x7D\x84\x10\xAC\x82\x5E" + "\xD2\xE4\xAD\x74\x7A\x6D\xCC\x3A" + "\x7B\x62\xD8\xD6\x07\x2D\xF7\xDF" + "\x9B\xB3\x82\xCF\x9C\x1D\x76\x5C" + "\xAC\x7B\xD4\x9B\x45\xA1\x64\x11" + "\x66\xF1\xA7\x0B\xF9\xDD\x00\xDD" + "\xA4\x45\x3D\x3E\x03\xC9\x2E\xCB" + "\xC3\x14\x84\x72\xFD\x41\xDC\xBD" + "\x75\xBE\xA8\xE5\x16\x48\x64\x39" + "\xCA\xF3\xE6\xDC\x25\x24\xF1\x6D" + "\xB2\x8D\xC5\x38\x54\xD3\x5D\x6D" + "\x0B\x29\x10\x15\x0E\x13\x3B\xAC" + "\x7E\xCC\x9E\x3E\x18\x48\xA6\x02" + "\xEF\x03\xB2\x2E\xE3\xD2\x70\x21" + "\xB4\x19\x26\xBE\x3A\x3D\x05\xE0" + "\xF8\x09\xAF\xE4\x31\x26\x92\x2F" + "\x8F\x55\xAC\xED\x0B\xB2\xA5\x34" + "\xBE\x50\xB1\x02\x22\x96\xE3\x40" + "\x7B\x70\x50\x6E\x3B\xD5\xE5\xA0" + "\x8E\xA2\xAD\x14\x60\x5C\x7A\x2B" + "\x3D\x1B\x7F\xC1\xC0\x2C\x56\x36" + "\xD2\x0A\x32\x06\x97\x34\xB9\xF4" + "\x6F\x9F\x7E\x80\xD0\x9D\xF7\x6A" + "\x21\xC1\xA2\x6A\xB1\x96\x5B\x4D" + "\x7A\x15\x6C\xC4\x4E\xB8\xE0\x9E" + "\x6C\x50\xF3\x9C\xC9\xB5\x23\xB7" + "\xF1\xD4\x29\x4A\x23\xC4\xAD\x1E" + "\x2C\x07\xD2\x43\x5F\x57\x93\xCA" + "\x85\xF9\x9F\xAD\x4C\xF1\xE4\xB1" + "\x1A\x8E\x28\xA4\xB6\x52\x77\x7E" + "\x68\xC6\x47\xB9\x76\xCC\x65\x5F" + "\x0B\xF9\x67\x93\xD8\x0E\x9A\x37" + "\x5F\x41\xED\x64\x6C\xAD\x5F\xED" + "\x3F\x8D\xFB\x8E\x1E\xA0\xE4\x1F" + "\xC2\xC7\xED\x18\x43\xE1\x20\x86" + "\x5D\xBC\x30\x70\x22\xA1\xDC\x53" + "\x10\x3A\x8D\x47\x82\xCD\x7F\x59" + "\x03\x2D\x6D\xF5\xE7\x79\xD4\x07" + "\x68\x2A\xA5\x42\x19\x4D\xAF\xF5" + "\xED\x47\x83\xBC\x5F\x62\x84\xDA" + "\xDA\x41\xFF\xB0\x1D\x64\xA3\xC8" + "\xBD\x4E\xE0\xB8\x7F\xEE\x55\x0A" + "\x4E\x61\xB2\x51\xF6\x9C\x95\xF6" + "\x92\xBB\xF6\xC5\xF0\x09\x86\xDE" + "\x37\x9E\x29\xF9\x2A\x18\x73\x0D" + "\xDC\x7E\x6B\x7B\x1B\x43\x8C\xEA" + "\x13\xC8\x1A\x47\x0A\x2D\x6D\x56" + "\xCD\xD2\xE7\x53\x1A\xAB\x1C\x3C" + "\xC5\x9B\x03\x70\x29\x2A\x49\x09" + "\x67\xA1\xEA\xD6\x3A\x5B\xBF\x71" + "\x1D\x48\x64\x6C\xFB\xC0\x9E\x36", + .rlen = 1008, }, }; @@ -22192,8 +22960,72 @@ static struct cipher_testvec camellia_ctr_dec_tv_template[] = { "\x7E\x42\xEC\xB6\x6F\x4D\x6B\x48" "\xE6\xA6\x50\x80\x78\x9E\xF1\xB0" "\x4D\xB2\x0D\x3D\xFC\x40\x25\x4D" - "\x93\x11\x1C", - .ilen = 499, + "\x93\x11\x1C\xE9\xD2\x9F\x6E\x90" + "\xE5\x41\x4A\xE2\x3C\x45\x29\x35" + "\xEC\xD6\x47\x50\xCB\x7B\xA2\x32" + "\xF7\x8B\x62\xF1\xE3\x9A\xFE\xC7" + "\x1D\x8C\x02\x72\x68\x09\xE9\xB6" + "\x4A\x80\xE6\xB1\x56\xDF\x90\xD4" + "\x93\x74\xA4\xCE\x20\x23\xBF\x48" + "\xA5\xDE\x1B\xFA\x40\x69\x31\x98" + "\x62\x6E\xA5\xC7\xBF\x0C\x62\xE5" + "\x6D\xE1\x93\xF1\x83\x10\x1C\xCA" + "\xF6\x5C\x19\xF8\x90\x78\xCB\xE4" + "\x0B\x3A\xB5\xF8\x43\x86\xD3\x3F" + "\xBA\x83\x34\x3C\x42\xCC\x7D\x28" + "\x29\x63\x4F\xD8\x02\x17\xC5\x07" + "\x2C\xA4\xAC\x79\xCB\xC3\xA9\x09" + "\x81\x45\x18\xED\xE4\xCB\x42\x3B" + "\x87\x2D\x23\xDC\xC5\xBA\x45\xBD" + "\x92\xE5\x02\x97\x96\xCE\xAD\xEC" + "\xBA\xD8\x76\xF8\xCA\xC1\x31\xEC" + "\x1E\x4F\x3F\x83\xF8\x33\xE8\x6E" + "\xCC\xF8\x5F\xDD\x65\x50\x99\x69" + "\xAF\x48\xCE\xA5\xBA\xB6\x14\x9F" + "\x05\x93\xB2\xE6\x59\xC8\x28\xFE" + "\x8F\x37\xF9\x64\xB9\xA5\x56\x8F" + "\xF1\x1B\x90\xEF\xAE\xEB\xFC\x09" + "\x11\x7A\xF2\x19\x0A\x0A\x9A\x3C" + "\xE2\x5E\x29\xFA\x31\x9B\xC1\x74" + "\x1E\x10\x3E\x07\xA9\x31\x6D\xF8" + "\x81\xF5\xD5\x8A\x04\x23\x51\xAC" + "\xA2\xE2\x63\xFD\x27\x1F\x79\x5B" + "\x1F\xE8\xDA\x11\x49\x4D\x1C\xBA" + "\x54\xCC\x0F\xBA\x92\x69\xE5\xCB" + "\x41\x1A\x67\xA6\x40\x82\x70\x8C" + "\x19\x79\x08\xA4\x51\x20\x7D\xC9" + "\x12\x27\xAE\x20\x0D\x2C\xA1\x6D" + "\xF4\x55\xD4\xE7\xE6\xD4\x28\x08" + "\x00\x70\x12\x56\x56\x50\xAD\x14" + "\x5C\x3E\xA2\xD1\x36\x3F\x36\x48" + "\xED\xB1\x57\x3E\x5D\x15\xF6\x1E" + "\x53\xE9\xA4\x3E\xED\x7D\xCF\x7D" + "\x29\xAF\xF3\x1E\x51\xA8\x9F\x85" + "\x8B\xF0\xBB\xCE\xCC\x39\xC3\x64" + "\x4B\xF2\xAD\x70\x19\xD4\x44\x8F" + "\x91\x76\xE8\x15\x66\x34\x9F\xF6" + "\x0F\x15\xA4\xA8\x24\xF8\x58\xB1" + "\x38\x46\x47\xC7\x9B\xCA\xE9\x42" + "\x44\xAA\xE6\xB5\x9C\x91\xA4\xD3" + "\x16\xA0\xED\x42\xBE\xB5\x06\x19" + "\xBE\x67\xE8\xBC\x22\x32\xA4\x1E" + "\x93\xEB\xBE\xE9\xE1\x93\xE5\x31" + "\x3A\xA2\x75\xDF\xE3\x6B\xE7\xCC" + "\xB4\x70\x20\xE0\x6D\x82\x7C\xC8" + "\x94\x5C\x5E\x37\x18\xAD\xED\x8B" + "\x44\x86\xCA\x5E\x07\xB7\x70\x8D" + "\x40\x48\x19\x73\x7C\x78\x64\x0B" + "\xDB\x01\xCA\xAE\x63\x19\xE9\xD1" + "\x6B\x2C\x84\x10\x45\x42\x2E\xC3" + "\xDF\x7F\xAA\xE8\x87\x1B\x63\x46" + "\x74\x28\x9D\x05\x30\x20\x62\x41" + "\xC0\x9F\x2C\x36\x2B\x78\xD7\x26" + "\xDF\x58\x51\xED\xFA\xDC\x87\x79" + "\xBF\x8C\xBF\xC4\x0F\xE5\x05\xDA" + "\x45\xE3\x35\x0D\x69\x91\x54\x1C" + "\xE7\x2C\x49\x08\x8B\x72\xFA\x5C" + "\xF1\x6B\xD9", + .ilen = 1011, .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31" "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3" "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15" @@ -22256,11 +23088,75 @@ static struct cipher_testvec camellia_ctr_dec_tv_template[] = { "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3" "\x6A\x01\x75\x0C\xA3\x17\xAE\x45" "\xDC\x50\xE7\x7E\x15\x89\x20\xB7" - "\x2B\xC2\x59", - .rlen = 499, + "\x2B\xC2\x59\xF0\x64\xFB\x92\x06" + "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78" + "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA" + "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C" + "\xF3\x67\xFE\x95\x09\xA0\x37\xCE" + "\x42\xD9\x70\x07\x7B\x12\xA9\x1D" + "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F" + "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01" + "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73" + "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5" + "\x59\xF0\x87\x1E\x92\x29\xC0\x34" + "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6" + "\x3D\xD4\x48\xDF\x76\x0D\x81\x18" + "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A" + "\x21\x95\x2C\xC3\x37\xCE\x65\xFC" + "\x70\x07\x9E\x12\xA9\x40\xD7\x4B" + "\xE2\x79\x10\x84\x1B\xB2\x26\xBD" + "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F" + "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1" + "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13" + "\x87\x1E\xB5\x29\xC0\x57\xEE\x62" + "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4" + "\x6B\x02\x76\x0D\xA4\x18\xAF\x46" + "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8" + "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07" + "\x9E\x35\xCC\x40\xD7\x6E\x05\x79" + "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB" + "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D" + "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF" + "\x43\xDA\x71\x08\x7C\x13\xAA\x1E" + "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90" + "\x27\xBE\x32\xC9\x60\xF7\x6B\x02" + "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74" + "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6" + "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35" + "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7" + "\x3E\xD5\x49\xE0\x77\x0E\x82\x19" + "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B" + "\x22\x96\x2D\xC4\x38\xCF\x66\xFD" + "\x71\x08\x9F\x13\xAA\x41\xD8\x4C" + "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE" + "\x55\xEC\x60\xF7\x8E\x02\x99\x30" + "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2" + "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14" + "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63" + "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5" + "\x6C\x03\x77\x0E\xA5\x19\xB0\x47" + "\xDE\x52\xE9\x80\x17\x8B\x22\xB9" + "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08" + "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A" + "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC" + "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E" + "\xF5\x69\x00\x97\x0B\xA2\x39\xD0" + "\x44\xDB\x72\x09\x7D\x14\xAB\x1F" + "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91" + "\x28\xBF\x33\xCA\x61\xF8\x6C\x03" + "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75" + "\x0C\x80\x17\xAE\x22\xB9\x50\xE7" + "\x5B\xF2\x89\x20\x94\x2B\xC2\x36" + "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8" + "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A" + "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C" + "\x00\x97\x2E\xC5\x39\xD0\x67\xFE" + "\x72\x09\xA0\x14\xAB\x42\xD9\x4D" + "\xE4\x7B\x12", + .rlen = 1011, .also_non_np = 1, .np = 2, - .tap = { 499 - 16, 16 }, + .tap = { 1011 - 16, 16 }, }, { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -22330,8 +23226,72 @@ static struct cipher_testvec camellia_ctr_dec_tv_template[] = { "\xB4\x3A\x5F\x19\xCF\x42\x1B\x22" "\x0B\x2D\x7B\xF1\xC5\x43\xF7\x5E" "\x12\xA8\x01\x64\x16\x0B\x26\x5A" - "\x0C\x95\x0F\x40\xC5\x5A\x06\x7C", - .ilen = 496, + "\x0C\x95\x0F\x40\xC5\x5A\x06\x7C" + "\xCF\xF5\xD5\xB7\x7A\x34\x23\xB6" + "\xAA\x9E\xA8\x98\xA2\xF8\x3D\xD3" + "\x3F\x23\x69\x63\x56\x96\x45\xD6" + "\x74\x23\x1D\x5C\x63\xCC\xD8\x78" + "\x16\xE2\x9C\xD2\x80\x02\xF2\x28" + "\x69\x2F\xC4\xA8\x15\x15\x24\x3B" + "\xCB\xF0\x14\xE4\x62\xC8\xF3\xD1" + "\x03\x58\x1B\x33\x77\x74\x1F\xB4" + "\x07\x86\xF2\x21\xB7\x41\xAE\xBF" + "\x25\xC2\xFF\x51\xEF\xEA\xCE\xC4" + "\x5F\xD9\xB8\x18\x6A\xF0\x0F\x0D" + "\xF8\x04\xBB\x6D\x62\x33\x87\x26" + "\x4F\x2F\x14\x6E\xDC\xDB\x66\x09" + "\x2A\xEF\x7D\x84\x10\xAC\x82\x5E" + "\xD2\xE4\xAD\x74\x7A\x6D\xCC\x3A" + "\x7B\x62\xD8\xD6\x07\x2D\xF7\xDF" + "\x9B\xB3\x82\xCF\x9C\x1D\x76\x5C" + "\xAC\x7B\xD4\x9B\x45\xA1\x64\x11" + "\x66\xF1\xA7\x0B\xF9\xDD\x00\xDD" + "\xA4\x45\x3D\x3E\x03\xC9\x2E\xCB" + "\xC3\x14\x84\x72\xFD\x41\xDC\xBD" + "\x75\xBE\xA8\xE5\x16\x48\x64\x39" + "\xCA\xF3\xE6\xDC\x25\x24\xF1\x6D" + "\xB2\x8D\xC5\x38\x54\xD3\x5D\x6D" + "\x0B\x29\x10\x15\x0E\x13\x3B\xAC" + "\x7E\xCC\x9E\x3E\x18\x48\xA6\x02" + "\xEF\x03\xB2\x2E\xE3\xD2\x70\x21" + "\xB4\x19\x26\xBE\x3A\x3D\x05\xE0" + "\xF8\x09\xAF\xE4\x31\x26\x92\x2F" + "\x8F\x55\xAC\xED\x0B\xB2\xA5\x34" + "\xBE\x50\xB1\x02\x22\x96\xE3\x40" + "\x7B\x70\x50\x6E\x3B\xD5\xE5\xA0" + "\x8E\xA2\xAD\x14\x60\x5C\x7A\x2B" + "\x3D\x1B\x7F\xC1\xC0\x2C\x56\x36" + "\xD2\x0A\x32\x06\x97\x34\xB9\xF4" + "\x6F\x9F\x7E\x80\xD0\x9D\xF7\x6A" + "\x21\xC1\xA2\x6A\xB1\x96\x5B\x4D" + "\x7A\x15\x6C\xC4\x4E\xB8\xE0\x9E" + "\x6C\x50\xF3\x9C\xC9\xB5\x23\xB7" + "\xF1\xD4\x29\x4A\x23\xC4\xAD\x1E" + "\x2C\x07\xD2\x43\x5F\x57\x93\xCA" + "\x85\xF9\x9F\xAD\x4C\xF1\xE4\xB1" + "\x1A\x8E\x28\xA4\xB6\x52\x77\x7E" + "\x68\xC6\x47\xB9\x76\xCC\x65\x5F" + "\x0B\xF9\x67\x93\xD8\x0E\x9A\x37" + "\x5F\x41\xED\x64\x6C\xAD\x5F\xED" + "\x3F\x8D\xFB\x8E\x1E\xA0\xE4\x1F" + "\xC2\xC7\xED\x18\x43\xE1\x20\x86" + "\x5D\xBC\x30\x70\x22\xA1\xDC\x53" + "\x10\x3A\x8D\x47\x82\xCD\x7F\x59" + "\x03\x2D\x6D\xF5\xE7\x79\xD4\x07" + "\x68\x2A\xA5\x42\x19\x4D\xAF\xF5" + "\xED\x47\x83\xBC\x5F\x62\x84\xDA" + "\xDA\x41\xFF\xB0\x1D\x64\xA3\xC8" + "\xBD\x4E\xE0\xB8\x7F\xEE\x55\x0A" + "\x4E\x61\xB2\x51\xF6\x9C\x95\xF6" + "\x92\xBB\xF6\xC5\xF0\x09\x86\xDE" + "\x37\x9E\x29\xF9\x2A\x18\x73\x0D" + "\xDC\x7E\x6B\x7B\x1B\x43\x8C\xEA" + "\x13\xC8\x1A\x47\x0A\x2D\x6D\x56" + "\xCD\xD2\xE7\x53\x1A\xAB\x1C\x3C" + "\xC5\x9B\x03\x70\x29\x2A\x49\x09" + "\x67\xA1\xEA\xD6\x3A\x5B\xBF\x71" + "\x1D\x48\x64\x6C\xFB\xC0\x9E\x36", + .ilen = 1008, .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31" "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3" "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15" @@ -22393,8 +23353,72 @@ static struct cipher_testvec camellia_ctr_dec_tv_template[] = { "\x86\x1D\xB4\x28\xBF\x56\xED\x61" "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3" "\x6A\x01\x75\x0C\xA3\x17\xAE\x45" - "\xDC\x50\xE7\x7E\x15\x89\x20\xB7", - .rlen = 496, + "\xDC\x50\xE7\x7E\x15\x89\x20\xB7" + "\x2B\xC2\x59\xF0\x64\xFB\x92\x06" + "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78" + "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA" + "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C" + "\xF3\x67\xFE\x95\x09\xA0\x37\xCE" + "\x42\xD9\x70\x07\x7B\x12\xA9\x1D" + "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F" + "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01" + "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73" + "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5" + "\x59\xF0\x87\x1E\x92\x29\xC0\x34" + "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6" + "\x3D\xD4\x48\xDF\x76\x0D\x81\x18" + "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A" + "\x21\x95\x2C\xC3\x37\xCE\x65\xFC" + "\x70\x07\x9E\x12\xA9\x40\xD7\x4B" + "\xE2\x79\x10\x84\x1B\xB2\x26\xBD" + "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F" + "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1" + "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13" + "\x87\x1E\xB5\x29\xC0\x57\xEE\x62" + "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4" + "\x6B\x02\x76\x0D\xA4\x18\xAF\x46" + "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8" + "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07" + "\x9E\x35\xCC\x40\xD7\x6E\x05\x79" + "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB" + "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D" + "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF" + "\x43\xDA\x71\x08\x7C\x13\xAA\x1E" + "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90" + "\x27\xBE\x32\xC9\x60\xF7\x6B\x02" + "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74" + "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6" + "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35" + "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7" + "\x3E\xD5\x49\xE0\x77\x0E\x82\x19" + "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B" + "\x22\x96\x2D\xC4\x38\xCF\x66\xFD" + "\x71\x08\x9F\x13\xAA\x41\xD8\x4C" + "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE" + "\x55\xEC\x60\xF7\x8E\x02\x99\x30" + "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2" + "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14" + "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63" + "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5" + "\x6C\x03\x77\x0E\xA5\x19\xB0\x47" + "\xDE\x52\xE9\x80\x17\x8B\x22\xB9" + "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08" + "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A" + "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC" + "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E" + "\xF5\x69\x00\x97\x0B\xA2\x39\xD0" + "\x44\xDB\x72\x09\x7D\x14\xAB\x1F" + "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91" + "\x28\xBF\x33\xCA\x61\xF8\x6C\x03" + "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75" + "\x0C\x80\x17\xAE\x22\xB9\x50\xE7" + "\x5B\xF2\x89\x20\x94\x2B\xC2\x36" + "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8" + "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A" + "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C" + "\x00\x97\x2E\xC5\x39\xD0\x67\xFE" + "\x72\x09\xA0\x14\xAB\x42\xD9\x4D", + .rlen = 1008, }, }; -- cgit v0.10.2 From ad8b7c3e92868dd86c54d9d5321000bbb4096f0d Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sat, 13 Apr 2013 13:46:40 +0300 Subject: crypto: tcrypt - add async cipher speed tests for blowfish Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 24ea7df..66d254c 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1768,6 +1768,21 @@ static int do_test(int m) speed_template_32_64); break; + case 509: + test_acipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0, + speed_template_8_32); + test_acipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0, + speed_template_8_32); + test_acipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0, + speed_template_8_32); + test_acipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0, + speed_template_8_32); + test_acipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0, + speed_template_8_32); + test_acipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0, + speed_template_8_32); + break; + case 1000: test_available(); break; -- cgit v0.10.2 From 604880107010a1e5794552d184cd5471ea31b973 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sat, 13 Apr 2013 13:46:45 +0300 Subject: crypto: blowfish - add AVX2/x86_64 implementation of blowfish cipher Patch adds AVX2/x86-64 implementation of Blowfish cipher, requiring 32 parallel blocks for input (256 bytes). Table look-ups are performed using vpgatherdd instruction directly from vector registers and thus should be faster than earlier implementations. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 03cd731..28464ef 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -3,6 +3,8 @@ # avx_supported := $(call as-instr,vpxor %xmm0$(comma)%xmm0$(comma)%xmm0,yes,no) +avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ + $(comma)4)$(comma)%ymm2,yes,no) obj-$(CONFIG_CRYPTO_ABLK_HELPER_X86) += ablk_helper.o obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o @@ -38,6 +40,11 @@ ifeq ($(avx_supported),yes) obj-$(CONFIG_CRYPTO_SERPENT_AVX_X86_64) += serpent-avx-x86_64.o endif +# These modules require assembler to support AVX2. +ifeq ($(avx2_supported),yes) + obj-$(CONFIG_CRYPTO_BLOWFISH_AVX2_X86_64) += blowfish-avx2.o +endif + aes-i586-y := aes-i586-asm_32.o aes_glue.o twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o @@ -62,6 +69,10 @@ ifeq ($(avx_supported),yes) serpent_avx_glue.o endif +ifeq ($(avx2_supported),yes) + blowfish-avx2-y := blowfish-avx2-asm_64.o blowfish_avx2_glue.o +endif + aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o diff --git a/arch/x86/crypto/blowfish-avx2-asm_64.S b/arch/x86/crypto/blowfish-avx2-asm_64.S new file mode 100644 index 0000000..784452e --- /dev/null +++ b/arch/x86/crypto/blowfish-avx2-asm_64.S @@ -0,0 +1,449 @@ +/* + * x86_64/AVX2 assembler optimized version of Blowfish + * + * Copyright © 2012-2013 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#include + +.file "blowfish-avx2-asm_64.S" + +.data +.align 32 + +.Lprefetch_mask: +.long 0*64 +.long 1*64 +.long 2*64 +.long 3*64 +.long 4*64 +.long 5*64 +.long 6*64 +.long 7*64 + +.Lbswap32_mask: +.long 0x00010203 +.long 0x04050607 +.long 0x08090a0b +.long 0x0c0d0e0f + +.Lbswap128_mask: + .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +.Lbswap_iv_mask: + .byte 7, 6, 5, 4, 3, 2, 1, 0, 7, 6, 5, 4, 3, 2, 1, 0 + +.text +/* structure of crypto context */ +#define p 0 +#define s0 ((16 + 2) * 4) +#define s1 ((16 + 2 + (1 * 256)) * 4) +#define s2 ((16 + 2 + (2 * 256)) * 4) +#define s3 ((16 + 2 + (3 * 256)) * 4) + +/* register macros */ +#define CTX %rdi +#define RIO %rdx + +#define RS0 %rax +#define RS1 %r8 +#define RS2 %r9 +#define RS3 %r10 + +#define RLOOP %r11 +#define RLOOPd %r11d + +#define RXr0 %ymm8 +#define RXr1 %ymm9 +#define RXr2 %ymm10 +#define RXr3 %ymm11 +#define RXl0 %ymm12 +#define RXl1 %ymm13 +#define RXl2 %ymm14 +#define RXl3 %ymm15 + +/* temp regs */ +#define RT0 %ymm0 +#define RT0x %xmm0 +#define RT1 %ymm1 +#define RT1x %xmm1 +#define RIDX0 %ymm2 +#define RIDX1 %ymm3 +#define RIDX1x %xmm3 +#define RIDX2 %ymm4 +#define RIDX3 %ymm5 + +/* vpgatherdd mask and '-1' */ +#define RNOT %ymm6 + +/* byte mask, (-1 >> 24) */ +#define RBYTE %ymm7 + +/*********************************************************************** + * 32-way AVX2 blowfish + ***********************************************************************/ +#define F(xl, xr) \ + vpsrld $24, xl, RIDX0; \ + vpsrld $16, xl, RIDX1; \ + vpsrld $8, xl, RIDX2; \ + vpand RBYTE, RIDX1, RIDX1; \ + vpand RBYTE, RIDX2, RIDX2; \ + vpand RBYTE, xl, RIDX3; \ + \ + vpgatherdd RNOT, (RS0, RIDX0, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpcmpeqd RIDX0, RIDX0, RIDX0; \ + \ + vpgatherdd RNOT, (RS1, RIDX1, 4), RT1; \ + vpcmpeqd RIDX1, RIDX1, RIDX1; \ + vpaddd RT0, RT1, RT0; \ + \ + vpgatherdd RIDX0, (RS2, RIDX2, 4), RT1; \ + vpxor RT0, RT1, RT0; \ + \ + vpgatherdd RIDX1, (RS3, RIDX3, 4), RT1; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpaddd RT0, RT1, RT0; \ + \ + vpxor RT0, xr, xr; + +#define add_roundkey(xl, nmem) \ + vpbroadcastd nmem, RT0; \ + vpxor RT0, xl ## 0, xl ## 0; \ + vpxor RT0, xl ## 1, xl ## 1; \ + vpxor RT0, xl ## 2, xl ## 2; \ + vpxor RT0, xl ## 3, xl ## 3; + +#define round_enc() \ + add_roundkey(RXr, p(CTX,RLOOP,4)); \ + F(RXl0, RXr0); \ + F(RXl1, RXr1); \ + F(RXl2, RXr2); \ + F(RXl3, RXr3); \ + \ + add_roundkey(RXl, p+4(CTX,RLOOP,4)); \ + F(RXr0, RXl0); \ + F(RXr1, RXl1); \ + F(RXr2, RXl2); \ + F(RXr3, RXl3); + +#define round_dec() \ + add_roundkey(RXr, p+4*2(CTX,RLOOP,4)); \ + F(RXl0, RXr0); \ + F(RXl1, RXr1); \ + F(RXl2, RXr2); \ + F(RXl3, RXr3); \ + \ + add_roundkey(RXl, p+4(CTX,RLOOP,4)); \ + F(RXr0, RXl0); \ + F(RXr1, RXl1); \ + F(RXr2, RXl2); \ + F(RXr3, RXl3); + +#define init_round_constants() \ + vpcmpeqd RNOT, RNOT, RNOT; \ + leaq s0(CTX), RS0; \ + leaq s1(CTX), RS1; \ + leaq s2(CTX), RS2; \ + leaq s3(CTX), RS3; \ + vpsrld $24, RNOT, RBYTE; + +#define transpose_2x2(x0, x1, t0) \ + vpunpckldq x0, x1, t0; \ + vpunpckhdq x0, x1, x1; \ + \ + vpunpcklqdq t0, x1, x0; \ + vpunpckhqdq t0, x1, x1; + +#define read_block(xl, xr) \ + vbroadcasti128 .Lbswap32_mask, RT1; \ + \ + vpshufb RT1, xl ## 0, xl ## 0; \ + vpshufb RT1, xr ## 0, xr ## 0; \ + vpshufb RT1, xl ## 1, xl ## 1; \ + vpshufb RT1, xr ## 1, xr ## 1; \ + vpshufb RT1, xl ## 2, xl ## 2; \ + vpshufb RT1, xr ## 2, xr ## 2; \ + vpshufb RT1, xl ## 3, xl ## 3; \ + vpshufb RT1, xr ## 3, xr ## 3; \ + \ + transpose_2x2(xl ## 0, xr ## 0, RT0); \ + transpose_2x2(xl ## 1, xr ## 1, RT0); \ + transpose_2x2(xl ## 2, xr ## 2, RT0); \ + transpose_2x2(xl ## 3, xr ## 3, RT0); + +#define write_block(xl, xr) \ + vbroadcasti128 .Lbswap32_mask, RT1; \ + \ + transpose_2x2(xl ## 0, xr ## 0, RT0); \ + transpose_2x2(xl ## 1, xr ## 1, RT0); \ + transpose_2x2(xl ## 2, xr ## 2, RT0); \ + transpose_2x2(xl ## 3, xr ## 3, RT0); \ + \ + vpshufb RT1, xl ## 0, xl ## 0; \ + vpshufb RT1, xr ## 0, xr ## 0; \ + vpshufb RT1, xl ## 1, xl ## 1; \ + vpshufb RT1, xr ## 1, xr ## 1; \ + vpshufb RT1, xl ## 2, xl ## 2; \ + vpshufb RT1, xr ## 2, xr ## 2; \ + vpshufb RT1, xl ## 3, xl ## 3; \ + vpshufb RT1, xr ## 3, xr ## 3; + +.align 8 +__blowfish_enc_blk32: + /* input: + * %rdi: ctx, CTX + * RXl0..4, RXr0..4: plaintext + * output: + * RXl0..4, RXr0..4: ciphertext (RXl <=> RXr swapped) + */ + init_round_constants(); + + read_block(RXl, RXr); + + movl $1, RLOOPd; + add_roundkey(RXl, p+4*(0)(CTX)); + +.align 4 +.L__enc_loop: + round_enc(); + + leal 2(RLOOPd), RLOOPd; + cmpl $17, RLOOPd; + jne .L__enc_loop; + + add_roundkey(RXr, p+4*(17)(CTX)); + + write_block(RXl, RXr); + + ret; +ENDPROC(__blowfish_enc_blk32) + +.align 8 +__blowfish_dec_blk32: + /* input: + * %rdi: ctx, CTX + * RXl0..4, RXr0..4: ciphertext + * output: + * RXl0..4, RXr0..4: plaintext (RXl <=> RXr swapped) + */ + init_round_constants(); + + read_block(RXl, RXr); + + movl $14, RLOOPd; + add_roundkey(RXl, p+4*(17)(CTX)); + +.align 4 +.L__dec_loop: + round_dec(); + + addl $-2, RLOOPd; + jns .L__dec_loop; + + add_roundkey(RXr, p+4*(0)(CTX)); + + write_block(RXl, RXr); + + ret; +ENDPROC(__blowfish_dec_blk32) + +ENTRY(blowfish_ecb_enc_32way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + + vmovdqu 0*32(%rdx), RXl0; + vmovdqu 1*32(%rdx), RXr0; + vmovdqu 2*32(%rdx), RXl1; + vmovdqu 3*32(%rdx), RXr1; + vmovdqu 4*32(%rdx), RXl2; + vmovdqu 5*32(%rdx), RXr2; + vmovdqu 6*32(%rdx), RXl3; + vmovdqu 7*32(%rdx), RXr3; + + call __blowfish_enc_blk32; + + vmovdqu RXr0, 0*32(%rsi); + vmovdqu RXl0, 1*32(%rsi); + vmovdqu RXr1, 2*32(%rsi); + vmovdqu RXl1, 3*32(%rsi); + vmovdqu RXr2, 4*32(%rsi); + vmovdqu RXl2, 5*32(%rsi); + vmovdqu RXr3, 6*32(%rsi); + vmovdqu RXl3, 7*32(%rsi); + + vzeroupper; + + ret; +ENDPROC(blowfish_ecb_enc_32way) + +ENTRY(blowfish_ecb_dec_32way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + + vmovdqu 0*32(%rdx), RXl0; + vmovdqu 1*32(%rdx), RXr0; + vmovdqu 2*32(%rdx), RXl1; + vmovdqu 3*32(%rdx), RXr1; + vmovdqu 4*32(%rdx), RXl2; + vmovdqu 5*32(%rdx), RXr2; + vmovdqu 6*32(%rdx), RXl3; + vmovdqu 7*32(%rdx), RXr3; + + call __blowfish_dec_blk32; + + vmovdqu RXr0, 0*32(%rsi); + vmovdqu RXl0, 1*32(%rsi); + vmovdqu RXr1, 2*32(%rsi); + vmovdqu RXl1, 3*32(%rsi); + vmovdqu RXr2, 4*32(%rsi); + vmovdqu RXl2, 5*32(%rsi); + vmovdqu RXr3, 6*32(%rsi); + vmovdqu RXl3, 7*32(%rsi); + + vzeroupper; + + ret; +ENDPROC(blowfish_ecb_dec_32way) + +ENTRY(blowfish_cbc_dec_32way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + + vmovdqu 0*32(%rdx), RXl0; + vmovdqu 1*32(%rdx), RXr0; + vmovdqu 2*32(%rdx), RXl1; + vmovdqu 3*32(%rdx), RXr1; + vmovdqu 4*32(%rdx), RXl2; + vmovdqu 5*32(%rdx), RXr2; + vmovdqu 6*32(%rdx), RXl3; + vmovdqu 7*32(%rdx), RXr3; + + call __blowfish_dec_blk32; + + /* xor with src */ + vmovq (%rdx), RT0x; + vpshufd $0x4f, RT0x, RT0x; + vinserti128 $1, 8(%rdx), RT0, RT0; + vpxor RT0, RXr0, RXr0; + vpxor 0*32+24(%rdx), RXl0, RXl0; + vpxor 1*32+24(%rdx), RXr1, RXr1; + vpxor 2*32+24(%rdx), RXl1, RXl1; + vpxor 3*32+24(%rdx), RXr2, RXr2; + vpxor 4*32+24(%rdx), RXl2, RXl2; + vpxor 5*32+24(%rdx), RXr3, RXr3; + vpxor 6*32+24(%rdx), RXl3, RXl3; + + vmovdqu RXr0, (0*32)(%rsi); + vmovdqu RXl0, (1*32)(%rsi); + vmovdqu RXr1, (2*32)(%rsi); + vmovdqu RXl1, (3*32)(%rsi); + vmovdqu RXr2, (4*32)(%rsi); + vmovdqu RXl2, (5*32)(%rsi); + vmovdqu RXr3, (6*32)(%rsi); + vmovdqu RXl3, (7*32)(%rsi); + + vzeroupper; + + ret; +ENDPROC(blowfish_cbc_dec_32way) + +ENTRY(blowfish_ctr_32way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: iv (big endian, 64bit) + */ + + vzeroupper; + + vpcmpeqd RT0, RT0, RT0; + vpsrldq $8, RT0, RT0; /* a: -1, b: 0, c: -1, d: 0 */ + + vpcmpeqd RT1x, RT1x, RT1x; + vpaddq RT1x, RT1x, RT1x; /* a: -2, b: -2 */ + vpxor RIDX0, RIDX0, RIDX0; + vinserti128 $1, RT1x, RIDX0, RIDX0; /* a: 0, b: 0, c: -2, d: -2 */ + + vpaddq RIDX0, RT0, RT0; /* a: -1, b: 0, c: -3, d: -2 */ + + vpcmpeqd RT1, RT1, RT1; + vpaddq RT1, RT1, RT1; /* a: -2, b: -2, c: -2, d: -2 */ + vpaddq RT1, RT1, RIDX2; /* a: -4, b: -4, c: -4, d: -4 */ + + vbroadcasti128 .Lbswap_iv_mask, RIDX0; + vbroadcasti128 .Lbswap128_mask, RIDX1; + + /* load IV and byteswap */ + vmovq (%rcx), RT1x; + vinserti128 $1, RT1x, RT1, RT1; /* a: BE, b: 0, c: BE, d: 0 */ + vpshufb RIDX0, RT1, RT1; /* a: LE, b: LE, c: LE, d: LE */ + + /* construct IVs */ + vpsubq RT0, RT1, RT1; /* a: le1, b: le0, c: le3, d: le2 */ + vpshufb RIDX1, RT1, RXl0; /* a: be0, b: be1, c: be2, d: be3 */ + vpsubq RIDX2, RT1, RT1; /* le5, le4, le7, le6 */ + vpshufb RIDX1, RT1, RXr0; /* be4, be5, be6, be7 */ + vpsubq RIDX2, RT1, RT1; + vpshufb RIDX1, RT1, RXl1; + vpsubq RIDX2, RT1, RT1; + vpshufb RIDX1, RT1, RXr1; + vpsubq RIDX2, RT1, RT1; + vpshufb RIDX1, RT1, RXl2; + vpsubq RIDX2, RT1, RT1; + vpshufb RIDX1, RT1, RXr2; + vpsubq RIDX2, RT1, RT1; + vpshufb RIDX1, RT1, RXl3; + vpsubq RIDX2, RT1, RT1; + vpshufb RIDX1, RT1, RXr3; + + /* store last IV */ + vpsubq RIDX2, RT1, RT1; /* a: le33, b: le32, ... */ + vpshufb RIDX1x, RT1x, RT1x; /* a: be32, ... */ + vmovq RT1x, (%rcx); + + call __blowfish_enc_blk32; + + /* dst = src ^ iv */ + vpxor 0*32(%rdx), RXr0, RXr0; + vpxor 1*32(%rdx), RXl0, RXl0; + vpxor 2*32(%rdx), RXr1, RXr1; + vpxor 3*32(%rdx), RXl1, RXl1; + vpxor 4*32(%rdx), RXr2, RXr2; + vpxor 5*32(%rdx), RXl2, RXl2; + vpxor 6*32(%rdx), RXr3, RXr3; + vpxor 7*32(%rdx), RXl3, RXl3; + vmovdqu RXr0, (0*32)(%rsi); + vmovdqu RXl0, (1*32)(%rsi); + vmovdqu RXr1, (2*32)(%rsi); + vmovdqu RXl1, (3*32)(%rsi); + vmovdqu RXr2, (4*32)(%rsi); + vmovdqu RXl2, (5*32)(%rsi); + vmovdqu RXr3, (6*32)(%rsi); + vmovdqu RXl3, (7*32)(%rsi); + + vzeroupper; + + ret; +ENDPROC(blowfish_ctr_32way) diff --git a/arch/x86/crypto/blowfish_avx2_glue.c b/arch/x86/crypto/blowfish_avx2_glue.c new file mode 100644 index 0000000..4417e9a --- /dev/null +++ b/arch/x86/crypto/blowfish_avx2_glue.c @@ -0,0 +1,585 @@ +/* + * Glue Code for x86_64/AVX2 assembler optimized version of Blowfish + * + * Copyright © 2012-2013 Jussi Kivilinna + * + * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: + * Copyright (c) 2006 Herbert Xu + * CTR part based on code (crypto/ctr.c) by: + * (C) Copyright IBM Corp. 2007 - Joy Latten + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BF_AVX2_PARALLEL_BLOCKS 32 + +/* 32-way AVX2 parallel cipher functions */ +asmlinkage void blowfish_ecb_enc_32way(struct bf_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void blowfish_ecb_dec_32way(struct bf_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void blowfish_cbc_dec_32way(struct bf_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void blowfish_ctr_32way(struct bf_ctx *ctx, u8 *dst, const u8 *src, + __be64 *iv); + +static inline bool bf_fpu_begin(bool fpu_enabled, unsigned int nbytes) +{ + if (fpu_enabled) + return true; + + /* FPU is only used when chunk to be processed is large enough, so + * do not enable FPU until it is necessary. + */ + if (nbytes < BF_BLOCK_SIZE * BF_AVX2_PARALLEL_BLOCKS) + return false; + + kernel_fpu_begin(); + return true; +} + +static inline void bf_fpu_end(bool fpu_enabled) +{ + if (fpu_enabled) + kernel_fpu_end(); +} + +static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, + bool enc) +{ + bool fpu_enabled = false; + struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + const unsigned int bsize = BF_BLOCK_SIZE; + unsigned int nbytes; + int err; + + err = blkcipher_walk_virt(desc, walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + while ((nbytes = walk->nbytes)) { + u8 *wsrc = walk->src.virt.addr; + u8 *wdst = walk->dst.virt.addr; + + fpu_enabled = bf_fpu_begin(fpu_enabled, nbytes); + + /* Process multi-block AVX2 batch */ + if (nbytes >= bsize * BF_AVX2_PARALLEL_BLOCKS) { + do { + if (enc) + blowfish_ecb_enc_32way(ctx, wdst, wsrc); + else + blowfish_ecb_dec_32way(ctx, wdst, wsrc); + + wsrc += bsize * BF_AVX2_PARALLEL_BLOCKS; + wdst += bsize * BF_AVX2_PARALLEL_BLOCKS; + nbytes -= bsize * BF_AVX2_PARALLEL_BLOCKS; + } while (nbytes >= bsize * BF_AVX2_PARALLEL_BLOCKS); + + if (nbytes < bsize) + goto done; + } + + /* Process multi-block batch */ + if (nbytes >= bsize * BF_PARALLEL_BLOCKS) { + do { + if (enc) + blowfish_enc_blk_4way(ctx, wdst, wsrc); + else + blowfish_dec_blk_4way(ctx, wdst, wsrc); + + wsrc += bsize * BF_PARALLEL_BLOCKS; + wdst += bsize * BF_PARALLEL_BLOCKS; + nbytes -= bsize * BF_PARALLEL_BLOCKS; + } while (nbytes >= bsize * BF_PARALLEL_BLOCKS); + + if (nbytes < bsize) + goto done; + } + + /* Handle leftovers */ + do { + if (enc) + blowfish_enc_blk(ctx, wdst, wsrc); + else + blowfish_dec_blk(ctx, wdst, wsrc); + + wsrc += bsize; + wdst += bsize; + nbytes -= bsize; + } while (nbytes >= bsize); + +done: + err = blkcipher_walk_done(desc, walk, nbytes); + } + + bf_fpu_end(fpu_enabled); + return err; +} + +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ecb_crypt(desc, &walk, true); +} + +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ecb_crypt(desc, &walk, false); +} + +static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int bsize = BF_BLOCK_SIZE; + unsigned int nbytes = walk->nbytes; + u64 *src = (u64 *)walk->src.virt.addr; + u64 *dst = (u64 *)walk->dst.virt.addr; + u64 *iv = (u64 *)walk->iv; + + do { + *dst = *src ^ *iv; + blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst); + iv = dst; + + src += 1; + dst += 1; + nbytes -= bsize; + } while (nbytes >= bsize); + + *(u64 *)walk->iv = *iv; + return nbytes; +} + +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while ((nbytes = walk.nbytes)) { + nbytes = __cbc_encrypt(desc, &walk); + err = blkcipher_walk_done(desc, &walk, nbytes); + } + + return err; +} + +static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + const unsigned int bsize = BF_BLOCK_SIZE; + unsigned int nbytes = walk->nbytes; + u64 *src = (u64 *)walk->src.virt.addr; + u64 *dst = (u64 *)walk->dst.virt.addr; + u64 last_iv; + int i; + + /* Start of the last block. */ + src += nbytes / bsize - 1; + dst += nbytes / bsize - 1; + + last_iv = *src; + + /* Process multi-block AVX2 batch */ + if (nbytes >= bsize * BF_AVX2_PARALLEL_BLOCKS) { + do { + nbytes -= bsize * (BF_AVX2_PARALLEL_BLOCKS - 1); + src -= BF_AVX2_PARALLEL_BLOCKS - 1; + dst -= BF_AVX2_PARALLEL_BLOCKS - 1; + + blowfish_cbc_dec_32way(ctx, (u8 *)dst, (u8 *)src); + + nbytes -= bsize; + if (nbytes < bsize) + goto done; + + *dst ^= *(src - 1); + src -= 1; + dst -= 1; + } while (nbytes >= bsize * BF_AVX2_PARALLEL_BLOCKS); + + if (nbytes < bsize) + goto done; + } + + /* Process multi-block batch */ + if (nbytes >= bsize * BF_PARALLEL_BLOCKS) { + u64 ivs[BF_PARALLEL_BLOCKS - 1]; + + do { + nbytes -= bsize * (BF_PARALLEL_BLOCKS - 1); + src -= BF_PARALLEL_BLOCKS - 1; + dst -= BF_PARALLEL_BLOCKS - 1; + + for (i = 0; i < BF_PARALLEL_BLOCKS - 1; i++) + ivs[i] = src[i]; + + blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src); + + for (i = 0; i < BF_PARALLEL_BLOCKS - 1; i++) + dst[i + 1] ^= ivs[i]; + + nbytes -= bsize; + if (nbytes < bsize) + goto done; + + *dst ^= *(src - 1); + src -= 1; + dst -= 1; + } while (nbytes >= bsize * BF_PARALLEL_BLOCKS); + + if (nbytes < bsize) + goto done; + } + + /* Handle leftovers */ + for (;;) { + blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src); + + nbytes -= bsize; + if (nbytes < bsize) + break; + + *dst ^= *(src - 1); + src -= 1; + dst -= 1; + } + +done: + *dst ^= *(u64 *)walk->iv; + *(u64 *)walk->iv = last_iv; + + return nbytes; +} + +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + bool fpu_enabled = false; + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + while ((nbytes = walk.nbytes)) { + fpu_enabled = bf_fpu_begin(fpu_enabled, nbytes); + nbytes = __cbc_decrypt(desc, &walk); + err = blkcipher_walk_done(desc, &walk, nbytes); + } + + bf_fpu_end(fpu_enabled); + return err; +} + +static void ctr_crypt_final(struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + u8 *ctrblk = walk->iv; + u8 keystream[BF_BLOCK_SIZE]; + u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; + unsigned int nbytes = walk->nbytes; + + blowfish_enc_blk(ctx, keystream, ctrblk); + crypto_xor(keystream, src, nbytes); + memcpy(dst, keystream, nbytes); + + crypto_inc(ctrblk, BF_BLOCK_SIZE); +} + +static unsigned int __ctr_crypt(struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int bsize = BF_BLOCK_SIZE; + unsigned int nbytes = walk->nbytes; + u64 *src = (u64 *)walk->src.virt.addr; + u64 *dst = (u64 *)walk->dst.virt.addr; + int i; + + /* Process multi-block AVX2 batch */ + if (nbytes >= bsize * BF_AVX2_PARALLEL_BLOCKS) { + do { + blowfish_ctr_32way(ctx, (u8 *)dst, (u8 *)src, + (__be64 *)walk->iv); + + src += BF_AVX2_PARALLEL_BLOCKS; + dst += BF_AVX2_PARALLEL_BLOCKS; + nbytes -= bsize * BF_AVX2_PARALLEL_BLOCKS; + } while (nbytes >= bsize * BF_AVX2_PARALLEL_BLOCKS); + + if (nbytes < bsize) + goto done; + } + + /* Process four block batch */ + if (nbytes >= bsize * BF_PARALLEL_BLOCKS) { + __be64 ctrblocks[BF_PARALLEL_BLOCKS]; + u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv); + + do { + /* create ctrblks for parallel encrypt */ + for (i = 0; i < BF_PARALLEL_BLOCKS; i++) { + if (dst != src) + dst[i] = src[i]; + + ctrblocks[i] = cpu_to_be64(ctrblk++); + } + + blowfish_enc_blk_xor_4way(ctx, (u8 *)dst, + (u8 *)ctrblocks); + + src += BF_PARALLEL_BLOCKS; + dst += BF_PARALLEL_BLOCKS; + nbytes -= bsize * BF_PARALLEL_BLOCKS; + } while (nbytes >= bsize * BF_PARALLEL_BLOCKS); + + *(__be64 *)walk->iv = cpu_to_be64(ctrblk); + + if (nbytes < bsize) + goto done; + } + + /* Handle leftovers */ + do { + u64 ctrblk; + + if (dst != src) + *dst = *src; + + ctrblk = *(u64 *)walk->iv; + be64_add_cpu((__be64 *)walk->iv, 1); + + blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk); + + src += 1; + dst += 1; + } while ((nbytes -= bsize) >= bsize); + +done: + return nbytes; +} + +static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + bool fpu_enabled = false; + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) { + fpu_enabled = bf_fpu_begin(fpu_enabled, nbytes); + nbytes = __ctr_crypt(desc, &walk); + err = blkcipher_walk_done(desc, &walk, nbytes); + } + + bf_fpu_end(fpu_enabled); + + if (walk.nbytes) { + ctr_crypt_final(desc, &walk); + err = blkcipher_walk_done(desc, &walk, 0); + } + + return err; +} + +static struct crypto_alg bf_algs[6] = { { + .cra_name = "__ecb-blowfish-avx2", + .cra_driver_name = "__driver-ecb-blowfish-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = BF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct bf_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = BF_MIN_KEY_SIZE, + .max_keysize = BF_MAX_KEY_SIZE, + .setkey = blowfish_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "__cbc-blowfish-avx2", + .cra_driver_name = "__driver-cbc-blowfish-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = BF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct bf_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = BF_MIN_KEY_SIZE, + .max_keysize = BF_MAX_KEY_SIZE, + .setkey = blowfish_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "__ctr-blowfish-avx2", + .cra_driver_name = "__driver-ctr-blowfish-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct bf_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = BF_MIN_KEY_SIZE, + .max_keysize = BF_MAX_KEY_SIZE, + .ivsize = BF_BLOCK_SIZE, + .setkey = blowfish_setkey, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + }, + }, +}, { + .cra_name = "ecb(blowfish)", + .cra_driver_name = "ecb-blowfish-avx2", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = BF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = BF_MIN_KEY_SIZE, + .max_keysize = BF_MAX_KEY_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "cbc(blowfish)", + .cra_driver_name = "cbc-blowfish-avx2", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = BF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = BF_MIN_KEY_SIZE, + .max_keysize = BF_MAX_KEY_SIZE, + .ivsize = BF_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = __ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "ctr(blowfish)", + .cra_driver_name = "ctr-blowfish-avx2", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = BF_MIN_KEY_SIZE, + .max_keysize = BF_MAX_KEY_SIZE, + .ivsize = BF_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_encrypt, + .geniv = "chainiv", + }, + }, +} }; + + +static int __init init(void) +{ + u64 xcr0; + + if (!cpu_has_avx2 || !cpu_has_osxsave) { + pr_info("AVX2 instructions are not detected.\n"); + return -ENODEV; + } + + xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); + if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { + pr_info("AVX detected but unusable.\n"); + return -ENODEV; + } + + return crypto_register_algs(bf_algs, ARRAY_SIZE(bf_algs)); +} + +static void __exit fini(void) +{ + crypto_unregister_algs(bf_algs, ARRAY_SIZE(bf_algs)); +} + +module_init(init); +module_exit(fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Blowfish Cipher Algorithm, AVX2 optimized"); +MODULE_ALIAS("blowfish"); +MODULE_ALIAS("blowfish-asm"); diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c index 50ec333..3548d76 100644 --- a/arch/x86/crypto/blowfish_glue.c +++ b/arch/x86/crypto/blowfish_glue.c @@ -1,7 +1,7 @@ /* * Glue Code for assembler optimized version of Blowfish * - * Copyright (c) 2011 Jussi Kivilinna + * Copyright © 2011-2013 Jussi Kivilinna * * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: * Copyright (c) 2006 Herbert Xu @@ -32,40 +32,24 @@ #include #include #include +#include /* regular block cipher functions */ asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src, bool xor); +EXPORT_SYMBOL_GPL(__blowfish_enc_blk); + asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src); +EXPORT_SYMBOL_GPL(blowfish_dec_blk); /* 4-way parallel cipher functions */ asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, const u8 *src, bool xor); +EXPORT_SYMBOL_GPL(__blowfish_enc_blk_4way); + asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst, const u8 *src); - -static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src) -{ - __blowfish_enc_blk(ctx, dst, src, false); -} - -static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst, - const u8 *src) -{ - __blowfish_enc_blk(ctx, dst, src, true); -} - -static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, - const u8 *src) -{ - __blowfish_enc_blk_4way(ctx, dst, src, false); -} - -static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst, - const u8 *src) -{ - __blowfish_enc_blk_4way(ctx, dst, src, true); -} +EXPORT_SYMBOL_GPL(blowfish_dec_blk_4way); static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 93fe929..1243272 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -278,6 +278,7 @@ extern const char * const x86_power_flags[32]; #define cpu_has_ssse3 boot_cpu_has(X86_FEATURE_SSSE3) #define cpu_has_aes boot_cpu_has(X86_FEATURE_AES) #define cpu_has_avx boot_cpu_has(X86_FEATURE_AVX) +#define cpu_has_avx2 boot_cpu_has(X86_FEATURE_AVX2) #define cpu_has_ht boot_cpu_has(X86_FEATURE_HT) #define cpu_has_mp boot_cpu_has(X86_FEATURE_MP) #define cpu_has_nx boot_cpu_has(X86_FEATURE_NX) diff --git a/arch/x86/include/asm/crypto/blowfish.h b/arch/x86/include/asm/crypto/blowfish.h new file mode 100644 index 0000000..f097b2f --- /dev/null +++ b/arch/x86/include/asm/crypto/blowfish.h @@ -0,0 +1,43 @@ +#ifndef ASM_X86_BLOWFISH_H +#define ASM_X86_BLOWFISH_H + +#include +#include + +#define BF_PARALLEL_BLOCKS 4 + +/* regular block cipher functions */ +asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src, + bool xor); +asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src); + +/* 4-way parallel cipher functions */ +asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, + const u8 *src, bool xor); +asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst, + const u8 *src); + +static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src) +{ + __blowfish_enc_blk(ctx, dst, src, false); +} + +static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst, + const u8 *src) +{ + __blowfish_enc_blk(ctx, dst, src, true); +} + +static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, + const u8 *src) +{ + __blowfish_enc_blk_4way(ctx, dst, src, false); +} + +static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst, + const u8 *src) +{ + __blowfish_enc_blk_4way(ctx, dst, src, true); +} + +#endif diff --git a/crypto/Kconfig b/crypto/Kconfig index 0e7a237..6b9564f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -820,6 +820,24 @@ config CRYPTO_BLOWFISH_X86_64 See also: +config CRYPTO_BLOWFISH_AVX2_X86_64 + tristate "Blowfish cipher algorithm (x86_64/AVX2)" + depends on X86 && 64BIT + select CRYPTO_ALGAPI + select CRYPTO_CRYPTD + select CRYPTO_ABLK_HELPER_X86 + select CRYPTO_BLOWFISH_COMMON + select CRYPTO_BLOWFISH_X86_64 + help + Blowfish cipher algorithm (x86_64/AVX2), by Bruce Schneier. + + This is a variable key length cipher which can use keys from 32 + bits to 448 bits in length. It's fast, simple and specifically + designed for use on "large microprocessors". + + See also: + + config CRYPTO_CAMELLIA tristate "Camellia cipher algorithms" depends on CRYPTO diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 3807084..f3effb42 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1655,6 +1655,9 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + .alg = "__driver-cbc-blowfish-avx2", + .test = alg_test_null, + }, { .alg = "__driver-cbc-camellia-aesni", .test = alg_test_null, }, { @@ -1677,6 +1680,9 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + .alg = "__driver-ecb-blowfish-avx2", + .test = alg_test_null, + }, { .alg = "__driver-ecb-camellia-aesni", .test = alg_test_null, }, { @@ -1948,6 +1954,9 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + .alg = "cryptd(__driver-cbc-blowfish-avx2)", + .test = alg_test_null, + }, { .alg = "cryptd(__driver-cbc-camellia-aesni)", .test = alg_test_null, }, { @@ -1955,6 +1964,9 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + .alg = "cryptd(__driver-ecb-blowfish-avx2)", + .test = alg_test_null, + }, { .alg = "cryptd(__driver-ecb-camellia-aesni)", .test = alg_test_null, }, { -- cgit v0.10.2 From cf1521a1a5e21fd1e79a458605c4282fbfbbeee2 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sat, 13 Apr 2013 13:46:50 +0300 Subject: crypto: twofish - add AVX2/x86_64 assembler implementation of twofish cipher Patch adds AVX2/x86-64 implementation of Twofish cipher, requiring 16 parallel blocks for input (256 bytes). Table look-ups are performed using vpgatherdd instruction directly from vector registers and thus should be faster than earlier implementations. Implementation also uses 256-bit wide YMM registers, which should give additional speed up compared to the AVX implementation. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 28464ef..1f6e0c2 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -43,6 +43,7 @@ endif # These modules require assembler to support AVX2. ifeq ($(avx2_supported),yes) obj-$(CONFIG_CRYPTO_BLOWFISH_AVX2_X86_64) += blowfish-avx2.o + obj-$(CONFIG_CRYPTO_TWOFISH_AVX2_X86_64) += twofish-avx2.o endif aes-i586-y := aes-i586-asm_32.o aes_glue.o @@ -71,6 +72,7 @@ endif ifeq ($(avx2_supported),yes) blowfish-avx2-y := blowfish-avx2-asm_64.o blowfish_avx2_glue.o + twofish-avx2-y := twofish-avx2-asm_64.o twofish_avx2_glue.o endif aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o diff --git a/arch/x86/crypto/glue_helper-asm-avx2.S b/arch/x86/crypto/glue_helper-asm-avx2.S new file mode 100644 index 0000000..a53ac11 --- /dev/null +++ b/arch/x86/crypto/glue_helper-asm-avx2.S @@ -0,0 +1,180 @@ +/* + * Shared glue code for 128bit block ciphers, AVX2 assembler macros + * + * Copyright © 2012-2013 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#define load_16way(src, x0, x1, x2, x3, x4, x5, x6, x7) \ + vmovdqu (0*32)(src), x0; \ + vmovdqu (1*32)(src), x1; \ + vmovdqu (2*32)(src), x2; \ + vmovdqu (3*32)(src), x3; \ + vmovdqu (4*32)(src), x4; \ + vmovdqu (5*32)(src), x5; \ + vmovdqu (6*32)(src), x6; \ + vmovdqu (7*32)(src), x7; + +#define store_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7) \ + vmovdqu x0, (0*32)(dst); \ + vmovdqu x1, (1*32)(dst); \ + vmovdqu x2, (2*32)(dst); \ + vmovdqu x3, (3*32)(dst); \ + vmovdqu x4, (4*32)(dst); \ + vmovdqu x5, (5*32)(dst); \ + vmovdqu x6, (6*32)(dst); \ + vmovdqu x7, (7*32)(dst); + +#define store_cbc_16way(src, dst, x0, x1, x2, x3, x4, x5, x6, x7, t0) \ + vpxor t0, t0, t0; \ + vinserti128 $1, (src), t0, t0; \ + vpxor t0, x0, x0; \ + vpxor (0*32+16)(src), x1, x1; \ + vpxor (1*32+16)(src), x2, x2; \ + vpxor (2*32+16)(src), x3, x3; \ + vpxor (3*32+16)(src), x4, x4; \ + vpxor (4*32+16)(src), x5, x5; \ + vpxor (5*32+16)(src), x6, x6; \ + vpxor (6*32+16)(src), x7, x7; \ + store_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7); + +#define inc_le128(x, minus_one, tmp) \ + vpcmpeqq minus_one, x, tmp; \ + vpsubq minus_one, x, x; \ + vpslldq $8, tmp, tmp; \ + vpsubq tmp, x, x; + +#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ + vpcmpeqq minus_one, x, tmp1; \ + vpcmpeqq minus_two, x, tmp2; \ + vpsubq minus_two, x, x; \ + vpor tmp2, tmp1, tmp1; \ + vpslldq $8, tmp1, tmp1; \ + vpsubq tmp1, x, x; + +#define load_ctr_16way(iv, bswap, x0, x1, x2, x3, x4, x5, x6, x7, t0, t0x, t1, \ + t1x, t2, t2x, t3, t3x, t4, t5) \ + vpcmpeqd t0, t0, t0; \ + vpsrldq $8, t0, t0; /* ab: -1:0 ; cd: -1:0 */ \ + vpaddq t0, t0, t4; /* ab: -2:0 ; cd: -2:0 */\ + \ + /* load IV and byteswap */ \ + vmovdqu (iv), t2x; \ + vmovdqa t2x, t3x; \ + inc_le128(t2x, t0x, t1x); \ + vbroadcasti128 bswap, t1; \ + vinserti128 $1, t2x, t3, t2; /* ab: le0 ; cd: le1 */ \ + vpshufb t1, t2, x0; \ + \ + /* construct IVs */ \ + add2_le128(t2, t0, t4, t3, t5); /* ab: le2 ; cd: le3 */ \ + vpshufb t1, t2, x1; \ + add2_le128(t2, t0, t4, t3, t5); \ + vpshufb t1, t2, x2; \ + add2_le128(t2, t0, t4, t3, t5); \ + vpshufb t1, t2, x3; \ + add2_le128(t2, t0, t4, t3, t5); \ + vpshufb t1, t2, x4; \ + add2_le128(t2, t0, t4, t3, t5); \ + vpshufb t1, t2, x5; \ + add2_le128(t2, t0, t4, t3, t5); \ + vpshufb t1, t2, x6; \ + add2_le128(t2, t0, t4, t3, t5); \ + vpshufb t1, t2, x7; \ + vextracti128 $1, t2, t2x; \ + inc_le128(t2x, t0x, t3x); \ + vmovdqu t2x, (iv); + +#define store_ctr_16way(src, dst, x0, x1, x2, x3, x4, x5, x6, x7) \ + vpxor (0*32)(src), x0, x0; \ + vpxor (1*32)(src), x1, x1; \ + vpxor (2*32)(src), x2, x2; \ + vpxor (3*32)(src), x3, x3; \ + vpxor (4*32)(src), x4, x4; \ + vpxor (5*32)(src), x5, x5; \ + vpxor (6*32)(src), x6, x6; \ + vpxor (7*32)(src), x7, x7; \ + store_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7); + +#define gf128mul_x_ble(iv, mask, tmp) \ + vpsrad $31, iv, tmp; \ + vpaddq iv, iv, iv; \ + vpshufd $0x13, tmp, tmp; \ + vpand mask, tmp, tmp; \ + vpxor tmp, iv, iv; + +#define gf128mul_x2_ble(iv, mask1, mask2, tmp0, tmp1) \ + vpsrad $31, iv, tmp0; \ + vpaddq iv, iv, tmp1; \ + vpsllq $2, iv, iv; \ + vpshufd $0x13, tmp0, tmp0; \ + vpsrad $31, tmp1, tmp1; \ + vpand mask2, tmp0, tmp0; \ + vpshufd $0x13, tmp1, tmp1; \ + vpxor tmp0, iv, iv; \ + vpand mask1, tmp1, tmp1; \ + vpxor tmp1, iv, iv; + +#define load_xts_16way(iv, src, dst, x0, x1, x2, x3, x4, x5, x6, x7, tiv, \ + tivx, t0, t0x, t1, t1x, t2, t2x, t3, \ + xts_gf128mul_and_shl1_mask_0, \ + xts_gf128mul_and_shl1_mask_1) \ + vbroadcasti128 xts_gf128mul_and_shl1_mask_0, t1; \ + \ + /* load IV and construct second IV */ \ + vmovdqu (iv), tivx; \ + vmovdqa tivx, t0x; \ + gf128mul_x_ble(tivx, t1x, t2x); \ + vbroadcasti128 xts_gf128mul_and_shl1_mask_1, t2; \ + vinserti128 $1, tivx, t0, tiv; \ + vpxor (0*32)(src), tiv, x0; \ + vmovdqu tiv, (0*32)(dst); \ + \ + /* construct and store IVs, also xor with source */ \ + gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ + vpxor (1*32)(src), tiv, x1; \ + vmovdqu tiv, (1*32)(dst); \ + \ + gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ + vpxor (2*32)(src), tiv, x2; \ + vmovdqu tiv, (2*32)(dst); \ + \ + gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ + vpxor (3*32)(src), tiv, x3; \ + vmovdqu tiv, (3*32)(dst); \ + \ + gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ + vpxor (4*32)(src), tiv, x4; \ + vmovdqu tiv, (4*32)(dst); \ + \ + gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ + vpxor (5*32)(src), tiv, x5; \ + vmovdqu tiv, (5*32)(dst); \ + \ + gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ + vpxor (6*32)(src), tiv, x6; \ + vmovdqu tiv, (6*32)(dst); \ + \ + gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ + vpxor (7*32)(src), tiv, x7; \ + vmovdqu tiv, (7*32)(dst); \ + \ + vextracti128 $1, tiv, tivx; \ + gf128mul_x_ble(tivx, t1x, t2x); \ + vmovdqu tivx, (iv); + +#define store_xts_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7) \ + vpxor (0*32)(dst), x0, x0; \ + vpxor (1*32)(dst), x1, x1; \ + vpxor (2*32)(dst), x2, x2; \ + vpxor (3*32)(dst), x3, x3; \ + vpxor (4*32)(dst), x4, x4; \ + vpxor (5*32)(dst), x5, x5; \ + vpxor (6*32)(dst), x6, x6; \ + vpxor (7*32)(dst), x7, x7; \ + store_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7); diff --git a/arch/x86/crypto/twofish-avx2-asm_64.S b/arch/x86/crypto/twofish-avx2-asm_64.S new file mode 100644 index 0000000..e1a83b9 --- /dev/null +++ b/arch/x86/crypto/twofish-avx2-asm_64.S @@ -0,0 +1,600 @@ +/* + * x86_64/AVX2 assembler optimized version of Twofish + * + * Copyright © 2012-2013 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#include +#include "glue_helper-asm-avx2.S" + +.file "twofish-avx2-asm_64.S" + +.data +.align 16 + +.Lvpshufb_mask0: +.long 0x80808000 +.long 0x80808004 +.long 0x80808008 +.long 0x8080800c + +.Lbswap128_mask: + .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +.Lxts_gf128mul_and_shl1_mask_0: + .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 +.Lxts_gf128mul_and_shl1_mask_1: + .byte 0x0e, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0 + +.text + +/* structure of crypto context */ +#define s0 0 +#define s1 1024 +#define s2 2048 +#define s3 3072 +#define w 4096 +#define k 4128 + +/* register macros */ +#define CTX %rdi + +#define RS0 CTX +#define RS1 %r8 +#define RS2 %r9 +#define RS3 %r10 +#define RK %r11 +#define RW %rax +#define RROUND %r12 +#define RROUNDd %r12d + +#define RA0 %ymm8 +#define RB0 %ymm9 +#define RC0 %ymm10 +#define RD0 %ymm11 +#define RA1 %ymm12 +#define RB1 %ymm13 +#define RC1 %ymm14 +#define RD1 %ymm15 + +/* temp regs */ +#define RX0 %ymm0 +#define RY0 %ymm1 +#define RX1 %ymm2 +#define RY1 %ymm3 +#define RT0 %ymm4 +#define RIDX %ymm5 + +#define RX0x %xmm0 +#define RY0x %xmm1 +#define RX1x %xmm2 +#define RY1x %xmm3 +#define RT0x %xmm4 + +/* vpgatherdd mask and '-1' */ +#define RNOT %ymm6 + +/* byte mask, (-1 >> 24) */ +#define RBYTE %ymm7 + +/********************************************************************** + 16-way AVX2 twofish + **********************************************************************/ +#define init_round_constants() \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpsrld $24, RNOT, RBYTE; \ + leaq k(CTX), RK; \ + leaq w(CTX), RW; \ + leaq s1(CTX), RS1; \ + leaq s2(CTX), RS2; \ + leaq s3(CTX), RS3; \ + +#define g16(ab, rs0, rs1, rs2, rs3, xy) \ + vpand RBYTE, ab ## 0, RIDX; \ + vpgatherdd RNOT, (rs0, RIDX, 4), xy ## 0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + \ + vpand RBYTE, ab ## 1, RIDX; \ + vpgatherdd RNOT, (rs0, RIDX, 4), xy ## 1; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + \ + vpsrld $8, ab ## 0, RIDX; \ + vpand RBYTE, RIDX, RIDX; \ + vpgatherdd RNOT, (rs1, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 0, xy ## 0; \ + \ + vpsrld $8, ab ## 1, RIDX; \ + vpand RBYTE, RIDX, RIDX; \ + vpgatherdd RNOT, (rs1, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 1, xy ## 1; \ + \ + vpsrld $16, ab ## 0, RIDX; \ + vpand RBYTE, RIDX, RIDX; \ + vpgatherdd RNOT, (rs2, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 0, xy ## 0; \ + \ + vpsrld $16, ab ## 1, RIDX; \ + vpand RBYTE, RIDX, RIDX; \ + vpgatherdd RNOT, (rs2, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 1, xy ## 1; \ + \ + vpsrld $24, ab ## 0, RIDX; \ + vpgatherdd RNOT, (rs3, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 0, xy ## 0; \ + \ + vpsrld $24, ab ## 1, RIDX; \ + vpgatherdd RNOT, (rs3, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 1, xy ## 1; + +#define g1_16(a, x) \ + g16(a, RS0, RS1, RS2, RS3, x); + +#define g2_16(b, y) \ + g16(b, RS1, RS2, RS3, RS0, y); + +#define encrypt_round_end16(a, b, c, d, nk) \ + vpaddd RY0, RX0, RX0; \ + vpaddd RX0, RY0, RY0; \ + vpbroadcastd nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RX0, RX0; \ + vpbroadcastd 4+nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RY0, RY0; \ + \ + vpxor RY0, d ## 0, d ## 0; \ + \ + vpxor RX0, c ## 0, c ## 0; \ + vpsrld $1, c ## 0, RT0; \ + vpslld $31, c ## 0, c ## 0; \ + vpor RT0, c ## 0, c ## 0; \ + \ + vpaddd RY1, RX1, RX1; \ + vpaddd RX1, RY1, RY1; \ + vpbroadcastd nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RX1, RX1; \ + vpbroadcastd 4+nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RY1, RY1; \ + \ + vpxor RY1, d ## 1, d ## 1; \ + \ + vpxor RX1, c ## 1, c ## 1; \ + vpsrld $1, c ## 1, RT0; \ + vpslld $31, c ## 1, c ## 1; \ + vpor RT0, c ## 1, c ## 1; \ + +#define encrypt_round16(a, b, c, d, nk) \ + g2_16(b, RY); \ + \ + vpslld $1, b ## 0, RT0; \ + vpsrld $31, b ## 0, b ## 0; \ + vpor RT0, b ## 0, b ## 0; \ + \ + vpslld $1, b ## 1, RT0; \ + vpsrld $31, b ## 1, b ## 1; \ + vpor RT0, b ## 1, b ## 1; \ + \ + g1_16(a, RX); \ + \ + encrypt_round_end16(a, b, c, d, nk); + +#define encrypt_round_first16(a, b, c, d, nk) \ + vpslld $1, d ## 0, RT0; \ + vpsrld $31, d ## 0, d ## 0; \ + vpor RT0, d ## 0, d ## 0; \ + \ + vpslld $1, d ## 1, RT0; \ + vpsrld $31, d ## 1, d ## 1; \ + vpor RT0, d ## 1, d ## 1; \ + \ + encrypt_round16(a, b, c, d, nk); + +#define encrypt_round_last16(a, b, c, d, nk) \ + g2_16(b, RY); \ + \ + g1_16(a, RX); \ + \ + encrypt_round_end16(a, b, c, d, nk); + +#define decrypt_round_end16(a, b, c, d, nk) \ + vpaddd RY0, RX0, RX0; \ + vpaddd RX0, RY0, RY0; \ + vpbroadcastd nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RX0, RX0; \ + vpbroadcastd 4+nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RY0, RY0; \ + \ + vpxor RX0, c ## 0, c ## 0; \ + \ + vpxor RY0, d ## 0, d ## 0; \ + vpsrld $1, d ## 0, RT0; \ + vpslld $31, d ## 0, d ## 0; \ + vpor RT0, d ## 0, d ## 0; \ + \ + vpaddd RY1, RX1, RX1; \ + vpaddd RX1, RY1, RY1; \ + vpbroadcastd nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RX1, RX1; \ + vpbroadcastd 4+nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RY1, RY1; \ + \ + vpxor RX1, c ## 1, c ## 1; \ + \ + vpxor RY1, d ## 1, d ## 1; \ + vpsrld $1, d ## 1, RT0; \ + vpslld $31, d ## 1, d ## 1; \ + vpor RT0, d ## 1, d ## 1; + +#define decrypt_round16(a, b, c, d, nk) \ + g1_16(a, RX); \ + \ + vpslld $1, a ## 0, RT0; \ + vpsrld $31, a ## 0, a ## 0; \ + vpor RT0, a ## 0, a ## 0; \ + \ + vpslld $1, a ## 1, RT0; \ + vpsrld $31, a ## 1, a ## 1; \ + vpor RT0, a ## 1, a ## 1; \ + \ + g2_16(b, RY); \ + \ + decrypt_round_end16(a, b, c, d, nk); + +#define decrypt_round_first16(a, b, c, d, nk) \ + vpslld $1, c ## 0, RT0; \ + vpsrld $31, c ## 0, c ## 0; \ + vpor RT0, c ## 0, c ## 0; \ + \ + vpslld $1, c ## 1, RT0; \ + vpsrld $31, c ## 1, c ## 1; \ + vpor RT0, c ## 1, c ## 1; \ + \ + decrypt_round16(a, b, c, d, nk) + +#define decrypt_round_last16(a, b, c, d, nk) \ + g1_16(a, RX); \ + \ + g2_16(b, RY); \ + \ + decrypt_round_end16(a, b, c, d, nk); + +#define encrypt_cycle16() \ + encrypt_round16(RA, RB, RC, RD, 0); \ + encrypt_round16(RC, RD, RA, RB, 8); + +#define encrypt_cycle_first16() \ + encrypt_round_first16(RA, RB, RC, RD, 0); \ + encrypt_round16(RC, RD, RA, RB, 8); + +#define encrypt_cycle_last16() \ + encrypt_round16(RA, RB, RC, RD, 0); \ + encrypt_round_last16(RC, RD, RA, RB, 8); + +#define decrypt_cycle16(n) \ + decrypt_round16(RC, RD, RA, RB, 8); \ + decrypt_round16(RA, RB, RC, RD, 0); + +#define decrypt_cycle_first16(n) \ + decrypt_round_first16(RC, RD, RA, RB, 8); \ + decrypt_round16(RA, RB, RC, RD, 0); + +#define decrypt_cycle_last16(n) \ + decrypt_round16(RC, RD, RA, RB, 8); \ + decrypt_round_last16(RA, RB, RC, RD, 0); + +#define transpose_4x4(x0,x1,x2,x3,t1,t2) \ + vpunpckhdq x1, x0, t2; \ + vpunpckldq x1, x0, x0; \ + \ + vpunpckldq x3, x2, t1; \ + vpunpckhdq x3, x2, x2; \ + \ + vpunpckhqdq t1, x0, x1; \ + vpunpcklqdq t1, x0, x0; \ + \ + vpunpckhqdq x2, t2, x3; \ + vpunpcklqdq x2, t2, x2; + +#define read_blocks8(offs,a,b,c,d) \ + transpose_4x4(a, b, c, d, RX0, RY0); + +#define write_blocks8(offs,a,b,c,d) \ + transpose_4x4(a, b, c, d, RX0, RY0); + +#define inpack_enc8(a,b,c,d) \ + vpbroadcastd 4*0(RW), RT0; \ + vpxor RT0, a, a; \ + \ + vpbroadcastd 4*1(RW), RT0; \ + vpxor RT0, b, b; \ + \ + vpbroadcastd 4*2(RW), RT0; \ + vpxor RT0, c, c; \ + \ + vpbroadcastd 4*3(RW), RT0; \ + vpxor RT0, d, d; + +#define outunpack_enc8(a,b,c,d) \ + vpbroadcastd 4*4(RW), RX0; \ + vpbroadcastd 4*5(RW), RY0; \ + vpxor RX0, c, RX0; \ + vpxor RY0, d, RY0; \ + \ + vpbroadcastd 4*6(RW), RT0; \ + vpxor RT0, a, c; \ + vpbroadcastd 4*7(RW), RT0; \ + vpxor RT0, b, d; \ + \ + vmovdqa RX0, a; \ + vmovdqa RY0, b; + +#define inpack_dec8(a,b,c,d) \ + vpbroadcastd 4*4(RW), RX0; \ + vpbroadcastd 4*5(RW), RY0; \ + vpxor RX0, a, RX0; \ + vpxor RY0, b, RY0; \ + \ + vpbroadcastd 4*6(RW), RT0; \ + vpxor RT0, c, a; \ + vpbroadcastd 4*7(RW), RT0; \ + vpxor RT0, d, b; \ + \ + vmovdqa RX0, c; \ + vmovdqa RY0, d; + +#define outunpack_dec8(a,b,c,d) \ + vpbroadcastd 4*0(RW), RT0; \ + vpxor RT0, a, a; \ + \ + vpbroadcastd 4*1(RW), RT0; \ + vpxor RT0, b, b; \ + \ + vpbroadcastd 4*2(RW), RT0; \ + vpxor RT0, c, c; \ + \ + vpbroadcastd 4*3(RW), RT0; \ + vpxor RT0, d, d; + +#define read_blocks16(a,b,c,d) \ + read_blocks8(0, a ## 0, b ## 0, c ## 0, d ## 0); \ + read_blocks8(8, a ## 1, b ## 1, c ## 1, d ## 1); + +#define write_blocks16(a,b,c,d) \ + write_blocks8(0, a ## 0, b ## 0, c ## 0, d ## 0); \ + write_blocks8(8, a ## 1, b ## 1, c ## 1, d ## 1); + +#define xor_blocks16(a,b,c,d) \ + xor_blocks8(0, a ## 0, b ## 0, c ## 0, d ## 0); \ + xor_blocks8(8, a ## 1, b ## 1, c ## 1, d ## 1); + +#define inpack_enc16(a,b,c,d) \ + inpack_enc8(a ## 0, b ## 0, c ## 0, d ## 0); \ + inpack_enc8(a ## 1, b ## 1, c ## 1, d ## 1); + +#define outunpack_enc16(a,b,c,d) \ + outunpack_enc8(a ## 0, b ## 0, c ## 0, d ## 0); \ + outunpack_enc8(a ## 1, b ## 1, c ## 1, d ## 1); + +#define inpack_dec16(a,b,c,d) \ + inpack_dec8(a ## 0, b ## 0, c ## 0, d ## 0); \ + inpack_dec8(a ## 1, b ## 1, c ## 1, d ## 1); + +#define outunpack_dec16(a,b,c,d) \ + outunpack_dec8(a ## 0, b ## 0, c ## 0, d ## 0); \ + outunpack_dec8(a ## 1, b ## 1, c ## 1, d ## 1); + +.align 8 +__twofish_enc_blk16: + /* input: + * %rdi: ctx, CTX + * RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1: plaintext + * output: + * RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1: ciphertext + */ + init_round_constants(); + + read_blocks16(RA, RB, RC, RD); + inpack_enc16(RA, RB, RC, RD); + + xorl RROUNDd, RROUNDd; + encrypt_cycle_first16(); + movl $2, RROUNDd; + +.align 4 +.L__enc_loop: + encrypt_cycle16(); + + addl $2, RROUNDd; + cmpl $14, RROUNDd; + jne .L__enc_loop; + + encrypt_cycle_last16(); + + outunpack_enc16(RA, RB, RC, RD); + write_blocks16(RA, RB, RC, RD); + + ret; +ENDPROC(__twofish_enc_blk16) + +.align 8 +__twofish_dec_blk16: + /* input: + * %rdi: ctx, CTX + * RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1: ciphertext + * output: + * RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1: plaintext + */ + init_round_constants(); + + read_blocks16(RA, RB, RC, RD); + inpack_dec16(RA, RB, RC, RD); + + movl $14, RROUNDd; + decrypt_cycle_first16(); + movl $12, RROUNDd; + +.align 4 +.L__dec_loop: + decrypt_cycle16(); + + addl $-2, RROUNDd; + jnz .L__dec_loop; + + decrypt_cycle_last16(); + + outunpack_dec16(RA, RB, RC, RD); + write_blocks16(RA, RB, RC, RD); + + ret; +ENDPROC(__twofish_dec_blk16) + +ENTRY(twofish_ecb_enc_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + pushq %r12; + + load_16way(%rdx, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + call __twofish_enc_blk16; + + store_16way(%rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + popq %r12; + vzeroupper; + + ret; +ENDPROC(twofish_ecb_enc_16way) + +ENTRY(twofish_ecb_dec_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + pushq %r12; + + load_16way(%rdx, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + call __twofish_dec_blk16; + + store_16way(%rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + popq %r12; + vzeroupper; + + ret; +ENDPROC(twofish_ecb_dec_16way) + +ENTRY(twofish_cbc_dec_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + pushq %r12; + + load_16way(%rdx, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + call __twofish_dec_blk16; + + store_cbc_16way(%rdx, %rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1, + RX0); + + popq %r12; + vzeroupper; + + ret; +ENDPROC(twofish_cbc_dec_16way) + +ENTRY(twofish_ctr_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (little endian, 128bit) + */ + + vzeroupper; + pushq %r12; + + load_ctr_16way(%rcx, .Lbswap128_mask, RA0, RB0, RC0, RD0, RA1, RB1, RC1, + RD1, RX0, RX0x, RX1, RX1x, RY0, RY0x, RY1, RY1x, RNOT, + RBYTE); + + call __twofish_enc_blk16; + + store_ctr_16way(%rdx, %rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + popq %r12; + vzeroupper; + + ret; +ENDPROC(twofish_ctr_16way) + +.align 8 +twofish_xts_crypt_16way: + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + * %r8: pointer to __twofish_enc_blk16 or __twofish_dec_blk16 + */ + + vzeroupper; + pushq %r12; + + load_xts_16way(%rcx, %rdx, %rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, + RD1, RX0, RX0x, RX1, RX1x, RY0, RY0x, RY1, RY1x, RNOT, + .Lxts_gf128mul_and_shl1_mask_0, + .Lxts_gf128mul_and_shl1_mask_1); + + call *%r8; + + store_xts_16way(%rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + popq %r12; + vzeroupper; + + ret; +ENDPROC(twofish_xts_crypt_16way) + +ENTRY(twofish_xts_enc_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + leaq __twofish_enc_blk16, %r8; + jmp twofish_xts_crypt_16way; +ENDPROC(twofish_xts_enc_16way) + +ENTRY(twofish_xts_dec_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + leaq __twofish_dec_blk16, %r8; + jmp twofish_xts_crypt_16way; +ENDPROC(twofish_xts_dec_16way) diff --git a/arch/x86/crypto/twofish_avx2_glue.c b/arch/x86/crypto/twofish_avx2_glue.c new file mode 100644 index 0000000..ce33b5b --- /dev/null +++ b/arch/x86/crypto/twofish_avx2_glue.c @@ -0,0 +1,584 @@ +/* + * Glue Code for x86_64/AVX2 assembler optimized version of Twofish + * + * Copyright © 2012-2013 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TF_AVX2_PARALLEL_BLOCKS 16 + +/* 16-way AVX2 parallel cipher functions */ +asmlinkage void twofish_ecb_enc_16way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void twofish_ecb_dec_16way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void twofish_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src); + +asmlinkage void twofish_ctr_16way(void *ctx, u128 *dst, const u128 *src, + le128 *iv); + +asmlinkage void twofish_xts_enc_16way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +asmlinkage void twofish_xts_dec_16way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + +static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src) +{ + __twofish_enc_blk_3way(ctx, dst, src, false); +} + +static const struct common_glue_ctx twofish_enc = { + .num_funcs = 4, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_16way) } + }, { + .num_blocks = 8, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) } + }, { + .num_blocks = 3, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) } + } } +}; + +static const struct common_glue_ctx twofish_ctr = { + .num_funcs = 4, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_16way) } + }, { + .num_blocks = 8, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) } + }, { + .num_blocks = 3, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) } + }, { + .num_blocks = 1, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) } + } } +}; + +static const struct common_glue_ctx twofish_enc_xts = { + .num_funcs = 3, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_16way) } + }, { + .num_blocks = 8, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) } + } } +}; + +static const struct common_glue_ctx twofish_dec = { + .num_funcs = 4, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_16way) } + }, { + .num_blocks = 8, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) } + }, { + .num_blocks = 3, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) } + } } +}; + +static const struct common_glue_ctx twofish_dec_cbc = { + .num_funcs = 4, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_16way) } + }, { + .num_blocks = 8, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) } + }, { + .num_blocks = 3, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) } + }, { + .num_blocks = 1, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) } + } } +}; + +static const struct common_glue_ctx twofish_dec_xts = { + .num_funcs = 3, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_16way) } + }, { + .num_blocks = 8, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) } + } } +}; + +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes); +} + +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes); +} + +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc, + dst, src, nbytes); +} + +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src, + nbytes); +} + +static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes); +} + +static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes) +{ + /* since reusing AVX functions, starts using FPU at 8 parallel blocks */ + return glue_fpu_begin(TF_BLOCK_SIZE, 8, NULL, fpu_enabled, nbytes); +} + +static inline void twofish_fpu_end(bool fpu_enabled) +{ + glue_fpu_end(fpu_enabled); +} + +struct crypt_priv { + struct twofish_ctx *ctx; + bool fpu_enabled; +}; + +static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = TF_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); + + while (nbytes >= TF_AVX2_PARALLEL_BLOCKS * bsize) { + twofish_ecb_enc_16way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * TF_AVX2_PARALLEL_BLOCKS; + nbytes -= bsize * TF_AVX2_PARALLEL_BLOCKS; + } + + while (nbytes >= 8 * bsize) { + twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * 8; + nbytes -= bsize * 8; + } + + while (nbytes >= 3 * bsize) { + twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * 3; + nbytes -= bsize * 3; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + twofish_enc_blk(ctx->ctx, srcdst, srcdst); +} + +static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = TF_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); + + while (nbytes >= TF_AVX2_PARALLEL_BLOCKS * bsize) { + twofish_ecb_dec_16way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * TF_AVX2_PARALLEL_BLOCKS; + nbytes -= bsize * TF_AVX2_PARALLEL_BLOCKS; + } + + while (nbytes >= 8 * bsize) { + twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * 8; + nbytes -= bsize * 8; + } + + while (nbytes >= 3 * bsize) { + twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * 3; + nbytes -= bsize * 3; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + twofish_dec_blk(ctx->ctx, srcdst, srcdst); +} + +static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[TF_AVX2_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->twofish_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = encrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = lrw_crypt(desc, dst, src, nbytes, &req); + twofish_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[TF_AVX2_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->twofish_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = decrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = lrw_crypt(desc, dst, src, nbytes, &req); + twofish_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + + return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(twofish_enc_blk), + &ctx->tweak_ctx, &ctx->crypt_ctx); +} + +static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + + return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(twofish_enc_blk), + &ctx->tweak_ctx, &ctx->crypt_ctx); +} + +static struct crypto_alg tf_algs[10] = { { + .cra_name = "__ecb-twofish-avx2", + .cra_driver_name = "__driver-ecb-twofish-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .setkey = twofish_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "__cbc-twofish-avx2", + .cra_driver_name = "__driver-cbc-twofish-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .setkey = twofish_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "__ctr-twofish-avx2", + .cra_driver_name = "__driver-ctr-twofish-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct twofish_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = twofish_setkey, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + }, + }, +}, { + .cra_name = "__lrw-twofish-avx2", + .cra_driver_name = "__driver-lrw-twofish-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_lrw_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_exit = lrw_twofish_exit_tfm, + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE + + TF_BLOCK_SIZE, + .max_keysize = TF_MAX_KEY_SIZE + + TF_BLOCK_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = lrw_twofish_setkey, + .encrypt = lrw_encrypt, + .decrypt = lrw_decrypt, + }, + }, +}, { + .cra_name = "__xts-twofish-avx2", + .cra_driver_name = "__driver-xts-twofish-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_xts_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE * 2, + .max_keysize = TF_MAX_KEY_SIZE * 2, + .ivsize = TF_BLOCK_SIZE, + .setkey = xts_twofish_setkey, + .encrypt = xts_encrypt, + .decrypt = xts_decrypt, + }, + }, +}, { + .cra_name = "ecb(twofish)", + .cra_driver_name = "ecb-twofish-avx2", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "cbc(twofish)", + .cra_driver_name = "cbc-twofish-avx2", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = __ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "ctr(twofish)", + .cra_driver_name = "ctr-twofish-avx2", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_encrypt, + .geniv = "chainiv", + }, + }, +}, { + .cra_name = "lrw(twofish)", + .cra_driver_name = "lrw-twofish-avx2", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = TF_MIN_KEY_SIZE + + TF_BLOCK_SIZE, + .max_keysize = TF_MAX_KEY_SIZE + + TF_BLOCK_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "xts(twofish)", + .cra_driver_name = "xts-twofish-avx2", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = TF_MIN_KEY_SIZE * 2, + .max_keysize = TF_MAX_KEY_SIZE * 2, + .ivsize = TF_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +} }; + +static int __init init(void) +{ + u64 xcr0; + + if (!cpu_has_avx2 || !cpu_has_osxsave) { + pr_info("AVX2 instructions are not detected.\n"); + return -ENODEV; + } + + xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); + if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { + pr_info("AVX2 detected but unusable.\n"); + return -ENODEV; + } + + return crypto_register_algs(tf_algs, ARRAY_SIZE(tf_algs)); +} + +static void __exit fini(void) +{ + crypto_unregister_algs(tf_algs, ARRAY_SIZE(tf_algs)); +} + +module_init(init); +module_exit(fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX2 optimized"); +MODULE_ALIAS("twofish"); +MODULE_ALIAS("twofish-asm"); diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c index a62ba54..2047a56 100644 --- a/arch/x86/crypto/twofish_avx_glue.c +++ b/arch/x86/crypto/twofish_avx_glue.c @@ -50,18 +50,26 @@ /* 8-way parallel cipher functions */ asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst, const u8 *src); +EXPORT_SYMBOL_GPL(twofish_ecb_enc_8way); + asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst, const u8 *src); +EXPORT_SYMBOL_GPL(twofish_ecb_dec_8way); asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst, const u8 *src); +EXPORT_SYMBOL_GPL(twofish_cbc_dec_8way); + asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +EXPORT_SYMBOL_GPL(twofish_ctr_8way); asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +EXPORT_SYMBOL_GPL(twofish_xts_enc_8way); asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +EXPORT_SYMBOL_GPL(twofish_xts_dec_8way); static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, const u8 *src) @@ -69,17 +77,19 @@ static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, __twofish_enc_blk_3way(ctx, dst, src, false); } -static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) +void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) { glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(twofish_enc_blk)); } +EXPORT_SYMBOL_GPL(twofish_xts_enc); -static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) +void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) { glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(twofish_dec_blk)); } +EXPORT_SYMBOL_GPL(twofish_xts_dec); static const struct common_glue_ctx twofish_enc = { diff --git a/arch/x86/include/asm/crypto/twofish.h b/arch/x86/include/asm/crypto/twofish.h index 878c51c..e655c60 100644 --- a/arch/x86/include/asm/crypto/twofish.h +++ b/arch/x86/include/asm/crypto/twofish.h @@ -28,6 +28,20 @@ asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst, const u8 *src); +/* 8-way parallel cipher functions */ +asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + /* helpers from twofish_x86_64-3way module */ extern void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src); extern void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, @@ -43,4 +57,8 @@ extern void lrw_twofish_exit_tfm(struct crypto_tfm *tfm); extern int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen); +/* helpers from twofish-avx module */ +extern void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv); +extern void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv); + #endif /* ASM_X86_TWOFISH_H */ diff --git a/crypto/Kconfig b/crypto/Kconfig index 6b9564f..1ba48dd 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1250,6 +1250,30 @@ config CRYPTO_TWOFISH_AVX_X86_64 See also: +config CRYPTO_TWOFISH_AVX2_X86_64 + tristate "Twofish cipher algorithm (x86_64/AVX2)" + depends on X86 && 64BIT + select CRYPTO_ALGAPI + select CRYPTO_CRYPTD + select CRYPTO_ABLK_HELPER_X86 + select CRYPTO_GLUE_HELPER_X86 + select CRYPTO_TWOFISH_COMMON + select CRYPTO_TWOFISH_X86_64 + select CRYPTO_TWOFISH_X86_64_3WAY + select CRYPTO_TWOFISH_AVX_X86_64 + select CRYPTO_LRW + select CRYPTO_XTS + help + Twofish cipher algorithm (x86_64/AVX2). + + Twofish was submitted as an AES (Advanced Encryption Standard) + candidate cipher by researchers at CounterPane Systems. It is a + 16 round block cipher supporting key sizes of 128, 192, and 256 + bits. + + See also: + + comment "Compression" config CRYPTO_DEFLATE diff --git a/crypto/testmgr.c b/crypto/testmgr.c index f3effb42..fea7841 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1651,6 +1651,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "__cbc-twofish-avx", .test = alg_test_null, }, { + .alg = "__cbc-twofish-avx2", + .test = alg_test_null, + }, { .alg = "__driver-cbc-aes-aesni", .test = alg_test_null, .fips_allowed = 1, @@ -1676,6 +1679,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "__driver-cbc-twofish-avx", .test = alg_test_null, }, { + .alg = "__driver-cbc-twofish-avx2", + .test = alg_test_null, + }, { .alg = "__driver-ecb-aes-aesni", .test = alg_test_null, .fips_allowed = 1, @@ -1701,6 +1707,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "__driver-ecb-twofish-avx", .test = alg_test_null, }, { + .alg = "__driver-ecb-twofish-avx2", + .test = alg_test_null, + }, { .alg = "__ghash-pclmulqdqni", .test = alg_test_null, .fips_allowed = 1, @@ -1985,6 +1994,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "cryptd(__driver-ecb-twofish-avx)", .test = alg_test_null, }, { + .alg = "cryptd(__driver-ecb-twofish-avx2)", + .test = alg_test_null, + }, { .alg = "cryptd(__driver-gcm-aes-aesni)", .test = alg_test_null, .fips_allowed = 1, -- cgit v0.10.2 From 56d76c96a9f3e39ab733c5643b3ce5a1d4be242a Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sat, 13 Apr 2013 13:46:55 +0300 Subject: crypto: serpent - add AVX2/x86_64 assembler implementation of serpent cipher Patch adds AVX2/x86-64 implementation of Serpent cipher, requiring 16 parallel blocks for input (256 bytes). Implementation is based on the AVX implementation and extends to use the 256-bit wide YMM registers. Since serpent does not use table look-ups, this implementation should be close to two times faster than the AVX implementation. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 1f6e0c2..a21af59 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -43,6 +43,7 @@ endif # These modules require assembler to support AVX2. ifeq ($(avx2_supported),yes) obj-$(CONFIG_CRYPTO_BLOWFISH_AVX2_X86_64) += blowfish-avx2.o + obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o obj-$(CONFIG_CRYPTO_TWOFISH_AVX2_X86_64) += twofish-avx2.o endif @@ -72,6 +73,7 @@ endif ifeq ($(avx2_supported),yes) blowfish-avx2-y := blowfish-avx2-asm_64.o blowfish_avx2_glue.o + serpent-avx2-y := serpent-avx2-asm_64.o serpent_avx2_glue.o twofish-avx2-y := twofish-avx2-asm_64.o twofish_avx2_glue.o endif diff --git a/arch/x86/crypto/serpent-avx2-asm_64.S b/arch/x86/crypto/serpent-avx2-asm_64.S new file mode 100644 index 0000000..b222085 --- /dev/null +++ b/arch/x86/crypto/serpent-avx2-asm_64.S @@ -0,0 +1,800 @@ +/* + * x86_64/AVX2 assembler optimized version of Serpent + * + * Copyright © 2012-2013 Jussi Kivilinna + * + * Based on AVX assembler implementation of Serpent by: + * Copyright © 2012 Johannes Goetzfried + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#include +#include "glue_helper-asm-avx2.S" + +.file "serpent-avx2-asm_64.S" + +.data +.align 16 + +.Lbswap128_mask: + .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +.Lxts_gf128mul_and_shl1_mask_0: + .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 +.Lxts_gf128mul_and_shl1_mask_1: + .byte 0x0e, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0 + +.text + +#define CTX %rdi + +#define RNOT %ymm0 +#define tp %ymm1 + +#define RA1 %ymm2 +#define RA2 %ymm3 +#define RB1 %ymm4 +#define RB2 %ymm5 +#define RC1 %ymm6 +#define RC2 %ymm7 +#define RD1 %ymm8 +#define RD2 %ymm9 +#define RE1 %ymm10 +#define RE2 %ymm11 + +#define RK0 %ymm12 +#define RK1 %ymm13 +#define RK2 %ymm14 +#define RK3 %ymm15 + +#define RK0x %xmm12 +#define RK1x %xmm13 +#define RK2x %xmm14 +#define RK3x %xmm15 + +#define S0_1(x0, x1, x2, x3, x4) \ + vpor x0, x3, tp; \ + vpxor x3, x0, x0; \ + vpxor x2, x3, x4; \ + vpxor RNOT, x4, x4; \ + vpxor x1, tp, x3; \ + vpand x0, x1, x1; \ + vpxor x4, x1, x1; \ + vpxor x0, x2, x2; +#define S0_2(x0, x1, x2, x3, x4) \ + vpxor x3, x0, x0; \ + vpor x0, x4, x4; \ + vpxor x2, x0, x0; \ + vpand x1, x2, x2; \ + vpxor x2, x3, x3; \ + vpxor RNOT, x1, x1; \ + vpxor x4, x2, x2; \ + vpxor x2, x1, x1; + +#define S1_1(x0, x1, x2, x3, x4) \ + vpxor x0, x1, tp; \ + vpxor x3, x0, x0; \ + vpxor RNOT, x3, x3; \ + vpand tp, x1, x4; \ + vpor tp, x0, x0; \ + vpxor x2, x3, x3; \ + vpxor x3, x0, x0; \ + vpxor x3, tp, x1; +#define S1_2(x0, x1, x2, x3, x4) \ + vpxor x4, x3, x3; \ + vpor x4, x1, x1; \ + vpxor x2, x4, x4; \ + vpand x0, x2, x2; \ + vpxor x1, x2, x2; \ + vpor x0, x1, x1; \ + vpxor RNOT, x0, x0; \ + vpxor x2, x0, x0; \ + vpxor x1, x4, x4; + +#define S2_1(x0, x1, x2, x3, x4) \ + vpxor RNOT, x3, x3; \ + vpxor x0, x1, x1; \ + vpand x2, x0, tp; \ + vpxor x3, tp, tp; \ + vpor x0, x3, x3; \ + vpxor x1, x2, x2; \ + vpxor x1, x3, x3; \ + vpand tp, x1, x1; +#define S2_2(x0, x1, x2, x3, x4) \ + vpxor x2, tp, tp; \ + vpand x3, x2, x2; \ + vpor x1, x3, x3; \ + vpxor RNOT, tp, tp; \ + vpxor tp, x3, x3; \ + vpxor tp, x0, x4; \ + vpxor x2, tp, x0; \ + vpor x2, x1, x1; + +#define S3_1(x0, x1, x2, x3, x4) \ + vpxor x3, x1, tp; \ + vpor x0, x3, x3; \ + vpand x0, x1, x4; \ + vpxor x2, x0, x0; \ + vpxor tp, x2, x2; \ + vpand x3, tp, x1; \ + vpxor x3, x2, x2; \ + vpor x4, x0, x0; \ + vpxor x3, x4, x4; +#define S3_2(x0, x1, x2, x3, x4) \ + vpxor x0, x1, x1; \ + vpand x3, x0, x0; \ + vpand x4, x3, x3; \ + vpxor x2, x3, x3; \ + vpor x1, x4, x4; \ + vpand x1, x2, x2; \ + vpxor x3, x4, x4; \ + vpxor x3, x0, x0; \ + vpxor x2, x3, x3; + +#define S4_1(x0, x1, x2, x3, x4) \ + vpand x0, x3, tp; \ + vpxor x3, x0, x0; \ + vpxor x2, tp, tp; \ + vpor x3, x2, x2; \ + vpxor x1, x0, x0; \ + vpxor tp, x3, x4; \ + vpor x0, x2, x2; \ + vpxor x1, x2, x2; +#define S4_2(x0, x1, x2, x3, x4) \ + vpand x0, x1, x1; \ + vpxor x4, x1, x1; \ + vpand x2, x4, x4; \ + vpxor tp, x2, x2; \ + vpxor x0, x4, x4; \ + vpor x1, tp, x3; \ + vpxor RNOT, x1, x1; \ + vpxor x0, x3, x3; + +#define S5_1(x0, x1, x2, x3, x4) \ + vpor x0, x1, tp; \ + vpxor tp, x2, x2; \ + vpxor RNOT, x3, x3; \ + vpxor x0, x1, x4; \ + vpxor x2, x0, x0; \ + vpand x4, tp, x1; \ + vpor x3, x4, x4; \ + vpxor x0, x4, x4; +#define S5_2(x0, x1, x2, x3, x4) \ + vpand x3, x0, x0; \ + vpxor x3, x1, x1; \ + vpxor x2, x3, x3; \ + vpxor x1, x0, x0; \ + vpand x4, x2, x2; \ + vpxor x2, x1, x1; \ + vpand x0, x2, x2; \ + vpxor x2, x3, x3; + +#define S6_1(x0, x1, x2, x3, x4) \ + vpxor x0, x3, x3; \ + vpxor x2, x1, tp; \ + vpxor x0, x2, x2; \ + vpand x3, x0, x0; \ + vpor x3, tp, tp; \ + vpxor RNOT, x1, x4; \ + vpxor tp, x0, x0; \ + vpxor x2, tp, x1; +#define S6_2(x0, x1, x2, x3, x4) \ + vpxor x4, x3, x3; \ + vpxor x0, x4, x4; \ + vpand x0, x2, x2; \ + vpxor x1, x4, x4; \ + vpxor x3, x2, x2; \ + vpand x1, x3, x3; \ + vpxor x0, x3, x3; \ + vpxor x2, x1, x1; + +#define S7_1(x0, x1, x2, x3, x4) \ + vpxor RNOT, x1, tp; \ + vpxor RNOT, x0, x0; \ + vpand x2, tp, x1; \ + vpxor x3, x1, x1; \ + vpor tp, x3, x3; \ + vpxor x2, tp, x4; \ + vpxor x3, x2, x2; \ + vpxor x0, x3, x3; \ + vpor x1, x0, x0; +#define S7_2(x0, x1, x2, x3, x4) \ + vpand x0, x2, x2; \ + vpxor x4, x0, x0; \ + vpxor x3, x4, x4; \ + vpand x0, x3, x3; \ + vpxor x1, x4, x4; \ + vpxor x4, x2, x2; \ + vpxor x1, x3, x3; \ + vpor x0, x4, x4; \ + vpxor x1, x4, x4; + +#define SI0_1(x0, x1, x2, x3, x4) \ + vpxor x0, x1, x1; \ + vpor x1, x3, tp; \ + vpxor x1, x3, x4; \ + vpxor RNOT, x0, x0; \ + vpxor tp, x2, x2; \ + vpxor x0, tp, x3; \ + vpand x1, x0, x0; \ + vpxor x2, x0, x0; +#define SI0_2(x0, x1, x2, x3, x4) \ + vpand x3, x2, x2; \ + vpxor x4, x3, x3; \ + vpxor x3, x2, x2; \ + vpxor x3, x1, x1; \ + vpand x0, x3, x3; \ + vpxor x0, x1, x1; \ + vpxor x2, x0, x0; \ + vpxor x3, x4, x4; + +#define SI1_1(x0, x1, x2, x3, x4) \ + vpxor x3, x1, x1; \ + vpxor x2, x0, tp; \ + vpxor RNOT, x2, x2; \ + vpor x1, x0, x4; \ + vpxor x3, x4, x4; \ + vpand x1, x3, x3; \ + vpxor x2, x1, x1; \ + vpand x4, x2, x2; +#define SI1_2(x0, x1, x2, x3, x4) \ + vpxor x1, x4, x4; \ + vpor x3, x1, x1; \ + vpxor tp, x3, x3; \ + vpxor tp, x2, x2; \ + vpor x4, tp, x0; \ + vpxor x4, x2, x2; \ + vpxor x0, x1, x1; \ + vpxor x1, x4, x4; + +#define SI2_1(x0, x1, x2, x3, x4) \ + vpxor x1, x2, x2; \ + vpxor RNOT, x3, tp; \ + vpor x2, tp, tp; \ + vpxor x3, x2, x2; \ + vpxor x0, x3, x4; \ + vpxor x1, tp, x3; \ + vpor x2, x1, x1; \ + vpxor x0, x2, x2; +#define SI2_2(x0, x1, x2, x3, x4) \ + vpxor x4, x1, x1; \ + vpor x3, x4, x4; \ + vpxor x3, x2, x2; \ + vpxor x2, x4, x4; \ + vpand x1, x2, x2; \ + vpxor x3, x2, x2; \ + vpxor x4, x3, x3; \ + vpxor x0, x4, x4; + +#define SI3_1(x0, x1, x2, x3, x4) \ + vpxor x1, x2, x2; \ + vpand x2, x1, tp; \ + vpxor x0, tp, tp; \ + vpor x1, x0, x0; \ + vpxor x3, x1, x4; \ + vpxor x3, x0, x0; \ + vpor tp, x3, x3; \ + vpxor x2, tp, x1; +#define SI3_2(x0, x1, x2, x3, x4) \ + vpxor x3, x1, x1; \ + vpxor x2, x0, x0; \ + vpxor x3, x2, x2; \ + vpand x1, x3, x3; \ + vpxor x0, x1, x1; \ + vpand x2, x0, x0; \ + vpxor x3, x4, x4; \ + vpxor x0, x3, x3; \ + vpxor x1, x0, x0; + +#define SI4_1(x0, x1, x2, x3, x4) \ + vpxor x3, x2, x2; \ + vpand x1, x0, tp; \ + vpxor x2, tp, tp; \ + vpor x3, x2, x2; \ + vpxor RNOT, x0, x4; \ + vpxor tp, x1, x1; \ + vpxor x2, tp, x0; \ + vpand x4, x2, x2; +#define SI4_2(x0, x1, x2, x3, x4) \ + vpxor x0, x2, x2; \ + vpor x4, x0, x0; \ + vpxor x3, x0, x0; \ + vpand x2, x3, x3; \ + vpxor x3, x4, x4; \ + vpxor x1, x3, x3; \ + vpand x0, x1, x1; \ + vpxor x1, x4, x4; \ + vpxor x3, x0, x0; + +#define SI5_1(x0, x1, x2, x3, x4) \ + vpor x2, x1, tp; \ + vpxor x1, x2, x2; \ + vpxor x3, tp, tp; \ + vpand x1, x3, x3; \ + vpxor x3, x2, x2; \ + vpor x0, x3, x3; \ + vpxor RNOT, x0, x0; \ + vpxor x2, x3, x3; \ + vpor x0, x2, x2; +#define SI5_2(x0, x1, x2, x3, x4) \ + vpxor tp, x1, x4; \ + vpxor x4, x2, x2; \ + vpand x0, x4, x4; \ + vpxor tp, x0, x0; \ + vpxor x3, tp, x1; \ + vpand x2, x0, x0; \ + vpxor x3, x2, x2; \ + vpxor x2, x0, x0; \ + vpxor x4, x2, x2; \ + vpxor x3, x4, x4; + +#define SI6_1(x0, x1, x2, x3, x4) \ + vpxor x2, x0, x0; \ + vpand x3, x0, tp; \ + vpxor x3, x2, x2; \ + vpxor x2, tp, tp; \ + vpxor x1, x3, x3; \ + vpor x0, x2, x2; \ + vpxor x3, x2, x2; \ + vpand tp, x3, x3; +#define SI6_2(x0, x1, x2, x3, x4) \ + vpxor RNOT, tp, tp; \ + vpxor x1, x3, x3; \ + vpand x2, x1, x1; \ + vpxor tp, x0, x4; \ + vpxor x4, x3, x3; \ + vpxor x2, x4, x4; \ + vpxor x1, tp, x0; \ + vpxor x0, x2, x2; + +#define SI7_1(x0, x1, x2, x3, x4) \ + vpand x0, x3, tp; \ + vpxor x2, x0, x0; \ + vpor x3, x2, x2; \ + vpxor x1, x3, x4; \ + vpxor RNOT, x0, x0; \ + vpor tp, x1, x1; \ + vpxor x0, x4, x4; \ + vpand x2, x0, x0; \ + vpxor x1, x0, x0; +#define SI7_2(x0, x1, x2, x3, x4) \ + vpand x2, x1, x1; \ + vpxor x2, tp, x3; \ + vpxor x3, x4, x4; \ + vpand x3, x2, x2; \ + vpor x0, x3, x3; \ + vpxor x4, x1, x1; \ + vpxor x4, x3, x3; \ + vpand x0, x4, x4; \ + vpxor x2, x4, x4; + +#define get_key(i,j,t) \ + vpbroadcastd (4*(i)+(j))*4(CTX), t; + +#define K2(x0, x1, x2, x3, x4, i) \ + get_key(i, 0, RK0); \ + get_key(i, 1, RK1); \ + get_key(i, 2, RK2); \ + get_key(i, 3, RK3); \ + vpxor RK0, x0 ## 1, x0 ## 1; \ + vpxor RK1, x1 ## 1, x1 ## 1; \ + vpxor RK2, x2 ## 1, x2 ## 1; \ + vpxor RK3, x3 ## 1, x3 ## 1; \ + vpxor RK0, x0 ## 2, x0 ## 2; \ + vpxor RK1, x1 ## 2, x1 ## 2; \ + vpxor RK2, x2 ## 2, x2 ## 2; \ + vpxor RK3, x3 ## 2, x3 ## 2; + +#define LK2(x0, x1, x2, x3, x4, i) \ + vpslld $13, x0 ## 1, x4 ## 1; \ + vpsrld $(32 - 13), x0 ## 1, x0 ## 1; \ + vpor x4 ## 1, x0 ## 1, x0 ## 1; \ + vpxor x0 ## 1, x1 ## 1, x1 ## 1; \ + vpslld $3, x2 ## 1, x4 ## 1; \ + vpsrld $(32 - 3), x2 ## 1, x2 ## 1; \ + vpor x4 ## 1, x2 ## 1, x2 ## 1; \ + vpxor x2 ## 1, x1 ## 1, x1 ## 1; \ + vpslld $13, x0 ## 2, x4 ## 2; \ + vpsrld $(32 - 13), x0 ## 2, x0 ## 2; \ + vpor x4 ## 2, x0 ## 2, x0 ## 2; \ + vpxor x0 ## 2, x1 ## 2, x1 ## 2; \ + vpslld $3, x2 ## 2, x4 ## 2; \ + vpsrld $(32 - 3), x2 ## 2, x2 ## 2; \ + vpor x4 ## 2, x2 ## 2, x2 ## 2; \ + vpxor x2 ## 2, x1 ## 2, x1 ## 2; \ + vpslld $1, x1 ## 1, x4 ## 1; \ + vpsrld $(32 - 1), x1 ## 1, x1 ## 1; \ + vpor x4 ## 1, x1 ## 1, x1 ## 1; \ + vpslld $3, x0 ## 1, x4 ## 1; \ + vpxor x2 ## 1, x3 ## 1, x3 ## 1; \ + vpxor x4 ## 1, x3 ## 1, x3 ## 1; \ + get_key(i, 1, RK1); \ + vpslld $1, x1 ## 2, x4 ## 2; \ + vpsrld $(32 - 1), x1 ## 2, x1 ## 2; \ + vpor x4 ## 2, x1 ## 2, x1 ## 2; \ + vpslld $3, x0 ## 2, x4 ## 2; \ + vpxor x2 ## 2, x3 ## 2, x3 ## 2; \ + vpxor x4 ## 2, x3 ## 2, x3 ## 2; \ + get_key(i, 3, RK3); \ + vpslld $7, x3 ## 1, x4 ## 1; \ + vpsrld $(32 - 7), x3 ## 1, x3 ## 1; \ + vpor x4 ## 1, x3 ## 1, x3 ## 1; \ + vpslld $7, x1 ## 1, x4 ## 1; \ + vpxor x1 ## 1, x0 ## 1, x0 ## 1; \ + vpxor x3 ## 1, x0 ## 1, x0 ## 1; \ + vpxor x3 ## 1, x2 ## 1, x2 ## 1; \ + vpxor x4 ## 1, x2 ## 1, x2 ## 1; \ + get_key(i, 0, RK0); \ + vpslld $7, x3 ## 2, x4 ## 2; \ + vpsrld $(32 - 7), x3 ## 2, x3 ## 2; \ + vpor x4 ## 2, x3 ## 2, x3 ## 2; \ + vpslld $7, x1 ## 2, x4 ## 2; \ + vpxor x1 ## 2, x0 ## 2, x0 ## 2; \ + vpxor x3 ## 2, x0 ## 2, x0 ## 2; \ + vpxor x3 ## 2, x2 ## 2, x2 ## 2; \ + vpxor x4 ## 2, x2 ## 2, x2 ## 2; \ + get_key(i, 2, RK2); \ + vpxor RK1, x1 ## 1, x1 ## 1; \ + vpxor RK3, x3 ## 1, x3 ## 1; \ + vpslld $5, x0 ## 1, x4 ## 1; \ + vpsrld $(32 - 5), x0 ## 1, x0 ## 1; \ + vpor x4 ## 1, x0 ## 1, x0 ## 1; \ + vpslld $22, x2 ## 1, x4 ## 1; \ + vpsrld $(32 - 22), x2 ## 1, x2 ## 1; \ + vpor x4 ## 1, x2 ## 1, x2 ## 1; \ + vpxor RK0, x0 ## 1, x0 ## 1; \ + vpxor RK2, x2 ## 1, x2 ## 1; \ + vpxor RK1, x1 ## 2, x1 ## 2; \ + vpxor RK3, x3 ## 2, x3 ## 2; \ + vpslld $5, x0 ## 2, x4 ## 2; \ + vpsrld $(32 - 5), x0 ## 2, x0 ## 2; \ + vpor x4 ## 2, x0 ## 2, x0 ## 2; \ + vpslld $22, x2 ## 2, x4 ## 2; \ + vpsrld $(32 - 22), x2 ## 2, x2 ## 2; \ + vpor x4 ## 2, x2 ## 2, x2 ## 2; \ + vpxor RK0, x0 ## 2, x0 ## 2; \ + vpxor RK2, x2 ## 2, x2 ## 2; + +#define KL2(x0, x1, x2, x3, x4, i) \ + vpxor RK0, x0 ## 1, x0 ## 1; \ + vpxor RK2, x2 ## 1, x2 ## 1; \ + vpsrld $5, x0 ## 1, x4 ## 1; \ + vpslld $(32 - 5), x0 ## 1, x0 ## 1; \ + vpor x4 ## 1, x0 ## 1, x0 ## 1; \ + vpxor RK3, x3 ## 1, x3 ## 1; \ + vpxor RK1, x1 ## 1, x1 ## 1; \ + vpsrld $22, x2 ## 1, x4 ## 1; \ + vpslld $(32 - 22), x2 ## 1, x2 ## 1; \ + vpor x4 ## 1, x2 ## 1, x2 ## 1; \ + vpxor x3 ## 1, x2 ## 1, x2 ## 1; \ + vpxor RK0, x0 ## 2, x0 ## 2; \ + vpxor RK2, x2 ## 2, x2 ## 2; \ + vpsrld $5, x0 ## 2, x4 ## 2; \ + vpslld $(32 - 5), x0 ## 2, x0 ## 2; \ + vpor x4 ## 2, x0 ## 2, x0 ## 2; \ + vpxor RK3, x3 ## 2, x3 ## 2; \ + vpxor RK1, x1 ## 2, x1 ## 2; \ + vpsrld $22, x2 ## 2, x4 ## 2; \ + vpslld $(32 - 22), x2 ## 2, x2 ## 2; \ + vpor x4 ## 2, x2 ## 2, x2 ## 2; \ + vpxor x3 ## 2, x2 ## 2, x2 ## 2; \ + vpxor x3 ## 1, x0 ## 1, x0 ## 1; \ + vpslld $7, x1 ## 1, x4 ## 1; \ + vpxor x1 ## 1, x0 ## 1, x0 ## 1; \ + vpxor x4 ## 1, x2 ## 1, x2 ## 1; \ + vpsrld $1, x1 ## 1, x4 ## 1; \ + vpslld $(32 - 1), x1 ## 1, x1 ## 1; \ + vpor x4 ## 1, x1 ## 1, x1 ## 1; \ + vpxor x3 ## 2, x0 ## 2, x0 ## 2; \ + vpslld $7, x1 ## 2, x4 ## 2; \ + vpxor x1 ## 2, x0 ## 2, x0 ## 2; \ + vpxor x4 ## 2, x2 ## 2, x2 ## 2; \ + vpsrld $1, x1 ## 2, x4 ## 2; \ + vpslld $(32 - 1), x1 ## 2, x1 ## 2; \ + vpor x4 ## 2, x1 ## 2, x1 ## 2; \ + vpsrld $7, x3 ## 1, x4 ## 1; \ + vpslld $(32 - 7), x3 ## 1, x3 ## 1; \ + vpor x4 ## 1, x3 ## 1, x3 ## 1; \ + vpxor x0 ## 1, x1 ## 1, x1 ## 1; \ + vpslld $3, x0 ## 1, x4 ## 1; \ + vpxor x4 ## 1, x3 ## 1, x3 ## 1; \ + vpsrld $7, x3 ## 2, x4 ## 2; \ + vpslld $(32 - 7), x3 ## 2, x3 ## 2; \ + vpor x4 ## 2, x3 ## 2, x3 ## 2; \ + vpxor x0 ## 2, x1 ## 2, x1 ## 2; \ + vpslld $3, x0 ## 2, x4 ## 2; \ + vpxor x4 ## 2, x3 ## 2, x3 ## 2; \ + vpsrld $13, x0 ## 1, x4 ## 1; \ + vpslld $(32 - 13), x0 ## 1, x0 ## 1; \ + vpor x4 ## 1, x0 ## 1, x0 ## 1; \ + vpxor x2 ## 1, x1 ## 1, x1 ## 1; \ + vpxor x2 ## 1, x3 ## 1, x3 ## 1; \ + vpsrld $3, x2 ## 1, x4 ## 1; \ + vpslld $(32 - 3), x2 ## 1, x2 ## 1; \ + vpor x4 ## 1, x2 ## 1, x2 ## 1; \ + vpsrld $13, x0 ## 2, x4 ## 2; \ + vpslld $(32 - 13), x0 ## 2, x0 ## 2; \ + vpor x4 ## 2, x0 ## 2, x0 ## 2; \ + vpxor x2 ## 2, x1 ## 2, x1 ## 2; \ + vpxor x2 ## 2, x3 ## 2, x3 ## 2; \ + vpsrld $3, x2 ## 2, x4 ## 2; \ + vpslld $(32 - 3), x2 ## 2, x2 ## 2; \ + vpor x4 ## 2, x2 ## 2, x2 ## 2; + +#define S(SBOX, x0, x1, x2, x3, x4) \ + SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ + SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ + SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \ + SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); + +#define SP(SBOX, x0, x1, x2, x3, x4, i) \ + get_key(i, 0, RK0); \ + SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ + get_key(i, 2, RK2); \ + SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ + get_key(i, 3, RK3); \ + SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \ + get_key(i, 1, RK1); \ + SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \ + +#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ + vpunpckldq x1, x0, t0; \ + vpunpckhdq x1, x0, t2; \ + vpunpckldq x3, x2, t1; \ + vpunpckhdq x3, x2, x3; \ + \ + vpunpcklqdq t1, t0, x0; \ + vpunpckhqdq t1, t0, x1; \ + vpunpcklqdq x3, t2, x2; \ + vpunpckhqdq x3, t2, x3; + +#define read_blocks(x0, x1, x2, x3, t0, t1, t2) \ + transpose_4x4(x0, x1, x2, x3, t0, t1, t2) + +#define write_blocks(x0, x1, x2, x3, t0, t1, t2) \ + transpose_4x4(x0, x1, x2, x3, t0, t1, t2) + +.align 8 +__serpent_enc_blk16: + /* input: + * %rdi: ctx, CTX + * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: plaintext + * output: + * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: ciphertext + */ + + vpcmpeqd RNOT, RNOT, RNOT; + + read_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); + read_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); + + K2(RA, RB, RC, RD, RE, 0); + S(S0, RA, RB, RC, RD, RE); LK2(RC, RB, RD, RA, RE, 1); + S(S1, RC, RB, RD, RA, RE); LK2(RE, RD, RA, RC, RB, 2); + S(S2, RE, RD, RA, RC, RB); LK2(RB, RD, RE, RC, RA, 3); + S(S3, RB, RD, RE, RC, RA); LK2(RC, RA, RD, RB, RE, 4); + S(S4, RC, RA, RD, RB, RE); LK2(RA, RD, RB, RE, RC, 5); + S(S5, RA, RD, RB, RE, RC); LK2(RC, RA, RD, RE, RB, 6); + S(S6, RC, RA, RD, RE, RB); LK2(RD, RB, RA, RE, RC, 7); + S(S7, RD, RB, RA, RE, RC); LK2(RC, RA, RE, RD, RB, 8); + S(S0, RC, RA, RE, RD, RB); LK2(RE, RA, RD, RC, RB, 9); + S(S1, RE, RA, RD, RC, RB); LK2(RB, RD, RC, RE, RA, 10); + S(S2, RB, RD, RC, RE, RA); LK2(RA, RD, RB, RE, RC, 11); + S(S3, RA, RD, RB, RE, RC); LK2(RE, RC, RD, RA, RB, 12); + S(S4, RE, RC, RD, RA, RB); LK2(RC, RD, RA, RB, RE, 13); + S(S5, RC, RD, RA, RB, RE); LK2(RE, RC, RD, RB, RA, 14); + S(S6, RE, RC, RD, RB, RA); LK2(RD, RA, RC, RB, RE, 15); + S(S7, RD, RA, RC, RB, RE); LK2(RE, RC, RB, RD, RA, 16); + S(S0, RE, RC, RB, RD, RA); LK2(RB, RC, RD, RE, RA, 17); + S(S1, RB, RC, RD, RE, RA); LK2(RA, RD, RE, RB, RC, 18); + S(S2, RA, RD, RE, RB, RC); LK2(RC, RD, RA, RB, RE, 19); + S(S3, RC, RD, RA, RB, RE); LK2(RB, RE, RD, RC, RA, 20); + S(S4, RB, RE, RD, RC, RA); LK2(RE, RD, RC, RA, RB, 21); + S(S5, RE, RD, RC, RA, RB); LK2(RB, RE, RD, RA, RC, 22); + S(S6, RB, RE, RD, RA, RC); LK2(RD, RC, RE, RA, RB, 23); + S(S7, RD, RC, RE, RA, RB); LK2(RB, RE, RA, RD, RC, 24); + S(S0, RB, RE, RA, RD, RC); LK2(RA, RE, RD, RB, RC, 25); + S(S1, RA, RE, RD, RB, RC); LK2(RC, RD, RB, RA, RE, 26); + S(S2, RC, RD, RB, RA, RE); LK2(RE, RD, RC, RA, RB, 27); + S(S3, RE, RD, RC, RA, RB); LK2(RA, RB, RD, RE, RC, 28); + S(S4, RA, RB, RD, RE, RC); LK2(RB, RD, RE, RC, RA, 29); + S(S5, RB, RD, RE, RC, RA); LK2(RA, RB, RD, RC, RE, 30); + S(S6, RA, RB, RD, RC, RE); LK2(RD, RE, RB, RC, RA, 31); + S(S7, RD, RE, RB, RC, RA); K2(RA, RB, RC, RD, RE, 32); + + write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); + write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); + + ret; +ENDPROC(__serpent_enc_blk16) + +.align 8 +__serpent_dec_blk16: + /* input: + * %rdi: ctx, CTX + * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: ciphertext + * output: + * RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2: plaintext + */ + + vpcmpeqd RNOT, RNOT, RNOT; + + read_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); + read_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); + + K2(RA, RB, RC, RD, RE, 32); + SP(SI7, RA, RB, RC, RD, RE, 31); KL2(RB, RD, RA, RE, RC, 31); + SP(SI6, RB, RD, RA, RE, RC, 30); KL2(RA, RC, RE, RB, RD, 30); + SP(SI5, RA, RC, RE, RB, RD, 29); KL2(RC, RD, RA, RE, RB, 29); + SP(SI4, RC, RD, RA, RE, RB, 28); KL2(RC, RA, RB, RE, RD, 28); + SP(SI3, RC, RA, RB, RE, RD, 27); KL2(RB, RC, RD, RE, RA, 27); + SP(SI2, RB, RC, RD, RE, RA, 26); KL2(RC, RA, RE, RD, RB, 26); + SP(SI1, RC, RA, RE, RD, RB, 25); KL2(RB, RA, RE, RD, RC, 25); + SP(SI0, RB, RA, RE, RD, RC, 24); KL2(RE, RC, RA, RB, RD, 24); + SP(SI7, RE, RC, RA, RB, RD, 23); KL2(RC, RB, RE, RD, RA, 23); + SP(SI6, RC, RB, RE, RD, RA, 22); KL2(RE, RA, RD, RC, RB, 22); + SP(SI5, RE, RA, RD, RC, RB, 21); KL2(RA, RB, RE, RD, RC, 21); + SP(SI4, RA, RB, RE, RD, RC, 20); KL2(RA, RE, RC, RD, RB, 20); + SP(SI3, RA, RE, RC, RD, RB, 19); KL2(RC, RA, RB, RD, RE, 19); + SP(SI2, RC, RA, RB, RD, RE, 18); KL2(RA, RE, RD, RB, RC, 18); + SP(SI1, RA, RE, RD, RB, RC, 17); KL2(RC, RE, RD, RB, RA, 17); + SP(SI0, RC, RE, RD, RB, RA, 16); KL2(RD, RA, RE, RC, RB, 16); + SP(SI7, RD, RA, RE, RC, RB, 15); KL2(RA, RC, RD, RB, RE, 15); + SP(SI6, RA, RC, RD, RB, RE, 14); KL2(RD, RE, RB, RA, RC, 14); + SP(SI5, RD, RE, RB, RA, RC, 13); KL2(RE, RC, RD, RB, RA, 13); + SP(SI4, RE, RC, RD, RB, RA, 12); KL2(RE, RD, RA, RB, RC, 12); + SP(SI3, RE, RD, RA, RB, RC, 11); KL2(RA, RE, RC, RB, RD, 11); + SP(SI2, RA, RE, RC, RB, RD, 10); KL2(RE, RD, RB, RC, RA, 10); + SP(SI1, RE, RD, RB, RC, RA, 9); KL2(RA, RD, RB, RC, RE, 9); + SP(SI0, RA, RD, RB, RC, RE, 8); KL2(RB, RE, RD, RA, RC, 8); + SP(SI7, RB, RE, RD, RA, RC, 7); KL2(RE, RA, RB, RC, RD, 7); + SP(SI6, RE, RA, RB, RC, RD, 6); KL2(RB, RD, RC, RE, RA, 6); + SP(SI5, RB, RD, RC, RE, RA, 5); KL2(RD, RA, RB, RC, RE, 5); + SP(SI4, RD, RA, RB, RC, RE, 4); KL2(RD, RB, RE, RC, RA, 4); + SP(SI3, RD, RB, RE, RC, RA, 3); KL2(RE, RD, RA, RC, RB, 3); + SP(SI2, RE, RD, RA, RC, RB, 2); KL2(RD, RB, RC, RA, RE, 2); + SP(SI1, RD, RB, RC, RA, RE, 1); KL2(RE, RB, RC, RA, RD, 1); + S(SI0, RE, RB, RC, RA, RD); K2(RC, RD, RB, RE, RA, 0); + + write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2); + write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); + + ret; +ENDPROC(__serpent_dec_blk16) + +ENTRY(serpent_ecb_enc_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + + load_16way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); + + call __serpent_enc_blk16; + + store_16way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); + + vzeroupper; + + ret; +ENDPROC(serpent_ecb_enc_16way) + +ENTRY(serpent_ecb_dec_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + + load_16way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); + + call __serpent_dec_blk16; + + store_16way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); + + vzeroupper; + + ret; +ENDPROC(serpent_ecb_dec_16way) + +ENTRY(serpent_cbc_dec_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + + load_16way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); + + call __serpent_dec_blk16; + + store_cbc_16way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2, + RK0); + + vzeroupper; + + ret; +ENDPROC(serpent_cbc_dec_16way) + +ENTRY(serpent_ctr_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (little endian, 128bit) + */ + + vzeroupper; + + load_ctr_16way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2, + RD2, RK0, RK0x, RK1, RK1x, RK2, RK2x, RK3, RK3x, RNOT, + tp); + + call __serpent_enc_blk16; + + store_ctr_16way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); + + vzeroupper; + + ret; +ENDPROC(serpent_ctr_16way) + +ENTRY(serpent_xts_enc_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + vzeroupper; + + load_xts_16way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, + RD2, RK0, RK0x, RK1, RK1x, RK2, RK2x, RK3, RK3x, RNOT, + .Lxts_gf128mul_and_shl1_mask_0, + .Lxts_gf128mul_and_shl1_mask_1); + + call __serpent_enc_blk16; + + store_xts_16way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); + + vzeroupper; + + ret; +ENDPROC(serpent_xts_enc_16way) + +ENTRY(serpent_xts_dec_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + vzeroupper; + + load_xts_16way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, + RD2, RK0, RK0x, RK1, RK1x, RK2, RK2x, RK3, RK3x, RNOT, + .Lxts_gf128mul_and_shl1_mask_0, + .Lxts_gf128mul_and_shl1_mask_1); + + call __serpent_dec_blk16; + + store_xts_16way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); + + vzeroupper; + + ret; +ENDPROC(serpent_xts_dec_16way) diff --git a/arch/x86/crypto/serpent_avx2_glue.c b/arch/x86/crypto/serpent_avx2_glue.c new file mode 100644 index 0000000..23aabc6 --- /dev/null +++ b/arch/x86/crypto/serpent_avx2_glue.c @@ -0,0 +1,562 @@ +/* + * Glue Code for x86_64/AVX2 assembler optimized version of Serpent + * + * Copyright © 2012-2013 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SERPENT_AVX2_PARALLEL_BLOCKS 16 + +/* 16-way AVX2 parallel cipher functions */ +asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src); + +asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src, + le128 *iv); +asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + +static const struct common_glue_ctx serpent_enc = { + .num_funcs = 3, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) } + }, { + .num_blocks = 8, + .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) } + } } +}; + +static const struct common_glue_ctx serpent_ctr = { + .num_funcs = 3, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) } + }, { + .num_blocks = 8, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) } + }, { + .num_blocks = 1, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) } + } } +}; + +static const struct common_glue_ctx serpent_enc_xts = { + .num_funcs = 3, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) } + }, { + .num_blocks = 8, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) } + } } +}; + +static const struct common_glue_ctx serpent_dec = { + .num_funcs = 3, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) } + }, { + .num_blocks = 8, + .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) } + } } +}; + +static const struct common_glue_ctx serpent_dec_cbc = { + .num_funcs = 3, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) } + }, { + .num_blocks = 8, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) } + }, { + .num_blocks = 1, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) } + } } +}; + +static const struct common_glue_ctx serpent_dec_xts = { + .num_funcs = 3, + .fpu_blocks_limit = 8, + + .funcs = { { + .num_blocks = 16, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) } + }, { + .num_blocks = 8, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) } + } } +}; + +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes); +} + +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes); +} + +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc, + dst, src, nbytes); +} + +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src, + nbytes); +} + +static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes); +} + +static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes) +{ + /* since reusing AVX functions, starts using FPU at 8 parallel blocks */ + return glue_fpu_begin(SERPENT_BLOCK_SIZE, 8, NULL, fpu_enabled, nbytes); +} + +static inline void serpent_fpu_end(bool fpu_enabled) +{ + glue_fpu_end(fpu_enabled); +} + +struct crypt_priv { + struct serpent_ctx *ctx; + bool fpu_enabled; +}; + +static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = SERPENT_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes); + + if (nbytes >= SERPENT_AVX2_PARALLEL_BLOCKS * bsize) { + serpent_ecb_enc_16way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * SERPENT_AVX2_PARALLEL_BLOCKS; + nbytes -= bsize * SERPENT_AVX2_PARALLEL_BLOCKS; + } + + while (nbytes >= SERPENT_PARALLEL_BLOCKS * bsize) { + serpent_ecb_enc_8way_avx(ctx->ctx, srcdst, srcdst); + srcdst += bsize * SERPENT_PARALLEL_BLOCKS; + nbytes -= bsize * SERPENT_PARALLEL_BLOCKS; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + __serpent_encrypt(ctx->ctx, srcdst, srcdst); +} + +static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = SERPENT_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes); + + if (nbytes >= SERPENT_AVX2_PARALLEL_BLOCKS * bsize) { + serpent_ecb_dec_16way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * SERPENT_AVX2_PARALLEL_BLOCKS; + nbytes -= bsize * SERPENT_AVX2_PARALLEL_BLOCKS; + } + + while (nbytes >= SERPENT_PARALLEL_BLOCKS * bsize) { + serpent_ecb_dec_8way_avx(ctx->ctx, srcdst, srcdst); + srcdst += bsize * SERPENT_PARALLEL_BLOCKS; + nbytes -= bsize * SERPENT_PARALLEL_BLOCKS; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + __serpent_decrypt(ctx->ctx, srcdst, srcdst); +} + +static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[SERPENT_AVX2_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->serpent_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = encrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = lrw_crypt(desc, dst, src, nbytes, &req); + serpent_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[SERPENT_AVX2_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->serpent_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = decrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = lrw_crypt(desc, dst, src, nbytes, &req); + serpent_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + + return glue_xts_crypt_128bit(&serpent_enc_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(__serpent_encrypt), + &ctx->tweak_ctx, &ctx->crypt_ctx); +} + +static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + + return glue_xts_crypt_128bit(&serpent_dec_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(__serpent_encrypt), + &ctx->tweak_ctx, &ctx->crypt_ctx); +} + +static struct crypto_alg srp_algs[10] = { { + .cra_name = "__ecb-serpent-avx2", + .cra_driver_name = "__driver-ecb-serpent-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(srp_algs[0].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .setkey = serpent_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "__cbc-serpent-avx2", + .cra_driver_name = "__driver-cbc-serpent-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(srp_algs[1].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .setkey = serpent_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "__ctr-serpent-avx2", + .cra_driver_name = "__driver-ctr-serpent-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct serpent_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(srp_algs[2].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = serpent_setkey, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + }, + }, +}, { + .cra_name = "__lrw-serpent-avx2", + .cra_driver_name = "__driver-lrw-serpent-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_lrw_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(srp_algs[3].cra_list), + .cra_exit = lrw_serpent_exit_tfm, + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = lrw_serpent_setkey, + .encrypt = lrw_encrypt, + .decrypt = lrw_decrypt, + }, + }, +}, { + .cra_name = "__xts-serpent-avx2", + .cra_driver_name = "__driver-xts-serpent-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_xts_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(srp_algs[4].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE * 2, + .max_keysize = SERPENT_MAX_KEY_SIZE * 2, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = xts_serpent_setkey, + .encrypt = xts_encrypt, + .decrypt = xts_decrypt, + }, + }, +}, { + .cra_name = "ecb(serpent)", + .cra_driver_name = "ecb-serpent-avx2", + .cra_priority = 600, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(srp_algs[5].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "cbc(serpent)", + .cra_driver_name = "cbc-serpent-avx2", + .cra_priority = 600, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(srp_algs[6].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = __ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "ctr(serpent)", + .cra_driver_name = "ctr-serpent-avx2", + .cra_priority = 600, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(srp_algs[7].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_encrypt, + .geniv = "chainiv", + }, + }, +}, { + .cra_name = "lrw(serpent)", + .cra_driver_name = "lrw-serpent-avx2", + .cra_priority = 600, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(srp_algs[8].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "xts(serpent)", + .cra_driver_name = "xts-serpent-avx2", + .cra_priority = 600, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(srp_algs[9].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE * 2, + .max_keysize = SERPENT_MAX_KEY_SIZE * 2, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +} }; + +static int __init init(void) +{ + u64 xcr0; + + if (!cpu_has_avx2 || !cpu_has_osxsave) { + pr_info("AVX2 instructions are not detected.\n"); + return -ENODEV; + } + + xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); + if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { + pr_info("AVX detected but unusable.\n"); + return -ENODEV; + } + + return crypto_register_algs(srp_algs, ARRAY_SIZE(srp_algs)); +} + +static void __exit fini(void) +{ + crypto_unregister_algs(srp_algs, ARRAY_SIZE(srp_algs)); +} + +module_init(init); +module_exit(fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized"); +MODULE_ALIAS("serpent"); +MODULE_ALIAS("serpent-asm"); diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c index 0f8519c..9ae83cf 100644 --- a/arch/x86/crypto/serpent_avx_glue.c +++ b/arch/x86/crypto/serpent_avx_glue.c @@ -41,7 +41,32 @@ #include #include -static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv) +/* 8-way parallel cipher functions */ +asmlinkage void serpent_ecb_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst, + const u8 *src); +EXPORT_SYMBOL_GPL(serpent_ecb_enc_8way_avx); + +asmlinkage void serpent_ecb_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst, + const u8 *src); +EXPORT_SYMBOL_GPL(serpent_ecb_dec_8way_avx); + +asmlinkage void serpent_cbc_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst, + const u8 *src); +EXPORT_SYMBOL_GPL(serpent_cbc_dec_8way_avx); + +asmlinkage void serpent_ctr_8way_avx(struct serpent_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +EXPORT_SYMBOL_GPL(serpent_ctr_8way_avx); + +asmlinkage void serpent_xts_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +EXPORT_SYMBOL_GPL(serpent_xts_enc_8way_avx); + +asmlinkage void serpent_xts_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +EXPORT_SYMBOL_GPL(serpent_xts_dec_8way_avx); + +void __serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv) { be128 ctrblk; @@ -51,18 +76,22 @@ static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv) __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); u128_xor(dst, src, (u128 *)&ctrblk); } +EXPORT_SYMBOL_GPL(__serpent_crypt_ctr); -static void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) +void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) { glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(__serpent_encrypt)); } +EXPORT_SYMBOL_GPL(serpent_xts_enc); -static void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) +void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) { glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(__serpent_decrypt)); } +EXPORT_SYMBOL_GPL(serpent_xts_dec); + static const struct common_glue_ctx serpent_enc = { .num_funcs = 2, @@ -86,7 +115,7 @@ static const struct common_glue_ctx serpent_ctr = { .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) } }, { .num_blocks = 1, - .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) } + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) } } } }; @@ -224,13 +253,8 @@ static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) __serpent_decrypt(ctx->ctx, srcdst, srcdst); } -struct serpent_lrw_ctx { - struct lrw_table_ctx lrw_table; - struct serpent_ctx serpent_ctx; -}; - -static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) +int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) { struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); int err; @@ -243,6 +267,7 @@ static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, return lrw_init_table(&ctx->lrw_table, key + keylen - SERPENT_BLOCK_SIZE); } +EXPORT_SYMBOL_GPL(lrw_serpent_setkey); static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) @@ -296,20 +321,16 @@ static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return ret; } -static void lrw_exit_tfm(struct crypto_tfm *tfm) +void lrw_serpent_exit_tfm(struct crypto_tfm *tfm) { struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); lrw_free_table(&ctx->lrw_table); } +EXPORT_SYMBOL_GPL(lrw_serpent_exit_tfm); -struct serpent_xts_ctx { - struct serpent_ctx tweak_ctx; - struct serpent_ctx crypt_ctx; -}; - -static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) +int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) { struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; @@ -331,6 +352,7 @@ static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, /* second half of xts-key is for tweak */ return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); } +EXPORT_SYMBOL_GPL(xts_serpent_setkey); static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) @@ -420,7 +442,7 @@ static struct crypto_alg serpent_algs[10] = { { .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, - .cra_exit = lrw_exit_tfm, + .cra_exit = lrw_serpent_exit_tfm, .cra_u = { .blkcipher = { .min_keysize = SERPENT_MIN_KEY_SIZE + diff --git a/arch/x86/include/asm/crypto/serpent-avx.h b/arch/x86/include/asm/crypto/serpent-avx.h index 56e79cc..33c2b8a 100644 --- a/arch/x86/include/asm/crypto/serpent-avx.h +++ b/arch/x86/include/asm/crypto/serpent-avx.h @@ -6,6 +6,16 @@ #define SERPENT_PARALLEL_BLOCKS 8 +struct serpent_lrw_ctx { + struct lrw_table_ctx lrw_table; + struct serpent_ctx serpent_ctx; +}; + +struct serpent_xts_ctx { + struct serpent_ctx tweak_ctx; + struct serpent_ctx crypt_ctx; +}; + asmlinkage void serpent_ecb_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst, const u8 *src); asmlinkage void serpent_ecb_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst, @@ -21,4 +31,18 @@ asmlinkage void serpent_xts_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst, asmlinkage void serpent_xts_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +extern void __serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, + le128 *iv); + +extern void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv); +extern void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv); + +extern int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen); + +extern void lrw_serpent_exit_tfm(struct crypto_tfm *tfm); + +extern int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen); + #endif diff --git a/crypto/Kconfig b/crypto/Kconfig index 1ba48dd..9ad3d78 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1131,6 +1131,29 @@ config CRYPTO_SERPENT_AVX_X86_64 See also: +config CRYPTO_SERPENT_AVX2_X86_64 + tristate "Serpent cipher algorithm (x86_64/AVX2)" + depends on X86 && 64BIT + select CRYPTO_ALGAPI + select CRYPTO_CRYPTD + select CRYPTO_ABLK_HELPER_X86 + select CRYPTO_GLUE_HELPER_X86 + select CRYPTO_SERPENT + select CRYPTO_SERPENT_AVX_X86_64 + select CRYPTO_LRW + select CRYPTO_XTS + help + Serpent cipher algorithm, by Anderson, Biham & Knudsen. + + Keys are allowed to be from 0 to 256 bits in length, in steps + of 8 bits. + + This module provides Serpent cipher algorithm that processes 16 + blocks parallel using AVX2 instruction set. + + See also: + + config CRYPTO_TEA tristate "TEA, XTEA and XETA cipher algorithms" select CRYPTO_ALGAPI diff --git a/crypto/testmgr.c b/crypto/testmgr.c index fea7841..f5e13de 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1645,6 +1645,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "__cbc-serpent-avx", .test = alg_test_null, }, { + .alg = "__cbc-serpent-avx2", + .test = alg_test_null, + }, { .alg = "__cbc-serpent-sse2", .test = alg_test_null, }, { @@ -1673,6 +1676,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "__driver-cbc-serpent-avx", .test = alg_test_null, }, { + .alg = "__driver-cbc-serpent-avx2", + .test = alg_test_null, + }, { .alg = "__driver-cbc-serpent-sse2", .test = alg_test_null, }, { @@ -1701,6 +1707,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "__driver-ecb-serpent-avx", .test = alg_test_null, }, { + .alg = "__driver-ecb-serpent-avx2", + .test = alg_test_null, + }, { .alg = "__driver-ecb-serpent-sse2", .test = alg_test_null, }, { @@ -1969,6 +1978,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "cryptd(__driver-cbc-camellia-aesni)", .test = alg_test_null, }, { + .alg = "cryptd(__driver-cbc-serpent-avx2)", + .test = alg_test_null, + }, { .alg = "cryptd(__driver-ecb-aes-aesni)", .test = alg_test_null, .fips_allowed = 1, @@ -1988,6 +2000,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "cryptd(__driver-ecb-serpent-avx)", .test = alg_test_null, }, { + .alg = "cryptd(__driver-ecb-serpent-avx2)", + .test = alg_test_null, + }, { .alg = "cryptd(__driver-ecb-serpent-sse2)", .test = alg_test_null, }, { -- cgit v0.10.2 From f3f935a76aa0eee68da2b273a08d84ba8ffc7a73 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sat, 13 Apr 2013 13:47:00 +0300 Subject: crypto: camellia - add AVX2/AES-NI/x86_64 assembler implementation of camellia cipher Patch adds AVX2/AES-NI/x86-64 implementation of Camellia cipher, requiring 32 parallel blocks for input (512 bytes). Compared to AVX implementation, this version is extended to use the 256-bit wide YMM registers. For AES-NI instructions data is split to two 128-bit registers and merged afterwards. Even with this additional handling, performance should be higher compared to the AES-NI/AVX implementation. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index a21af59..a3a0ed8 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -43,6 +43,7 @@ endif # These modules require assembler to support AVX2. ifeq ($(avx2_supported),yes) obj-$(CONFIG_CRYPTO_BLOWFISH_AVX2_X86_64) += blowfish-avx2.o + obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX2_X86_64) += camellia-aesni-avx2.o obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o obj-$(CONFIG_CRYPTO_TWOFISH_AVX2_X86_64) += twofish-avx2.o endif @@ -73,6 +74,7 @@ endif ifeq ($(avx2_supported),yes) blowfish-avx2-y := blowfish-avx2-asm_64.o blowfish_avx2_glue.o + camellia-aesni-avx2-y := camellia-aesni-avx2-asm_64.o camellia_aesni_avx2_glue.o serpent-avx2-y := serpent-avx2-asm_64.o serpent_avx2_glue.o twofish-avx2-y := twofish-avx2-asm_64.o twofish_avx2_glue.o endif diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S new file mode 100644 index 0000000..91a1878 --- /dev/null +++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S @@ -0,0 +1,1368 @@ +/* + * x86_64/AVX2/AES-NI assembler implementation of Camellia + * + * Copyright © 2013 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#include + +#define CAMELLIA_TABLE_BYTE_LEN 272 + +/* struct camellia_ctx: */ +#define key_table 0 +#define key_length CAMELLIA_TABLE_BYTE_LEN + +/* register macros */ +#define CTX %rdi +#define RIO %r8 + +/********************************************************************** + helper macros + **********************************************************************/ +#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \ + vpand x, mask4bit, tmp0; \ + vpandn x, mask4bit, x; \ + vpsrld $4, x, x; \ + \ + vpshufb tmp0, lo_t, tmp0; \ + vpshufb x, hi_t, x; \ + vpxor tmp0, x, x; + +#define ymm0_x xmm0 +#define ymm1_x xmm1 +#define ymm2_x xmm2 +#define ymm3_x xmm3 +#define ymm4_x xmm4 +#define ymm5_x xmm5 +#define ymm6_x xmm6 +#define ymm7_x xmm7 +#define ymm8_x xmm8 +#define ymm9_x xmm9 +#define ymm10_x xmm10 +#define ymm11_x xmm11 +#define ymm12_x xmm12 +#define ymm13_x xmm13 +#define ymm14_x xmm14 +#define ymm15_x xmm15 + +/* + * AES-NI instructions do not support ymmX registers, so we need splitting and + * merging. + */ +#define vaesenclast256(zero, yreg, tmp) \ + vextracti128 $1, yreg, tmp##_x; \ + vaesenclast zero##_x, yreg##_x, yreg##_x; \ + vaesenclast zero##_x, tmp##_x, tmp##_x; \ + vinserti128 $1, tmp##_x, yreg, yreg; + +/********************************************************************** + 32-way camellia + **********************************************************************/ + +/* + * IN: + * x0..x7: byte-sliced AB state + * mem_cd: register pointer storing CD state + * key: index for key material + * OUT: + * x0..x7: new byte-sliced CD state + */ +#define roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, t0, t1, t2, t3, t4, t5, t6, \ + t7, mem_cd, key) \ + /* \ + * S-function with AES subbytes \ + */ \ + vbroadcasti128 .Linv_shift_row, t4; \ + vpbroadcastb .L0f0f0f0f, t7; \ + vbroadcasti128 .Lpre_tf_lo_s1, t0; \ + vbroadcasti128 .Lpre_tf_hi_s1, t1; \ + \ + /* AES inverse shift rows */ \ + vpshufb t4, x0, x0; \ + vpshufb t4, x7, x7; \ + vpshufb t4, x1, x1; \ + vpshufb t4, x4, x4; \ + vpshufb t4, x2, x2; \ + vpshufb t4, x5, x5; \ + vpshufb t4, x3, x3; \ + vpshufb t4, x6, x6; \ + \ + /* prefilter sboxes 1, 2 and 3 */ \ + vbroadcasti128 .Lpre_tf_lo_s4, t2; \ + vbroadcasti128 .Lpre_tf_hi_s4, t3; \ + filter_8bit(x0, t0, t1, t7, t6); \ + filter_8bit(x7, t0, t1, t7, t6); \ + filter_8bit(x1, t0, t1, t7, t6); \ + filter_8bit(x4, t0, t1, t7, t6); \ + filter_8bit(x2, t0, t1, t7, t6); \ + filter_8bit(x5, t0, t1, t7, t6); \ + \ + /* prefilter sbox 4 */ \ + vpxor t4##_x, t4##_x, t4##_x; \ + filter_8bit(x3, t2, t3, t7, t6); \ + filter_8bit(x6, t2, t3, t7, t6); \ + \ + /* AES subbytes + AES shift rows */ \ + vbroadcasti128 .Lpost_tf_lo_s1, t0; \ + vbroadcasti128 .Lpost_tf_hi_s1, t1; \ + vaesenclast256(t4, x0, t5); \ + vaesenclast256(t4, x7, t5); \ + vaesenclast256(t4, x1, t5); \ + vaesenclast256(t4, x4, t5); \ + vaesenclast256(t4, x2, t5); \ + vaesenclast256(t4, x5, t5); \ + vaesenclast256(t4, x3, t5); \ + vaesenclast256(t4, x6, t5); \ + \ + /* postfilter sboxes 1 and 4 */ \ + vbroadcasti128 .Lpost_tf_lo_s3, t2; \ + vbroadcasti128 .Lpost_tf_hi_s3, t3; \ + filter_8bit(x0, t0, t1, t7, t6); \ + filter_8bit(x7, t0, t1, t7, t6); \ + filter_8bit(x3, t0, t1, t7, t6); \ + filter_8bit(x6, t0, t1, t7, t6); \ + \ + /* postfilter sbox 3 */ \ + vbroadcasti128 .Lpost_tf_lo_s2, t4; \ + vbroadcasti128 .Lpost_tf_hi_s2, t5; \ + filter_8bit(x2, t2, t3, t7, t6); \ + filter_8bit(x5, t2, t3, t7, t6); \ + \ + vpbroadcastq key, t0; /* higher 64-bit duplicate ignored */ \ + \ + /* postfilter sbox 2 */ \ + filter_8bit(x1, t4, t5, t7, t2); \ + filter_8bit(x4, t4, t5, t7, t2); \ + \ + vpsrldq $1, t0, t1; \ + vpsrldq $2, t0, t2; \ + vpsrldq $3, t0, t3; \ + vpsrldq $4, t0, t4; \ + vpsrldq $5, t0, t5; \ + vpsrldq $6, t0, t6; \ + vpsrldq $7, t0, t7; \ + vpbroadcastb t0##_x, t0; \ + vpbroadcastb t1##_x, t1; \ + vpbroadcastb t2##_x, t2; \ + vpbroadcastb t3##_x, t3; \ + vpbroadcastb t4##_x, t4; \ + vpbroadcastb t6##_x, t6; \ + vpbroadcastb t5##_x, t5; \ + vpbroadcastb t7##_x, t7; \ + \ + /* P-function */ \ + vpxor x5, x0, x0; \ + vpxor x6, x1, x1; \ + vpxor x7, x2, x2; \ + vpxor x4, x3, x3; \ + \ + vpxor x2, x4, x4; \ + vpxor x3, x5, x5; \ + vpxor x0, x6, x6; \ + vpxor x1, x7, x7; \ + \ + vpxor x7, x0, x0; \ + vpxor x4, x1, x1; \ + vpxor x5, x2, x2; \ + vpxor x6, x3, x3; \ + \ + vpxor x3, x4, x4; \ + vpxor x0, x5, x5; \ + vpxor x1, x6, x6; \ + vpxor x2, x7, x7; /* note: high and low parts swapped */ \ + \ + /* Add key material and result to CD (x becomes new CD) */ \ + \ + vpxor t7, x0, x0; \ + vpxor 4 * 32(mem_cd), x0, x0; \ + \ + vpxor t6, x1, x1; \ + vpxor 5 * 32(mem_cd), x1, x1; \ + \ + vpxor t5, x2, x2; \ + vpxor 6 * 32(mem_cd), x2, x2; \ + \ + vpxor t4, x3, x3; \ + vpxor 7 * 32(mem_cd), x3, x3; \ + \ + vpxor t3, x4, x4; \ + vpxor 0 * 32(mem_cd), x4, x4; \ + \ + vpxor t2, x5, x5; \ + vpxor 1 * 32(mem_cd), x5, x5; \ + \ + vpxor t1, x6, x6; \ + vpxor 2 * 32(mem_cd), x6, x6; \ + \ + vpxor t0, x7, x7; \ + vpxor 3 * 32(mem_cd), x7, x7; + +/* + * Size optimization... with inlined roundsm16 binary would be over 5 times + * larger and would only marginally faster. + */ +.align 8 +roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd: + roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, + %rcx, (%r9)); + ret; +ENDPROC(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) + +.align 8 +roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab: + roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3, + %ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11, + %rax, (%r9)); + ret; +ENDPROC(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) + +/* + * IN/OUT: + * x0..x7: byte-sliced AB state preloaded + * mem_ab: byte-sliced AB state in memory + * mem_cb: byte-sliced CD state in memory + */ +#define two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, mem_ab, mem_cd, i, dir, store_ab) \ + leaq (key_table + (i) * 8)(CTX), %r9; \ + call roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd; \ + \ + vmovdqu x0, 4 * 32(mem_cd); \ + vmovdqu x1, 5 * 32(mem_cd); \ + vmovdqu x2, 6 * 32(mem_cd); \ + vmovdqu x3, 7 * 32(mem_cd); \ + vmovdqu x4, 0 * 32(mem_cd); \ + vmovdqu x5, 1 * 32(mem_cd); \ + vmovdqu x6, 2 * 32(mem_cd); \ + vmovdqu x7, 3 * 32(mem_cd); \ + \ + leaq (key_table + ((i) + (dir)) * 8)(CTX), %r9; \ + call roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab; \ + \ + store_ab(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab); + +#define dummy_store(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) /* do nothing */ + +#define store_ab_state(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) \ + /* Store new AB state */ \ + vmovdqu x4, 4 * 32(mem_ab); \ + vmovdqu x5, 5 * 32(mem_ab); \ + vmovdqu x6, 6 * 32(mem_ab); \ + vmovdqu x7, 7 * 32(mem_ab); \ + vmovdqu x0, 0 * 32(mem_ab); \ + vmovdqu x1, 1 * 32(mem_ab); \ + vmovdqu x2, 2 * 32(mem_ab); \ + vmovdqu x3, 3 * 32(mem_ab); + +#define enc_rounds32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, mem_ab, mem_cd, i) \ + two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, mem_ab, mem_cd, (i) + 2, 1, store_ab_state); \ + two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, mem_ab, mem_cd, (i) + 4, 1, store_ab_state); \ + two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, mem_ab, mem_cd, (i) + 6, 1, dummy_store); + +#define dec_rounds32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, mem_ab, mem_cd, i) \ + two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, mem_ab, mem_cd, (i) + 7, -1, store_ab_state); \ + two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, mem_ab, mem_cd, (i) + 5, -1, store_ab_state); \ + two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, mem_ab, mem_cd, (i) + 3, -1, dummy_store); + +/* + * IN: + * v0..3: byte-sliced 32-bit integers + * OUT: + * v0..3: (IN <<< 1) + */ +#define rol32_1_32(v0, v1, v2, v3, t0, t1, t2, zero) \ + vpcmpgtb v0, zero, t0; \ + vpaddb v0, v0, v0; \ + vpabsb t0, t0; \ + \ + vpcmpgtb v1, zero, t1; \ + vpaddb v1, v1, v1; \ + vpabsb t1, t1; \ + \ + vpcmpgtb v2, zero, t2; \ + vpaddb v2, v2, v2; \ + vpabsb t2, t2; \ + \ + vpor t0, v1, v1; \ + \ + vpcmpgtb v3, zero, t0; \ + vpaddb v3, v3, v3; \ + vpabsb t0, t0; \ + \ + vpor t1, v2, v2; \ + vpor t2, v3, v3; \ + vpor t0, v0, v0; + +/* + * IN: + * r: byte-sliced AB state in memory + * l: byte-sliced CD state in memory + * OUT: + * x0..x7: new byte-sliced CD state + */ +#define fls32(l, l0, l1, l2, l3, l4, l5, l6, l7, r, t0, t1, t2, t3, tt0, \ + tt1, tt2, tt3, kll, klr, krl, krr) \ + /* \ + * t0 = kll; \ + * t0 &= ll; \ + * lr ^= rol32(t0, 1); \ + */ \ + vpbroadcastd kll, t0; /* only lowest 32-bit used */ \ + vpxor tt0, tt0, tt0; \ + vpbroadcastb t0##_x, t3; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t2; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t1; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t0; \ + \ + vpand l0, t0, t0; \ + vpand l1, t1, t1; \ + vpand l2, t2, t2; \ + vpand l3, t3, t3; \ + \ + rol32_1_32(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \ + \ + vpxor l4, t0, l4; \ + vmovdqu l4, 4 * 32(l); \ + vpxor l5, t1, l5; \ + vmovdqu l5, 5 * 32(l); \ + vpxor l6, t2, l6; \ + vmovdqu l6, 6 * 32(l); \ + vpxor l7, t3, l7; \ + vmovdqu l7, 7 * 32(l); \ + \ + /* \ + * t2 = krr; \ + * t2 |= rr; \ + * rl ^= t2; \ + */ \ + \ + vpbroadcastd krr, t0; /* only lowest 32-bit used */ \ + vpbroadcastb t0##_x, t3; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t2; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t1; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t0; \ + \ + vpor 4 * 32(r), t0, t0; \ + vpor 5 * 32(r), t1, t1; \ + vpor 6 * 32(r), t2, t2; \ + vpor 7 * 32(r), t3, t3; \ + \ + vpxor 0 * 32(r), t0, t0; \ + vpxor 1 * 32(r), t1, t1; \ + vpxor 2 * 32(r), t2, t2; \ + vpxor 3 * 32(r), t3, t3; \ + vmovdqu t0, 0 * 32(r); \ + vmovdqu t1, 1 * 32(r); \ + vmovdqu t2, 2 * 32(r); \ + vmovdqu t3, 3 * 32(r); \ + \ + /* \ + * t2 = krl; \ + * t2 &= rl; \ + * rr ^= rol32(t2, 1); \ + */ \ + vpbroadcastd krl, t0; /* only lowest 32-bit used */ \ + vpbroadcastb t0##_x, t3; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t2; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t1; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t0; \ + \ + vpand 0 * 32(r), t0, t0; \ + vpand 1 * 32(r), t1, t1; \ + vpand 2 * 32(r), t2, t2; \ + vpand 3 * 32(r), t3, t3; \ + \ + rol32_1_32(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \ + \ + vpxor 4 * 32(r), t0, t0; \ + vpxor 5 * 32(r), t1, t1; \ + vpxor 6 * 32(r), t2, t2; \ + vpxor 7 * 32(r), t3, t3; \ + vmovdqu t0, 4 * 32(r); \ + vmovdqu t1, 5 * 32(r); \ + vmovdqu t2, 6 * 32(r); \ + vmovdqu t3, 7 * 32(r); \ + \ + /* \ + * t0 = klr; \ + * t0 |= lr; \ + * ll ^= t0; \ + */ \ + \ + vpbroadcastd klr, t0; /* only lowest 32-bit used */ \ + vpbroadcastb t0##_x, t3; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t2; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t1; \ + vpsrldq $1, t0, t0; \ + vpbroadcastb t0##_x, t0; \ + \ + vpor l4, t0, t0; \ + vpor l5, t1, t1; \ + vpor l6, t2, t2; \ + vpor l7, t3, t3; \ + \ + vpxor l0, t0, l0; \ + vmovdqu l0, 0 * 32(l); \ + vpxor l1, t1, l1; \ + vmovdqu l1, 1 * 32(l); \ + vpxor l2, t2, l2; \ + vmovdqu l2, 2 * 32(l); \ + vpxor l3, t3, l3; \ + vmovdqu l3, 3 * 32(l); + +#define transpose_4x4(x0, x1, x2, x3, t1, t2) \ + vpunpckhdq x1, x0, t2; \ + vpunpckldq x1, x0, x0; \ + \ + vpunpckldq x3, x2, t1; \ + vpunpckhdq x3, x2, x2; \ + \ + vpunpckhqdq t1, x0, x1; \ + vpunpcklqdq t1, x0, x0; \ + \ + vpunpckhqdq x2, t2, x3; \ + vpunpcklqdq x2, t2, x2; + +#define byteslice_16x16b_fast(a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, \ + a3, b3, c3, d3, st0, st1) \ + vmovdqu d2, st0; \ + vmovdqu d3, st1; \ + transpose_4x4(a0, a1, a2, a3, d2, d3); \ + transpose_4x4(b0, b1, b2, b3, d2, d3); \ + vmovdqu st0, d2; \ + vmovdqu st1, d3; \ + \ + vmovdqu a0, st0; \ + vmovdqu a1, st1; \ + transpose_4x4(c0, c1, c2, c3, a0, a1); \ + transpose_4x4(d0, d1, d2, d3, a0, a1); \ + \ + vbroadcasti128 .Lshufb_16x16b, a0; \ + vmovdqu st1, a1; \ + vpshufb a0, a2, a2; \ + vpshufb a0, a3, a3; \ + vpshufb a0, b0, b0; \ + vpshufb a0, b1, b1; \ + vpshufb a0, b2, b2; \ + vpshufb a0, b3, b3; \ + vpshufb a0, a1, a1; \ + vpshufb a0, c0, c0; \ + vpshufb a0, c1, c1; \ + vpshufb a0, c2, c2; \ + vpshufb a0, c3, c3; \ + vpshufb a0, d0, d0; \ + vpshufb a0, d1, d1; \ + vpshufb a0, d2, d2; \ + vpshufb a0, d3, d3; \ + vmovdqu d3, st1; \ + vmovdqu st0, d3; \ + vpshufb a0, d3, a0; \ + vmovdqu d2, st0; \ + \ + transpose_4x4(a0, b0, c0, d0, d2, d3); \ + transpose_4x4(a1, b1, c1, d1, d2, d3); \ + vmovdqu st0, d2; \ + vmovdqu st1, d3; \ + \ + vmovdqu b0, st0; \ + vmovdqu b1, st1; \ + transpose_4x4(a2, b2, c2, d2, b0, b1); \ + transpose_4x4(a3, b3, c3, d3, b0, b1); \ + vmovdqu st0, b0; \ + vmovdqu st1, b1; \ + /* does not adjust output bytes inside vectors */ + +/* load blocks to registers and apply pre-whitening */ +#define inpack32_pre(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, rio, key) \ + vpbroadcastq key, x0; \ + vpshufb .Lpack_bswap, x0, x0; \ + \ + vpxor 0 * 32(rio), x0, y7; \ + vpxor 1 * 32(rio), x0, y6; \ + vpxor 2 * 32(rio), x0, y5; \ + vpxor 3 * 32(rio), x0, y4; \ + vpxor 4 * 32(rio), x0, y3; \ + vpxor 5 * 32(rio), x0, y2; \ + vpxor 6 * 32(rio), x0, y1; \ + vpxor 7 * 32(rio), x0, y0; \ + vpxor 8 * 32(rio), x0, x7; \ + vpxor 9 * 32(rio), x0, x6; \ + vpxor 10 * 32(rio), x0, x5; \ + vpxor 11 * 32(rio), x0, x4; \ + vpxor 12 * 32(rio), x0, x3; \ + vpxor 13 * 32(rio), x0, x2; \ + vpxor 14 * 32(rio), x0, x1; \ + vpxor 15 * 32(rio), x0, x0; + +/* byteslice pre-whitened blocks and store to temporary memory */ +#define inpack32_post(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, mem_ab, mem_cd) \ + byteslice_16x16b_fast(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, \ + y4, y5, y6, y7, (mem_ab), (mem_cd)); \ + \ + vmovdqu x0, 0 * 32(mem_ab); \ + vmovdqu x1, 1 * 32(mem_ab); \ + vmovdqu x2, 2 * 32(mem_ab); \ + vmovdqu x3, 3 * 32(mem_ab); \ + vmovdqu x4, 4 * 32(mem_ab); \ + vmovdqu x5, 5 * 32(mem_ab); \ + vmovdqu x6, 6 * 32(mem_ab); \ + vmovdqu x7, 7 * 32(mem_ab); \ + vmovdqu y0, 0 * 32(mem_cd); \ + vmovdqu y1, 1 * 32(mem_cd); \ + vmovdqu y2, 2 * 32(mem_cd); \ + vmovdqu y3, 3 * 32(mem_cd); \ + vmovdqu y4, 4 * 32(mem_cd); \ + vmovdqu y5, 5 * 32(mem_cd); \ + vmovdqu y6, 6 * 32(mem_cd); \ + vmovdqu y7, 7 * 32(mem_cd); + +/* de-byteslice, apply post-whitening and store blocks */ +#define outunpack32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, \ + y5, y6, y7, key, stack_tmp0, stack_tmp1) \ + byteslice_16x16b_fast(y0, y4, x0, x4, y1, y5, x1, x5, y2, y6, x2, x6, \ + y3, y7, x3, x7, stack_tmp0, stack_tmp1); \ + \ + vmovdqu x0, stack_tmp0; \ + \ + vpbroadcastq key, x0; \ + vpshufb .Lpack_bswap, x0, x0; \ + \ + vpxor x0, y7, y7; \ + vpxor x0, y6, y6; \ + vpxor x0, y5, y5; \ + vpxor x0, y4, y4; \ + vpxor x0, y3, y3; \ + vpxor x0, y2, y2; \ + vpxor x0, y1, y1; \ + vpxor x0, y0, y0; \ + vpxor x0, x7, x7; \ + vpxor x0, x6, x6; \ + vpxor x0, x5, x5; \ + vpxor x0, x4, x4; \ + vpxor x0, x3, x3; \ + vpxor x0, x2, x2; \ + vpxor x0, x1, x1; \ + vpxor stack_tmp0, x0, x0; + +#define write_output(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ + y6, y7, rio) \ + vmovdqu x0, 0 * 32(rio); \ + vmovdqu x1, 1 * 32(rio); \ + vmovdqu x2, 2 * 32(rio); \ + vmovdqu x3, 3 * 32(rio); \ + vmovdqu x4, 4 * 32(rio); \ + vmovdqu x5, 5 * 32(rio); \ + vmovdqu x6, 6 * 32(rio); \ + vmovdqu x7, 7 * 32(rio); \ + vmovdqu y0, 8 * 32(rio); \ + vmovdqu y1, 9 * 32(rio); \ + vmovdqu y2, 10 * 32(rio); \ + vmovdqu y3, 11 * 32(rio); \ + vmovdqu y4, 12 * 32(rio); \ + vmovdqu y5, 13 * 32(rio); \ + vmovdqu y6, 14 * 32(rio); \ + vmovdqu y7, 15 * 32(rio); + +.data +.align 32 + +#define SHUFB_BYTES(idx) \ + 0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx) + +.Lshufb_16x16b: + .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3) + .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3) + +.Lpack_bswap: + .long 0x00010203, 0x04050607, 0x80808080, 0x80808080 + .long 0x00010203, 0x04050607, 0x80808080, 0x80808080 + +/* For CTR-mode IV byteswap */ +.Lbswap128_mask: + .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 + +/* For XTS mode */ +.Lxts_gf128mul_and_shl1_mask_0: + .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 +.Lxts_gf128mul_and_shl1_mask_1: + .byte 0x0e, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0 + +/* + * pre-SubByte transform + * + * pre-lookup for sbox1, sbox2, sbox3: + * swap_bitendianness( + * isom_map_camellia_to_aes( + * camellia_f( + * swap_bitendianess(in) + * ) + * ) + * ) + * + * (note: '⊕ 0xc5' inside camellia_f()) + */ +.Lpre_tf_lo_s1: + .byte 0x45, 0xe8, 0x40, 0xed, 0x2e, 0x83, 0x2b, 0x86 + .byte 0x4b, 0xe6, 0x4e, 0xe3, 0x20, 0x8d, 0x25, 0x88 +.Lpre_tf_hi_s1: + .byte 0x00, 0x51, 0xf1, 0xa0, 0x8a, 0xdb, 0x7b, 0x2a + .byte 0x09, 0x58, 0xf8, 0xa9, 0x83, 0xd2, 0x72, 0x23 + +/* + * pre-SubByte transform + * + * pre-lookup for sbox4: + * swap_bitendianness( + * isom_map_camellia_to_aes( + * camellia_f( + * swap_bitendianess(in <<< 1) + * ) + * ) + * ) + * + * (note: '⊕ 0xc5' inside camellia_f()) + */ +.Lpre_tf_lo_s4: + .byte 0x45, 0x40, 0x2e, 0x2b, 0x4b, 0x4e, 0x20, 0x25 + .byte 0x14, 0x11, 0x7f, 0x7a, 0x1a, 0x1f, 0x71, 0x74 +.Lpre_tf_hi_s4: + .byte 0x00, 0xf1, 0x8a, 0x7b, 0x09, 0xf8, 0x83, 0x72 + .byte 0xad, 0x5c, 0x27, 0xd6, 0xa4, 0x55, 0x2e, 0xdf + +/* + * post-SubByte transform + * + * post-lookup for sbox1, sbox4: + * swap_bitendianness( + * camellia_h( + * isom_map_aes_to_camellia( + * swap_bitendianness( + * aes_inverse_affine_transform(in) + * ) + * ) + * ) + * ) + * + * (note: '⊕ 0x6e' inside camellia_h()) + */ +.Lpost_tf_lo_s1: + .byte 0x3c, 0xcc, 0xcf, 0x3f, 0x32, 0xc2, 0xc1, 0x31 + .byte 0xdc, 0x2c, 0x2f, 0xdf, 0xd2, 0x22, 0x21, 0xd1 +.Lpost_tf_hi_s1: + .byte 0x00, 0xf9, 0x86, 0x7f, 0xd7, 0x2e, 0x51, 0xa8 + .byte 0xa4, 0x5d, 0x22, 0xdb, 0x73, 0x8a, 0xf5, 0x0c + +/* + * post-SubByte transform + * + * post-lookup for sbox2: + * swap_bitendianness( + * camellia_h( + * isom_map_aes_to_camellia( + * swap_bitendianness( + * aes_inverse_affine_transform(in) + * ) + * ) + * ) + * ) <<< 1 + * + * (note: '⊕ 0x6e' inside camellia_h()) + */ +.Lpost_tf_lo_s2: + .byte 0x78, 0x99, 0x9f, 0x7e, 0x64, 0x85, 0x83, 0x62 + .byte 0xb9, 0x58, 0x5e, 0xbf, 0xa5, 0x44, 0x42, 0xa3 +.Lpost_tf_hi_s2: + .byte 0x00, 0xf3, 0x0d, 0xfe, 0xaf, 0x5c, 0xa2, 0x51 + .byte 0x49, 0xba, 0x44, 0xb7, 0xe6, 0x15, 0xeb, 0x18 + +/* + * post-SubByte transform + * + * post-lookup for sbox3: + * swap_bitendianness( + * camellia_h( + * isom_map_aes_to_camellia( + * swap_bitendianness( + * aes_inverse_affine_transform(in) + * ) + * ) + * ) + * ) >>> 1 + * + * (note: '⊕ 0x6e' inside camellia_h()) + */ +.Lpost_tf_lo_s3: + .byte 0x1e, 0x66, 0xe7, 0x9f, 0x19, 0x61, 0xe0, 0x98 + .byte 0x6e, 0x16, 0x97, 0xef, 0x69, 0x11, 0x90, 0xe8 +.Lpost_tf_hi_s3: + .byte 0x00, 0xfc, 0x43, 0xbf, 0xeb, 0x17, 0xa8, 0x54 + .byte 0x52, 0xae, 0x11, 0xed, 0xb9, 0x45, 0xfa, 0x06 + +/* For isolating SubBytes from AESENCLAST, inverse shift row */ +.Linv_shift_row: + .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b + .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03 + +.align 4 +/* 4-bit mask */ +.L0f0f0f0f: + .long 0x0f0f0f0f + +.text + +.align 8 +__camellia_enc_blk32: + /* input: + * %rdi: ctx, CTX + * %rax: temporary storage, 512 bytes + * %ymm0..%ymm15: 32 plaintext blocks + * output: + * %ymm0..%ymm15: 32 encrypted blocks, order swapped: + * 7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 + */ + + leaq 8 * 32(%rax), %rcx; + + inpack32_post(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rax, %rcx); + + enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rax, %rcx, 0); + + fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, + ((key_table + (8) * 8) + 0)(CTX), + ((key_table + (8) * 8) + 4)(CTX), + ((key_table + (8) * 8) + 8)(CTX), + ((key_table + (8) * 8) + 12)(CTX)); + + enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rax, %rcx, 8); + + fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, + ((key_table + (16) * 8) + 0)(CTX), + ((key_table + (16) * 8) + 4)(CTX), + ((key_table + (16) * 8) + 8)(CTX), + ((key_table + (16) * 8) + 12)(CTX)); + + enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rax, %rcx, 16); + + movl $24, %r8d; + cmpl $16, key_length(CTX); + jne .Lenc_max32; + +.Lenc_done: + /* load CD for output */ + vmovdqu 0 * 32(%rcx), %ymm8; + vmovdqu 1 * 32(%rcx), %ymm9; + vmovdqu 2 * 32(%rcx), %ymm10; + vmovdqu 3 * 32(%rcx), %ymm11; + vmovdqu 4 * 32(%rcx), %ymm12; + vmovdqu 5 * 32(%rcx), %ymm13; + vmovdqu 6 * 32(%rcx), %ymm14; + vmovdqu 7 * 32(%rcx), %ymm15; + + outunpack32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax)); + + ret; + +.align 8 +.Lenc_max32: + movl $32, %r8d; + + fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, + ((key_table + (24) * 8) + 0)(CTX), + ((key_table + (24) * 8) + 4)(CTX), + ((key_table + (24) * 8) + 8)(CTX), + ((key_table + (24) * 8) + 12)(CTX)); + + enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rax, %rcx, 24); + + jmp .Lenc_done; +ENDPROC(__camellia_enc_blk32) + +.align 8 +__camellia_dec_blk32: + /* input: + * %rdi: ctx, CTX + * %rax: temporary storage, 512 bytes + * %r8d: 24 for 16 byte key, 32 for larger + * %ymm0..%ymm15: 16 encrypted blocks + * output: + * %ymm0..%ymm15: 16 plaintext blocks, order swapped: + * 7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 + */ + + leaq 8 * 32(%rax), %rcx; + + inpack32_post(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rax, %rcx); + + cmpl $32, %r8d; + je .Ldec_max32; + +.Ldec_max24: + dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rax, %rcx, 16); + + fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, + ((key_table + (16) * 8) + 8)(CTX), + ((key_table + (16) * 8) + 12)(CTX), + ((key_table + (16) * 8) + 0)(CTX), + ((key_table + (16) * 8) + 4)(CTX)); + + dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rax, %rcx, 8); + + fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, + ((key_table + (8) * 8) + 8)(CTX), + ((key_table + (8) * 8) + 12)(CTX), + ((key_table + (8) * 8) + 0)(CTX), + ((key_table + (8) * 8) + 4)(CTX)); + + dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rax, %rcx, 0); + + /* load CD for output */ + vmovdqu 0 * 32(%rcx), %ymm8; + vmovdqu 1 * 32(%rcx), %ymm9; + vmovdqu 2 * 32(%rcx), %ymm10; + vmovdqu 3 * 32(%rcx), %ymm11; + vmovdqu 4 * 32(%rcx), %ymm12; + vmovdqu 5 * 32(%rcx), %ymm13; + vmovdqu 6 * 32(%rcx), %ymm14; + vmovdqu 7 * 32(%rcx), %ymm15; + + outunpack32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax)); + + ret; + +.align 8 +.Ldec_max32: + dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rax, %rcx, 24); + + fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, + ((key_table + (24) * 8) + 8)(CTX), + ((key_table + (24) * 8) + 12)(CTX), + ((key_table + (24) * 8) + 0)(CTX), + ((key_table + (24) * 8) + 4)(CTX)); + + jmp .Ldec_max24; +ENDPROC(__camellia_dec_blk32) + +ENTRY(camellia_ecb_enc_32way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (32 blocks) + * %rdx: src (32 blocks) + */ + + vzeroupper; + + inpack32_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rdx, (key_table)(CTX)); + + /* now dst can be used as temporary buffer (even in src == dst case) */ + movq %rsi, %rax; + + call __camellia_enc_blk32; + + write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, + %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, + %ymm8, %rsi); + + vzeroupper; + + ret; +ENDPROC(camellia_ecb_enc_32way) + +ENTRY(camellia_ecb_dec_32way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (32 blocks) + * %rdx: src (32 blocks) + */ + + vzeroupper; + + cmpl $16, key_length(CTX); + movl $32, %r8d; + movl $24, %eax; + cmovel %eax, %r8d; /* max */ + + inpack32_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rdx, (key_table)(CTX, %r8, 8)); + + /* now dst can be used as temporary buffer (even in src == dst case) */ + movq %rsi, %rax; + + call __camellia_dec_blk32; + + write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, + %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, + %ymm8, %rsi); + + vzeroupper; + + ret; +ENDPROC(camellia_ecb_dec_32way) + +ENTRY(camellia_cbc_dec_32way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (32 blocks) + * %rdx: src (32 blocks) + */ + + vzeroupper; + + cmpl $16, key_length(CTX); + movl $32, %r8d; + movl $24, %eax; + cmovel %eax, %r8d; /* max */ + + inpack32_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, + %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, + %ymm15, %rdx, (key_table)(CTX, %r8, 8)); + + movq %rsp, %r10; + cmpq %rsi, %rdx; + je .Lcbc_dec_use_stack; + + /* dst can be used as temporary storage, src is not overwritten. */ + movq %rsi, %rax; + jmp .Lcbc_dec_continue; + +.Lcbc_dec_use_stack: + /* + * dst still in-use (because dst == src), so use stack for temporary + * storage. + */ + subq $(16 * 32), %rsp; + movq %rsp, %rax; + +.Lcbc_dec_continue: + call __camellia_dec_blk32; + + vmovdqu %ymm7, (%rax); + vpxor %ymm7, %ymm7, %ymm7; + vinserti128 $1, (%rdx), %ymm7, %ymm7; + vpxor (%rax), %ymm7, %ymm7; + movq %r10, %rsp; + vpxor (0 * 32 + 16)(%rdx), %ymm6, %ymm6; + vpxor (1 * 32 + 16)(%rdx), %ymm5, %ymm5; + vpxor (2 * 32 + 16)(%rdx), %ymm4, %ymm4; + vpxor (3 * 32 + 16)(%rdx), %ymm3, %ymm3; + vpxor (4 * 32 + 16)(%rdx), %ymm2, %ymm2; + vpxor (5 * 32 + 16)(%rdx), %ymm1, %ymm1; + vpxor (6 * 32 + 16)(%rdx), %ymm0, %ymm0; + vpxor (7 * 32 + 16)(%rdx), %ymm15, %ymm15; + vpxor (8 * 32 + 16)(%rdx), %ymm14, %ymm14; + vpxor (9 * 32 + 16)(%rdx), %ymm13, %ymm13; + vpxor (10 * 32 + 16)(%rdx), %ymm12, %ymm12; + vpxor (11 * 32 + 16)(%rdx), %ymm11, %ymm11; + vpxor (12 * 32 + 16)(%rdx), %ymm10, %ymm10; + vpxor (13 * 32 + 16)(%rdx), %ymm9, %ymm9; + vpxor (14 * 32 + 16)(%rdx), %ymm8, %ymm8; + write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, + %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, + %ymm8, %rsi); + + vzeroupper; + + ret; +ENDPROC(camellia_cbc_dec_32way) + +#define inc_le128(x, minus_one, tmp) \ + vpcmpeqq minus_one, x, tmp; \ + vpsubq minus_one, x, x; \ + vpslldq $8, tmp, tmp; \ + vpsubq tmp, x, x; + +#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ + vpcmpeqq minus_one, x, tmp1; \ + vpcmpeqq minus_two, x, tmp2; \ + vpsubq minus_two, x, x; \ + vpor tmp2, tmp1, tmp1; \ + vpslldq $8, tmp1, tmp1; \ + vpsubq tmp1, x, x; + +ENTRY(camellia_ctr_32way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (32 blocks) + * %rdx: src (32 blocks) + * %rcx: iv (little endian, 128bit) + */ + + vzeroupper; + + movq %rsp, %r10; + cmpq %rsi, %rdx; + je .Lctr_use_stack; + + /* dst can be used as temporary storage, src is not overwritten. */ + movq %rsi, %rax; + jmp .Lctr_continue; + +.Lctr_use_stack: + subq $(16 * 32), %rsp; + movq %rsp, %rax; + +.Lctr_continue: + vpcmpeqd %ymm15, %ymm15, %ymm15; + vpsrldq $8, %ymm15, %ymm15; /* ab: -1:0 ; cd: -1:0 */ + vpaddq %ymm15, %ymm15, %ymm12; /* ab: -2:0 ; cd: -2:0 */ + + /* load IV and byteswap */ + vmovdqu (%rcx), %xmm0; + vmovdqa %xmm0, %xmm1; + inc_le128(%xmm0, %xmm15, %xmm14); + vbroadcasti128 .Lbswap128_mask, %ymm14; + vinserti128 $1, %xmm0, %ymm1, %ymm0; + vpshufb %ymm14, %ymm0, %ymm13; + vmovdqu %ymm13, 15 * 32(%rax); + + /* construct IVs */ + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); /* ab:le2 ; cd:le3 */ + vpshufb %ymm14, %ymm0, %ymm13; + vmovdqu %ymm13, 14 * 32(%rax); + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm13; + vmovdqu %ymm13, 13 * 32(%rax); + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm13; + vmovdqu %ymm13, 12 * 32(%rax); + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm13; + vmovdqu %ymm13, 11 * 32(%rax); + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm10; + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm9; + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm8; + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm7; + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm6; + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm5; + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm4; + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm3; + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm2; + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vpshufb %ymm14, %ymm0, %ymm1; + add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); + vextracti128 $1, %ymm0, %xmm13; + vpshufb %ymm14, %ymm0, %ymm0; + inc_le128(%xmm13, %xmm15, %xmm14); + vmovdqu %xmm13, (%rcx); + + /* inpack32_pre: */ + vpbroadcastq (key_table)(CTX), %ymm15; + vpshufb .Lpack_bswap, %ymm15, %ymm15; + vpxor %ymm0, %ymm15, %ymm0; + vpxor %ymm1, %ymm15, %ymm1; + vpxor %ymm2, %ymm15, %ymm2; + vpxor %ymm3, %ymm15, %ymm3; + vpxor %ymm4, %ymm15, %ymm4; + vpxor %ymm5, %ymm15, %ymm5; + vpxor %ymm6, %ymm15, %ymm6; + vpxor %ymm7, %ymm15, %ymm7; + vpxor %ymm8, %ymm15, %ymm8; + vpxor %ymm9, %ymm15, %ymm9; + vpxor %ymm10, %ymm15, %ymm10; + vpxor 11 * 32(%rax), %ymm15, %ymm11; + vpxor 12 * 32(%rax), %ymm15, %ymm12; + vpxor 13 * 32(%rax), %ymm15, %ymm13; + vpxor 14 * 32(%rax), %ymm15, %ymm14; + vpxor 15 * 32(%rax), %ymm15, %ymm15; + + call __camellia_enc_blk32; + + movq %r10, %rsp; + + vpxor 0 * 32(%rdx), %ymm7, %ymm7; + vpxor 1 * 32(%rdx), %ymm6, %ymm6; + vpxor 2 * 32(%rdx), %ymm5, %ymm5; + vpxor 3 * 32(%rdx), %ymm4, %ymm4; + vpxor 4 * 32(%rdx), %ymm3, %ymm3; + vpxor 5 * 32(%rdx), %ymm2, %ymm2; + vpxor 6 * 32(%rdx), %ymm1, %ymm1; + vpxor 7 * 32(%rdx), %ymm0, %ymm0; + vpxor 8 * 32(%rdx), %ymm15, %ymm15; + vpxor 9 * 32(%rdx), %ymm14, %ymm14; + vpxor 10 * 32(%rdx), %ymm13, %ymm13; + vpxor 11 * 32(%rdx), %ymm12, %ymm12; + vpxor 12 * 32(%rdx), %ymm11, %ymm11; + vpxor 13 * 32(%rdx), %ymm10, %ymm10; + vpxor 14 * 32(%rdx), %ymm9, %ymm9; + vpxor 15 * 32(%rdx), %ymm8, %ymm8; + write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, + %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, + %ymm8, %rsi); + + vzeroupper; + + ret; +ENDPROC(camellia_ctr_32way) + +#define gf128mul_x_ble(iv, mask, tmp) \ + vpsrad $31, iv, tmp; \ + vpaddq iv, iv, iv; \ + vpshufd $0x13, tmp, tmp; \ + vpand mask, tmp, tmp; \ + vpxor tmp, iv, iv; + +#define gf128mul_x2_ble(iv, mask1, mask2, tmp0, tmp1) \ + vpsrad $31, iv, tmp0; \ + vpaddq iv, iv, tmp1; \ + vpsllq $2, iv, iv; \ + vpshufd $0x13, tmp0, tmp0; \ + vpsrad $31, tmp1, tmp1; \ + vpand mask2, tmp0, tmp0; \ + vpshufd $0x13, tmp1, tmp1; \ + vpxor tmp0, iv, iv; \ + vpand mask1, tmp1, tmp1; \ + vpxor tmp1, iv, iv; + +.align 8 +camellia_xts_crypt_32way: + /* input: + * %rdi: ctx, CTX + * %rsi: dst (32 blocks) + * %rdx: src (32 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + * %r8: index for input whitening key + * %r9: pointer to __camellia_enc_blk32 or __camellia_dec_blk32 + */ + + vzeroupper; + + subq $(16 * 32), %rsp; + movq %rsp, %rax; + + vbroadcasti128 .Lxts_gf128mul_and_shl1_mask_0, %ymm12; + + /* load IV and construct second IV */ + vmovdqu (%rcx), %xmm0; + vmovdqa %xmm0, %xmm15; + gf128mul_x_ble(%xmm0, %xmm12, %xmm13); + vbroadcasti128 .Lxts_gf128mul_and_shl1_mask_1, %ymm13; + vinserti128 $1, %xmm0, %ymm15, %ymm0; + vpxor 0 * 32(%rdx), %ymm0, %ymm15; + vmovdqu %ymm15, 15 * 32(%rax); + vmovdqu %ymm0, 0 * 32(%rsi); + + /* construct IVs */ + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 1 * 32(%rdx), %ymm0, %ymm15; + vmovdqu %ymm15, 14 * 32(%rax); + vmovdqu %ymm0, 1 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 2 * 32(%rdx), %ymm0, %ymm15; + vmovdqu %ymm15, 13 * 32(%rax); + vmovdqu %ymm0, 2 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 3 * 32(%rdx), %ymm0, %ymm15; + vmovdqu %ymm15, 12 * 32(%rax); + vmovdqu %ymm0, 3 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 4 * 32(%rdx), %ymm0, %ymm11; + vmovdqu %ymm0, 4 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 5 * 32(%rdx), %ymm0, %ymm10; + vmovdqu %ymm0, 5 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 6 * 32(%rdx), %ymm0, %ymm9; + vmovdqu %ymm0, 6 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 7 * 32(%rdx), %ymm0, %ymm8; + vmovdqu %ymm0, 7 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 8 * 32(%rdx), %ymm0, %ymm7; + vmovdqu %ymm0, 8 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 9 * 32(%rdx), %ymm0, %ymm6; + vmovdqu %ymm0, 9 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 10 * 32(%rdx), %ymm0, %ymm5; + vmovdqu %ymm0, 10 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 11 * 32(%rdx), %ymm0, %ymm4; + vmovdqu %ymm0, 11 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 12 * 32(%rdx), %ymm0, %ymm3; + vmovdqu %ymm0, 12 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 13 * 32(%rdx), %ymm0, %ymm2; + vmovdqu %ymm0, 13 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 14 * 32(%rdx), %ymm0, %ymm1; + vmovdqu %ymm0, 14 * 32(%rsi); + + gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); + vpxor 15 * 32(%rdx), %ymm0, %ymm15; + vmovdqu %ymm15, 0 * 32(%rax); + vmovdqu %ymm0, 15 * 32(%rsi); + + vextracti128 $1, %ymm0, %xmm0; + gf128mul_x_ble(%xmm0, %xmm12, %xmm15); + vmovdqu %xmm0, (%rcx); + + /* inpack32_pre: */ + vpbroadcastq (key_table)(CTX, %r8, 8), %ymm15; + vpshufb .Lpack_bswap, %ymm15, %ymm15; + vpxor 0 * 32(%rax), %ymm15, %ymm0; + vpxor %ymm1, %ymm15, %ymm1; + vpxor %ymm2, %ymm15, %ymm2; + vpxor %ymm3, %ymm15, %ymm3; + vpxor %ymm4, %ymm15, %ymm4; + vpxor %ymm5, %ymm15, %ymm5; + vpxor %ymm6, %ymm15, %ymm6; + vpxor %ymm7, %ymm15, %ymm7; + vpxor %ymm8, %ymm15, %ymm8; + vpxor %ymm9, %ymm15, %ymm9; + vpxor %ymm10, %ymm15, %ymm10; + vpxor %ymm11, %ymm15, %ymm11; + vpxor 12 * 32(%rax), %ymm15, %ymm12; + vpxor 13 * 32(%rax), %ymm15, %ymm13; + vpxor 14 * 32(%rax), %ymm15, %ymm14; + vpxor 15 * 32(%rax), %ymm15, %ymm15; + + call *%r9; + + addq $(16 * 32), %rsp; + + vpxor 0 * 32(%rsi), %ymm7, %ymm7; + vpxor 1 * 32(%rsi), %ymm6, %ymm6; + vpxor 2 * 32(%rsi), %ymm5, %ymm5; + vpxor 3 * 32(%rsi), %ymm4, %ymm4; + vpxor 4 * 32(%rsi), %ymm3, %ymm3; + vpxor 5 * 32(%rsi), %ymm2, %ymm2; + vpxor 6 * 32(%rsi), %ymm1, %ymm1; + vpxor 7 * 32(%rsi), %ymm0, %ymm0; + vpxor 8 * 32(%rsi), %ymm15, %ymm15; + vpxor 9 * 32(%rsi), %ymm14, %ymm14; + vpxor 10 * 32(%rsi), %ymm13, %ymm13; + vpxor 11 * 32(%rsi), %ymm12, %ymm12; + vpxor 12 * 32(%rsi), %ymm11, %ymm11; + vpxor 13 * 32(%rsi), %ymm10, %ymm10; + vpxor 14 * 32(%rsi), %ymm9, %ymm9; + vpxor 15 * 32(%rsi), %ymm8, %ymm8; + write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, + %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, + %ymm8, %rsi); + + vzeroupper; + + ret; +ENDPROC(camellia_xts_crypt_32way) + +ENTRY(camellia_xts_enc_32way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (32 blocks) + * %rdx: src (32 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + xorl %r8d, %r8d; /* input whitening key, 0 for enc */ + + leaq __camellia_enc_blk32, %r9; + + jmp camellia_xts_crypt_32way; +ENDPROC(camellia_xts_enc_32way) + +ENTRY(camellia_xts_dec_32way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (32 blocks) + * %rdx: src (32 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + + cmpl $16, key_length(CTX); + movl $32, %r8d; + movl $24, %eax; + cmovel %eax, %r8d; /* input whitening key, last for dec */ + + leaq __camellia_dec_blk32, %r9; + + jmp camellia_xts_crypt_32way; +ENDPROC(camellia_xts_dec_32way) diff --git a/arch/x86/crypto/camellia_aesni_avx2_glue.c b/arch/x86/crypto/camellia_aesni_avx2_glue.c new file mode 100644 index 0000000..414fe5d --- /dev/null +++ b/arch/x86/crypto/camellia_aesni_avx2_glue.c @@ -0,0 +1,586 @@ +/* + * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia + * + * Copyright © 2013 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 +#define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32 + +/* 32-way AVX2/AES-NI parallel cipher functions */ +asmlinkage void camellia_ecb_enc_32way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void camellia_ecb_dec_32way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src); + +asmlinkage void camellia_cbc_dec_32way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void camellia_ctr_32way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + +asmlinkage void camellia_xts_enc_32way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +asmlinkage void camellia_xts_dec_32way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + +static const struct common_glue_ctx camellia_enc = { + .num_funcs = 4, + .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_32way) } + }, { + .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) } + }, { + .num_blocks = 2, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) } + } } +}; + +static const struct common_glue_ctx camellia_ctr = { + .num_funcs = 4, + .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_32way) } + }, { + .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) } + }, { + .num_blocks = 2, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) } + }, { + .num_blocks = 1, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) } + } } +}; + +static const struct common_glue_ctx camellia_enc_xts = { + .num_funcs = 3, + .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_32way) } + }, { + .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) } + } } +}; + +static const struct common_glue_ctx camellia_dec = { + .num_funcs = 4, + .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_32way) } + }, { + .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) } + }, { + .num_blocks = 2, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) } + } } +}; + +static const struct common_glue_ctx camellia_dec_cbc = { + .num_funcs = 4, + .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_32way) } + }, { + .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) } + }, { + .num_blocks = 2, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) } + }, { + .num_blocks = 1, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) } + } } +}; + +static const struct common_glue_ctx camellia_dec_xts = { + .num_funcs = 3, + .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_32way) } + }, { + .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) } + }, { + .num_blocks = 1, + .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) } + } } +}; + +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ecb_crypt_128bit(&camellia_enc, desc, dst, src, nbytes); +} + +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ecb_crypt_128bit(&camellia_dec, desc, dst, src, nbytes); +} + +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(camellia_enc_blk), desc, + dst, src, nbytes); +} + +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_cbc_decrypt_128bit(&camellia_dec_cbc, desc, dst, src, + nbytes); +} + +static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ctr_crypt_128bit(&camellia_ctr, desc, dst, src, nbytes); +} + +static inline bool camellia_fpu_begin(bool fpu_enabled, unsigned int nbytes) +{ + return glue_fpu_begin(CAMELLIA_BLOCK_SIZE, + CAMELLIA_AESNI_PARALLEL_BLOCKS, NULL, fpu_enabled, + nbytes); +} + +static inline void camellia_fpu_end(bool fpu_enabled) +{ + glue_fpu_end(fpu_enabled); +} + +static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + return __camellia_setkey(crypto_tfm_ctx(tfm), in_key, key_len, + &tfm->crt_flags); +} + +struct crypt_priv { + struct camellia_ctx *ctx; + bool fpu_enabled; +}; + +static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = CAMELLIA_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes); + + if (nbytes >= CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS * bsize) { + camellia_ecb_enc_32way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS; + nbytes -= bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS; + } + + if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) { + camellia_ecb_enc_16way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS; + nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS; + } + + while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) { + camellia_enc_blk_2way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS; + nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + camellia_enc_blk(ctx->ctx, srcdst, srcdst); +} + +static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = CAMELLIA_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes); + + if (nbytes >= CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS * bsize) { + camellia_ecb_dec_32way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS; + nbytes -= bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS; + } + + if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) { + camellia_ecb_dec_16way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS; + nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS; + } + + while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) { + camellia_dec_blk_2way(ctx->ctx, srcdst, srcdst); + srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS; + nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + camellia_dec_blk(ctx->ctx, srcdst, srcdst); +} + +static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->camellia_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = encrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = lrw_crypt(desc, dst, src, nbytes, &req); + camellia_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->camellia_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = decrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = lrw_crypt(desc, dst, src, nbytes, &req); + camellia_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + + return glue_xts_crypt_128bit(&camellia_enc_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(camellia_enc_blk), + &ctx->tweak_ctx, &ctx->crypt_ctx); +} + +static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + + return glue_xts_crypt_128bit(&camellia_dec_xts, desc, dst, src, nbytes, + XTS_TWEAK_CAST(camellia_enc_blk), + &ctx->tweak_ctx, &ctx->crypt_ctx); +} + +static struct crypto_alg cmll_algs[10] = { { + .cra_name = "__ecb-camellia-aesni-avx2", + .cra_driver_name = "__driver-ecb-camellia-aesni-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .setkey = camellia_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "__cbc-camellia-aesni-avx2", + .cra_driver_name = "__driver-cbc-camellia-aesni-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .setkey = camellia_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "__ctr-camellia-aesni-avx2", + .cra_driver_name = "__driver-ctr-camellia-aesni-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct camellia_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = camellia_setkey, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + }, + }, +}, { + .cra_name = "__lrw-camellia-aesni-avx2", + .cra_driver_name = "__driver-lrw-camellia-aesni-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_lrw_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_exit = lrw_camellia_exit_tfm, + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE + + CAMELLIA_BLOCK_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE + + CAMELLIA_BLOCK_SIZE, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = lrw_camellia_setkey, + .encrypt = lrw_encrypt, + .decrypt = lrw_decrypt, + }, + }, +}, { + .cra_name = "__xts-camellia-aesni-avx2", + .cra_driver_name = "__driver-xts-camellia-aesni-avx2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_xts_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2, + .max_keysize = CAMELLIA_MAX_KEY_SIZE * 2, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = xts_camellia_setkey, + .encrypt = xts_encrypt, + .decrypt = xts_decrypt, + }, + }, +}, { + .cra_name = "ecb(camellia)", + .cra_driver_name = "ecb-camellia-aesni-avx2", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "cbc(camellia)", + .cra_driver_name = "cbc-camellia-aesni-avx2", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = __ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "ctr(camellia)", + .cra_driver_name = "ctr-camellia-aesni-avx2", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_encrypt, + .geniv = "chainiv", + }, + }, +}, { + .cra_name = "lrw(camellia)", + .cra_driver_name = "lrw-camellia-aesni-avx2", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE + + CAMELLIA_BLOCK_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE + + CAMELLIA_BLOCK_SIZE, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "xts(camellia)", + .cra_driver_name = "xts-camellia-aesni-avx2", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2, + .max_keysize = CAMELLIA_MAX_KEY_SIZE * 2, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +} }; + +static int __init camellia_aesni_init(void) +{ + u64 xcr0; + + if (!cpu_has_avx2 || !cpu_has_avx || !cpu_has_aes || !cpu_has_osxsave) { + pr_info("AVX2 or AES-NI instructions are not detected.\n"); + return -ENODEV; + } + + xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); + if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { + pr_info("AVX2 detected but unusable.\n"); + return -ENODEV; + } + + return crypto_register_algs(cmll_algs, ARRAY_SIZE(cmll_algs)); +} + +static void __exit camellia_aesni_fini(void) +{ + crypto_unregister_algs(cmll_algs, ARRAY_SIZE(cmll_algs)); +} + +module_init(camellia_aesni_init); +module_exit(camellia_aesni_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized"); +MODULE_ALIAS("camellia"); +MODULE_ALIAS("camellia-asm"); diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c index 4ff7ed4..37fd0c0 100644 --- a/arch/x86/crypto/camellia_aesni_avx_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx_glue.c @@ -26,33 +26,44 @@ #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 -/* 16-way AES-NI parallel cipher functions */ +/* 16-way parallel cipher functions (avx/aes-ni) */ asmlinkage void camellia_ecb_enc_16way(struct camellia_ctx *ctx, u8 *dst, const u8 *src); +EXPORT_SYMBOL_GPL(camellia_ecb_enc_16way); + asmlinkage void camellia_ecb_dec_16way(struct camellia_ctx *ctx, u8 *dst, const u8 *src); +EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way); asmlinkage void camellia_cbc_dec_16way(struct camellia_ctx *ctx, u8 *dst, const u8 *src); +EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way); + asmlinkage void camellia_ctr_16way(struct camellia_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +EXPORT_SYMBOL_GPL(camellia_ctr_16way); asmlinkage void camellia_xts_enc_16way(struct camellia_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +EXPORT_SYMBOL_GPL(camellia_xts_enc_16way); + asmlinkage void camellia_xts_dec_16way(struct camellia_ctx *ctx, u8 *dst, const u8 *src, le128 *iv); +EXPORT_SYMBOL_GPL(camellia_xts_dec_16way); -static void camellia_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) +void camellia_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) { glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(camellia_enc_blk)); } +EXPORT_SYMBOL_GPL(camellia_xts_enc); -static void camellia_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) +void camellia_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) { glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(camellia_dec_blk)); } +EXPORT_SYMBOL_GPL(camellia_xts_dec); static const struct common_glue_ctx camellia_enc = { .num_funcs = 3, diff --git a/arch/x86/include/asm/crypto/camellia.h b/arch/x86/include/asm/crypto/camellia.h index 98038ad..bb93333 100644 --- a/arch/x86/include/asm/crypto/camellia.h +++ b/arch/x86/include/asm/crypto/camellia.h @@ -48,6 +48,22 @@ asmlinkage void __camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst, asmlinkage void camellia_dec_blk_2way(struct camellia_ctx *ctx, u8 *dst, const u8 *src); +/* 16-way parallel cipher functions (avx/aes-ni) */ +asmlinkage void camellia_ecb_enc_16way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void camellia_ecb_dec_16way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src); + +asmlinkage void camellia_cbc_dec_16way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void camellia_ctr_16way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + +asmlinkage void camellia_xts_enc_16way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); +asmlinkage void camellia_xts_dec_16way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src, le128 *iv); + static inline void camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst, const u8 *src) { @@ -79,4 +95,7 @@ extern void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, extern void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src, le128 *iv); +extern void camellia_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv); +extern void camellia_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv); + #endif /* ASM_X86_CAMELLIA_H */ diff --git a/crypto/Kconfig b/crypto/Kconfig index 9ad3d78..622d8a4 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -894,6 +894,29 @@ config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 See also: +config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64 + tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX2)" + depends on X86 && 64BIT + depends on CRYPTO + select CRYPTO_ALGAPI + select CRYPTO_CRYPTD + select CRYPTO_ABLK_HELPER_X86 + select CRYPTO_GLUE_HELPER_X86 + select CRYPTO_CAMELLIA_X86_64 + select CRYPTO_CAMELLIA_AESNI_AVX_X86_64 + select CRYPTO_LRW + select CRYPTO_XTS + help + Camellia cipher algorithm module (x86_64/AES-NI/AVX2). + + Camellia is a symmetric key block cipher developed jointly + at NTT and Mitsubishi Electric Corporation. + + The Camellia specifies three key sizes: 128, 192 and 256 bits. + + See also: + + config CRYPTO_CAMELLIA_SPARC64 tristate "Camellia cipher algorithm (SPARC64)" depends on SPARC64 diff --git a/crypto/testmgr.c b/crypto/testmgr.c index f5e13de..5823735 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1667,6 +1667,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "__driver-cbc-camellia-aesni", .test = alg_test_null, }, { + .alg = "__driver-cbc-camellia-aesni-avx2", + .test = alg_test_null, + }, { .alg = "__driver-cbc-cast5-avx", .test = alg_test_null, }, { @@ -1698,6 +1701,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "__driver-ecb-camellia-aesni", .test = alg_test_null, }, { + .alg = "__driver-ecb-camellia-aesni-avx2", + .test = alg_test_null, + }, { .alg = "__driver-ecb-cast5-avx", .test = alg_test_null, }, { @@ -1978,6 +1984,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "cryptd(__driver-cbc-camellia-aesni)", .test = alg_test_null, }, { + .alg = "cryptd(__driver-cbc-camellia-aesni-avx2)", + .test = alg_test_null, + }, { .alg = "cryptd(__driver-cbc-serpent-avx2)", .test = alg_test_null, }, { @@ -1991,6 +2000,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "cryptd(__driver-ecb-camellia-aesni)", .test = alg_test_null, }, { + .alg = "cryptd(__driver-ecb-camellia-aesni-avx2)", + .test = alg_test_null, + }, { .alg = "cryptd(__driver-ecb-cast5-avx)", .test = alg_test_null, }, { -- cgit v0.10.2 From 3862de1f6c442d53bd828d39f86d07d933a70605 Mon Sep 17 00:00:00 2001 From: Vakul Garg Date: Mon, 15 Apr 2013 09:55:51 +0530 Subject: crypto: caam - fix job ring cleanup code The job ring init function creates a platform device for each job ring. While the job ring is shutdown, e.g. while caam module removal, its platform device was not being removed. This leads to failure while reinsertion and then removal of caam module second time. The following kernel crash dump appears when caam module is reinserted and then removed again. This patch fixes it. root@p4080ds:~# rmmod caam.ko Unable to handle kernel paging request for data at address 0x00000008 Faulting instruction address: 0xf94aca18 Oops: Kernel access of bad area, sig: 11 [#1] SMP NR_CPUS=8 P4080 DS Modules linked in: caam(-) qoriq_dbg(O) [last unloaded: caam] NIP: f94aca18 LR: f94aca18 CTR: c029f950 REGS: eac47d60 TRAP: 0300 Tainted: G O (3.8.4-rt2) MSR: 00029002 CR: 22022484 XER: 20000000 DEAR: 00000008, ESR: 00000000 TASK = e49dfaf0[2110] 'rmmod' THREAD: eac46000 CPU: 1 GPR00: f94ad3f4 eac47e10 e49dfaf0 00000000 00000005 ea2ac210 ffffffff 00000000 GPR08: c286de68 e4977ce0 c029b1c0 00000001 c029f950 10029738 00000000 100e0000 GPR16: 00000000 10023d00 1000cbdc 1000cb8c 1000cbb8 00000000 c07dfecc 00000000 GPR24: c07e0000 00000000 1000cbd8 f94e0000 ffffffff 00000000 ea53cd40 00000000 NIP [f94aca18] caam_reset_hw_jr+0x18/0x1c0 [caam] LR [f94aca18] caam_reset_hw_jr+0x18/0x1c0 [caam] Call Trace: [eac47e10] [eac47e30] 0xeac47e30 (unreliable) [eac47e20] [f94ad3f4] caam_jr_shutdown+0x34/0x220 [caam] [eac47e60] [f94ac0e4] caam_remove+0x54/0xb0 [caam] [eac47e80] [c029fb38] __device_release_driver+0x68/0x120 [eac47e90] [c02a05c8] driver_detach+0xd8/0xe0 [eac47eb0] [c029f8e0] bus_remove_driver+0xa0/0x110 [eac47ed0] [c00768e4] sys_delete_module+0x144/0x270 [eac47f40] [c000e2f0] ret_from_syscall+0x0/0x3c Signed-off-by: Vakul Garg Signed-off-by: Bharat Bhushan Reviewed-by: Horia Geanta Signed-off-by: Herbert Xu diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 5cd4c1b..e4a16b7 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -41,6 +41,7 @@ struct caam_jrentry_info { /* Private sub-storage for a single JobR */ struct caam_drv_private_jr { struct device *parentdev; /* points back to controller dev */ + struct platform_device *jr_pdev;/* points to platform device for JR */ int ridx; struct caam_job_ring __iomem *rregs; /* JobR's register space */ struct tasklet_struct irqtask; diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 93d1407..b4aa773e 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -407,6 +407,7 @@ int caam_jr_shutdown(struct device *dev) dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, jrp->outring, outbusaddr); kfree(jrp->entinfo); + of_device_unregister(jrp->jr_pdev); return ret; } @@ -454,6 +455,8 @@ int caam_jr_probe(struct platform_device *pdev, struct device_node *np, kfree(jrpriv); return -EINVAL; } + + jrpriv->jr_pdev = jr_pdev; jrdev = &jr_pdev->dev; dev_set_drvdata(jrdev, jrpriv); ctrlpriv->jrdev[ring] = jrdev; @@ -472,6 +475,7 @@ int caam_jr_probe(struct platform_device *pdev, struct device_node *np, /* Now do the platform independent part */ error = caam_jr_init(jrdev); /* now turn on hardware */ if (error) { + of_device_unregister(jr_pdev); kfree(jrpriv); return error; } -- cgit v0.10.2