--- zzzz-none-000/linux-3.10.107/drivers/crypto/sahara.c 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/drivers/crypto/sahara.c 2021-02-04 17:41:59.000000000 +0000 @@ -3,6 +3,7 @@ * * Support for SAHARA cryptographic accelerator. * + * Copyright (c) 2014 Steffen Trumtrar * Copyright (c) 2013 Vista Silicon S.L. * Author: Javier Martin * @@ -15,6 +16,10 @@ #include #include +#include +#include +#include +#include #include #include @@ -22,12 +27,19 @@ #include #include #include +#include #include +#include #include +#include #include +#define SHA_BUFFER_LEN PAGE_SIZE +#define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE + #define SAHARA_NAME "sahara" #define SAHARA_VERSION_3 3 +#define SAHARA_VERSION_4 4 #define SAHARA_TIMEOUT_MS 1000 #define SAHARA_MAX_HW_DESC 2 #define SAHARA_MAX_HW_LINK 20 @@ -36,7 +48,6 @@ #define FLAGS_ENCRYPT BIT(0) #define FLAGS_CBC BIT(1) #define FLAGS_NEW_KEY BIT(3) -#define FLAGS_BUSY 4 #define SAHARA_HDR_BASE 0x00800000 #define SAHARA_HDR_SKHA_ALG_AES 0 @@ -50,6 +61,23 @@ #define SAHARA_HDR_CHA_MDHA (2 << 28) #define SAHARA_HDR_PARITY_BIT (1 << 31) +#define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000 +#define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000 +#define SAHARA_HDR_MDHA_HASH 0xA0850000 +#define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000 +#define SAHARA_HDR_MDHA_ALG_SHA1 0 +#define SAHARA_HDR_MDHA_ALG_MD5 1 +#define SAHARA_HDR_MDHA_ALG_SHA256 2 +#define SAHARA_HDR_MDHA_ALG_SHA224 3 +#define SAHARA_HDR_MDHA_PDATA (1 << 2) +#define SAHARA_HDR_MDHA_HMAC (1 << 3) +#define SAHARA_HDR_MDHA_INIT (1 << 5) +#define SAHARA_HDR_MDHA_IPAD (1 << 6) +#define SAHARA_HDR_MDHA_OPAD (1 << 7) +#define SAHARA_HDR_MDHA_SWAP (1 << 8) +#define SAHARA_HDR_MDHA_MAC_FULL (1 << 9) +#define SAHARA_HDR_MDHA_SSL (1 << 10) + /* SAHARA can only process one request at a time */ #define SAHARA_QUEUE_LENGTH 1 @@ -117,31 +145,72 @@ }; struct sahara_ctx { - struct sahara_dev *dev; unsigned long flags; + + /* AES-specific context */ int keylen; u8 key[AES_KEYSIZE_128]; struct crypto_ablkcipher *fallback; + + /* SHA-specific context */ + struct crypto_shash *shash_fallback; }; struct sahara_aes_reqctx { unsigned long mode; }; +/* + * struct sahara_sha_reqctx - private data per request + * @buf: holds data for requests smaller than block_size + * @rembuf: used to prepare one block_size-aligned request + * @context: hw-specific context for request. Digest is extracted from this + * @mode: specifies what type of hw-descriptor needs to be built + * @digest_size: length of digest for this request + * @context_size: length of hw-context for this request. + * Always digest_size + 4 + * @buf_cnt: number of bytes saved in buf + * @sg_in_idx: number of hw links + * @in_sg: scatterlist for input data + * @in_sg_chain: scatterlists for chained input data + * @total: total number of bytes for transfer + * @last: is this the last block + * @first: is this the first block + * @active: inside a transfer + */ +struct sahara_sha_reqctx { + u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE]; + u8 rembuf[SAHARA_MAX_SHA_BLOCK_SIZE]; + u8 context[SHA256_DIGEST_SIZE + 4]; + struct mutex mutex; + unsigned int mode; + unsigned int digest_size; + unsigned int context_size; + unsigned int buf_cnt; + unsigned int sg_in_idx; + struct scatterlist *in_sg; + struct scatterlist in_sg_chain[2]; + size_t total; + unsigned int last; + unsigned int first; + unsigned int active; +}; + struct sahara_dev { struct device *device; + unsigned int version; void __iomem *regs_base; struct clk *clk_ipg; struct clk *clk_ahb; + struct mutex queue_mutex; + struct task_struct *kthread; + struct completion dma_completion; struct sahara_ctx *ctx; spinlock_t lock; struct crypto_queue queue; unsigned long flags; - struct tasklet_struct done_task; - struct tasklet_struct queue_task; - struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC]; dma_addr_t hw_phys_desc[SAHARA_MAX_HW_DESC]; @@ -151,10 +220,12 @@ u8 *iv_base; dma_addr_t iv_phys_base; + u8 *context_base; + dma_addr_t context_phys_base; + struct sahara_hw_link *hw_link[SAHARA_MAX_HW_LINK]; dma_addr_t hw_phys_link[SAHARA_MAX_HW_LINK]; - struct ablkcipher_request *req; size_t total; struct scatterlist *in_sg; unsigned int nb_in_sg; @@ -162,7 +233,6 @@ unsigned int nb_out_sg; u32 error; - struct timer_list watchdog; }; static struct sahara_dev *dev_ptr; @@ -202,31 +272,7 @@ SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; } -static int sahara_sg_length(struct scatterlist *sg, - unsigned int total) -{ - int sg_nb; - unsigned int len; - struct scatterlist *sg_list; - - sg_nb = 0; - sg_list = sg; - - while (total) { - len = min(sg_list->length, total); - - sg_nb++; - total -= len; - - sg_list = sg_next(sg_list); - if (!sg_list) - total = 0; - } - - return sg_nb; -} - -static char *sahara_err_src[16] = { +static const char *sahara_err_src[16] = { "No error", "Header error", "Descriptor length error", @@ -245,14 +291,14 @@ "DMA error" }; -static char *sahara_err_dmasize[4] = { +static const char *sahara_err_dmasize[4] = { "Byte transfer", "Half-word transfer", "Word transfer", "Reserved" }; -static char *sahara_err_dmasrc[8] = { +static const char *sahara_err_dmasrc[8] = { "No error", "AHB bus error", "Internal IP bus error", @@ -263,7 +309,7 @@ "DMA HW error" }; -static char *sahara_cha_errsrc[12] = { +static const char *sahara_cha_errsrc[12] = { "Input buffer non-empty", "Illegal address", "Illegal mode", @@ -278,7 +324,7 @@ "Reserved" }; -static char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" }; +static const char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" }; static void sahara_decode_error(struct sahara_dev *dev, unsigned int error) { @@ -308,7 +354,7 @@ dev_err(dev->device, "\n"); } -static char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" }; +static const char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" }; static void sahara_decode_status(struct sahara_dev *dev, unsigned int status) { @@ -401,40 +447,13 @@ dev_dbg(dev->device, "\n"); } -static void sahara_aes_done_task(unsigned long data) -{ - struct sahara_dev *dev = (struct sahara_dev *)data; - - dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, - DMA_TO_DEVICE); - dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, - DMA_FROM_DEVICE); - - spin_lock(&dev->lock); - clear_bit(FLAGS_BUSY, &dev->flags); - spin_unlock(&dev->lock); - - dev->req->base.complete(&dev->req->base, dev->error); -} - -void sahara_watchdog(unsigned long data) -{ - struct sahara_dev *dev = (struct sahara_dev *)data; - unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS); - unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS); - - sahara_decode_status(dev, stat); - sahara_decode_error(dev, err); - dev->error = -ETIMEDOUT; - sahara_aes_done_task(data); -} - static int sahara_hw_descriptor_create(struct sahara_dev *dev) { struct sahara_ctx *ctx = dev->ctx; struct scatterlist *sg; int ret; int i, j; + int idx = 0; /* Copy new key if necessary */ if (ctx->flags & FLAGS_NEW_KEY) { @@ -442,20 +461,23 @@ ctx->flags &= ~FLAGS_NEW_KEY; if (dev->flags & FLAGS_CBC) { - dev->hw_desc[0]->len1 = AES_BLOCK_SIZE; - dev->hw_desc[0]->p1 = dev->iv_phys_base; + dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE; + dev->hw_desc[idx]->p1 = dev->iv_phys_base; } else { - dev->hw_desc[0]->len1 = 0; - dev->hw_desc[0]->p1 = 0; + dev->hw_desc[idx]->len1 = 0; + dev->hw_desc[idx]->p1 = 0; } - dev->hw_desc[0]->len2 = ctx->keylen; - dev->hw_desc[0]->p2 = dev->key_phys_base; - dev->hw_desc[0]->next = dev->hw_phys_desc[1]; + dev->hw_desc[idx]->len2 = ctx->keylen; + dev->hw_desc[idx]->p2 = dev->key_phys_base; + dev->hw_desc[idx]->next = dev->hw_phys_desc[1]; + + dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev); + + idx++; } - dev->hw_desc[0]->hdr = sahara_aes_key_hdr(dev); - dev->nb_in_sg = sahara_sg_length(dev->in_sg, dev->total); - dev->nb_out_sg = sahara_sg_length(dev->out_sg, dev->total); + dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total); + dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total); if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) { dev_err(dev->device, "not enough hw links (%d)\n", dev->nb_in_sg + dev->nb_out_sg); @@ -476,7 +498,7 @@ } /* Create input links */ - dev->hw_desc[1]->p1 = dev->hw_phys_link[0]; + dev->hw_desc[idx]->p1 = dev->hw_phys_link[0]; sg = dev->in_sg; for (i = 0; i < dev->nb_in_sg; i++) { dev->hw_link[i]->len = sg->length; @@ -490,7 +512,7 @@ } /* Create output links */ - dev->hw_desc[1]->p2 = dev->hw_phys_link[i]; + dev->hw_desc[idx]->p2 = dev->hw_phys_link[i]; sg = dev->out_sg; for (j = i; j < dev->nb_out_sg + i; j++) { dev->hw_link[j]->len = sg->length; @@ -504,17 +526,14 @@ } /* Fill remaining fields of hw_desc[1] */ - dev->hw_desc[1]->hdr = sahara_aes_data_link_hdr(dev); - dev->hw_desc[1]->len1 = dev->total; - dev->hw_desc[1]->len2 = dev->total; - dev->hw_desc[1]->next = 0; + dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev); + dev->hw_desc[idx]->len1 = dev->total; + dev->hw_desc[idx]->len2 = dev->total; + dev->hw_desc[idx]->next = 0; sahara_dump_descriptors(dev); sahara_dump_links(dev); - /* Start processing descriptor chain. */ - mod_timer(&dev->watchdog, - jiffies + msecs_to_jiffies(SAHARA_TIMEOUT_MS)); sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR); return 0; @@ -529,29 +548,13 @@ return -EINVAL; } -static void sahara_aes_queue_task(unsigned long data) +static int sahara_aes_process(struct ablkcipher_request *req) { - struct sahara_dev *dev = (struct sahara_dev *)data; - struct crypto_async_request *async_req, *backlog; + struct sahara_dev *dev = dev_ptr; struct sahara_ctx *ctx; struct sahara_aes_reqctx *rctx; - struct ablkcipher_request *req; int ret; - - spin_lock(&dev->lock); - backlog = crypto_get_backlog(&dev->queue); - async_req = crypto_dequeue_request(&dev->queue); - if (!async_req) - clear_bit(FLAGS_BUSY, &dev->flags); - spin_unlock(&dev->lock); - - if (!async_req) - return; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - req = ablkcipher_request_cast(async_req); + unsigned long timeout; /* Request is ready to be dispatched by the device */ dev_dbg(dev->device, @@ -559,7 +562,6 @@ req->nbytes, req->src, req->dst); /* assign new request to device */ - dev->req = req; dev->total = req->nbytes; dev->in_sg = req->src; dev->out_sg = req->dst; @@ -573,16 +575,27 @@ memcpy(dev->iv_base, req->info, AES_KEYSIZE_128); /* assign new context to device */ - ctx->dev = dev; dev->ctx = ctx; + reinit_completion(&dev->dma_completion); + ret = sahara_hw_descriptor_create(dev); - if (ret < 0) { - spin_lock(&dev->lock); - clear_bit(FLAGS_BUSY, &dev->flags); - spin_unlock(&dev->lock); - dev->req->base.complete(&dev->req->base, ret); + if (ret) + return -EINVAL; + + timeout = wait_for_completion_timeout(&dev->dma_completion, + msecs_to_jiffies(SAHARA_TIMEOUT_MS)); + if (!timeout) { + dev_err(dev->device, "AES timeout\n"); + return -ETIMEDOUT; } + + dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, + DMA_TO_DEVICE); + dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_FROM_DEVICE); + + return 0; } static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, @@ -624,12 +637,9 @@ static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode) { - struct sahara_ctx *ctx = crypto_ablkcipher_ctx( - crypto_ablkcipher_reqtfm(req)); struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req); struct sahara_dev *dev = dev_ptr; int err = 0; - int busy; dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n", req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC)); @@ -640,16 +650,13 @@ return -EINVAL; } - ctx->dev = dev; - rctx->mode = mode; - spin_lock_bh(&dev->lock); + + mutex_lock(&dev->queue_mutex); err = ablkcipher_enqueue_request(&dev->queue, req); - busy = test_and_set_bit(FLAGS_BUSY, &dev->flags); - spin_unlock_bh(&dev->lock); + mutex_unlock(&dev->queue_mutex); - if (!busy) - tasklet_schedule(&dev->queue_task); + wake_up_process(dev->kthread); return err; } @@ -728,7 +735,7 @@ static int sahara_aes_cra_init(struct crypto_tfm *tfm) { - const char *name = tfm->__crt_alg->cra_name; + const char *name = crypto_tfm_alg_name(tfm); struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); ctx->fallback = crypto_alloc_ablkcipher(name, 0, @@ -752,6 +759,452 @@ ctx->fallback = NULL; } +static u32 sahara_sha_init_hdr(struct sahara_dev *dev, + struct sahara_sha_reqctx *rctx) +{ + u32 hdr = 0; + + hdr = rctx->mode; + + if (rctx->first) { + hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH; + hdr |= SAHARA_HDR_MDHA_INIT; + } else { + hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY; + } + + if (rctx->last) + hdr |= SAHARA_HDR_MDHA_PDATA; + + if (hweight_long(hdr) % 2 == 0) + hdr |= SAHARA_HDR_PARITY_BIT; + + return hdr; +} + +static int sahara_sha_hw_links_create(struct sahara_dev *dev, + struct sahara_sha_reqctx *rctx, + int start) +{ + struct scatterlist *sg; + unsigned int i; + int ret; + + dev->in_sg = rctx->in_sg; + + dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total); + if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) { + dev_err(dev->device, "not enough hw links (%d)\n", + dev->nb_in_sg + dev->nb_out_sg); + return -EINVAL; + } + + sg = dev->in_sg; + ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); + if (!ret) + return -EFAULT; + + for (i = start; i < dev->nb_in_sg + start; i++) { + dev->hw_link[i]->len = sg->length; + dev->hw_link[i]->p = sg->dma_address; + if (i == (dev->nb_in_sg + start - 1)) { + dev->hw_link[i]->next = 0; + } else { + dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; + sg = sg_next(sg); + } + } + + return i; +} + +static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev, + struct sahara_sha_reqctx *rctx, + struct ahash_request *req, + int index) +{ + unsigned result_len; + int i = index; + + if (rctx->first) + /* Create initial descriptor: #8*/ + dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx); + else + /* Create hash descriptor: #10. Must follow #6. */ + dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH; + + dev->hw_desc[index]->len1 = rctx->total; + if (dev->hw_desc[index]->len1 == 0) { + /* if len1 is 0, p1 must be 0, too */ + dev->hw_desc[index]->p1 = 0; + rctx->sg_in_idx = 0; + } else { + /* Create input links */ + dev->hw_desc[index]->p1 = dev->hw_phys_link[index]; + i = sahara_sha_hw_links_create(dev, rctx, index); + + rctx->sg_in_idx = index; + if (i < 0) + return i; + } + + dev->hw_desc[index]->p2 = dev->hw_phys_link[i]; + + /* Save the context for the next operation */ + result_len = rctx->context_size; + dev->hw_link[i]->p = dev->context_phys_base; + + dev->hw_link[i]->len = result_len; + dev->hw_desc[index]->len2 = result_len; + + dev->hw_link[i]->next = 0; + + return 0; +} + +/* + * Load descriptor aka #6 + * + * To load a previously saved context back to the MDHA unit + * + * p1: Saved Context + * p2: NULL + * + */ +static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev, + struct sahara_sha_reqctx *rctx, + struct ahash_request *req, + int index) +{ + dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx); + + dev->hw_desc[index]->len1 = rctx->context_size; + dev->hw_desc[index]->p1 = dev->hw_phys_link[index]; + dev->hw_desc[index]->len2 = 0; + dev->hw_desc[index]->p2 = 0; + + dev->hw_link[index]->len = rctx->context_size; + dev->hw_link[index]->p = dev->context_phys_base; + dev->hw_link[index]->next = 0; + + return 0; +} + +static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes) +{ + if (!sg || !sg->length) + return nbytes; + + while (nbytes && sg) { + if (nbytes <= sg->length) { + sg->length = nbytes; + sg_mark_end(sg); + break; + } + nbytes -= sg->length; + sg = sg_next(sg); + } + + return nbytes; +} + +static int sahara_sha_prepare_request(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + unsigned int hash_later; + unsigned int block_size; + unsigned int len; + + block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + /* append bytes from previous operation */ + len = rctx->buf_cnt + req->nbytes; + + /* only the last transfer can be padded in hardware */ + if (!rctx->last && (len < block_size)) { + /* to few data, save for next operation */ + scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src, + 0, req->nbytes, 0); + rctx->buf_cnt += req->nbytes; + + return 0; + } + + /* add data from previous operation first */ + if (rctx->buf_cnt) + memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt); + + /* data must always be a multiple of block_size */ + hash_later = rctx->last ? 0 : len & (block_size - 1); + if (hash_later) { + unsigned int offset = req->nbytes - hash_later; + /* Save remaining bytes for later use */ + scatterwalk_map_and_copy(rctx->buf, req->src, offset, + hash_later, 0); + } + + /* nbytes should now be multiple of blocksize */ + req->nbytes = req->nbytes - hash_later; + + sahara_walk_and_recalc(req->src, req->nbytes); + + /* have data from previous operation and current */ + if (rctx->buf_cnt && req->nbytes) { + sg_init_table(rctx->in_sg_chain, 2); + sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt); + + sg_chain(rctx->in_sg_chain, 2, req->src); + + rctx->total = req->nbytes + rctx->buf_cnt; + rctx->in_sg = rctx->in_sg_chain; + + req->src = rctx->in_sg_chain; + /* only data from previous operation */ + } else if (rctx->buf_cnt) { + if (req->src) + rctx->in_sg = req->src; + else + rctx->in_sg = rctx->in_sg_chain; + /* buf was copied into rembuf above */ + sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt); + rctx->total = rctx->buf_cnt; + /* no data from previous operation */ + } else { + rctx->in_sg = req->src; + rctx->total = req->nbytes; + req->src = rctx->in_sg; + } + + /* on next call, we only have the remaining data in the buffer */ + rctx->buf_cnt = hash_later; + + return -EINPROGRESS; +} + +static int sahara_sha_process(struct ahash_request *req) +{ + struct sahara_dev *dev = dev_ptr; + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + int ret; + unsigned long timeout; + + ret = sahara_sha_prepare_request(req); + if (!ret) + return ret; + + if (rctx->first) { + sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0); + dev->hw_desc[0]->next = 0; + rctx->first = 0; + } else { + memcpy(dev->context_base, rctx->context, rctx->context_size); + + sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0); + dev->hw_desc[0]->next = dev->hw_phys_desc[1]; + sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1); + dev->hw_desc[1]->next = 0; + } + + sahara_dump_descriptors(dev); + sahara_dump_links(dev); + + reinit_completion(&dev->dma_completion); + + sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR); + + timeout = wait_for_completion_timeout(&dev->dma_completion, + msecs_to_jiffies(SAHARA_TIMEOUT_MS)); + if (!timeout) { + dev_err(dev->device, "SHA timeout\n"); + return -ETIMEDOUT; + } + + if (rctx->sg_in_idx) + dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_TO_DEVICE); + + memcpy(rctx->context, dev->context_base, rctx->context_size); + + if (req->result) + memcpy(req->result, rctx->context, rctx->digest_size); + + return 0; +} + +static int sahara_queue_manage(void *data) +{ + struct sahara_dev *dev = (struct sahara_dev *)data; + struct crypto_async_request *async_req; + struct crypto_async_request *backlog; + int ret = 0; + + do { + __set_current_state(TASK_INTERRUPTIBLE); + + mutex_lock(&dev->queue_mutex); + backlog = crypto_get_backlog(&dev->queue); + async_req = crypto_dequeue_request(&dev->queue); + mutex_unlock(&dev->queue_mutex); + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + if (async_req) { + if (crypto_tfm_alg_type(async_req->tfm) == + CRYPTO_ALG_TYPE_AHASH) { + struct ahash_request *req = + ahash_request_cast(async_req); + + ret = sahara_sha_process(req); + } else { + struct ablkcipher_request *req = + ablkcipher_request_cast(async_req); + + ret = sahara_aes_process(req); + } + + async_req->complete(async_req, ret); + + continue; + } + + schedule(); + } while (!kthread_should_stop()); + + return 0; +} + +static int sahara_sha_enqueue(struct ahash_request *req, int last) +{ + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + struct sahara_dev *dev = dev_ptr; + int ret; + + if (!req->nbytes && !last) + return 0; + + mutex_lock(&rctx->mutex); + rctx->last = last; + + if (!rctx->active) { + rctx->active = 1; + rctx->first = 1; + } + + mutex_lock(&dev->queue_mutex); + ret = crypto_enqueue_request(&dev->queue, &req->base); + mutex_unlock(&dev->queue_mutex); + + wake_up_process(dev->kthread); + mutex_unlock(&rctx->mutex); + + return ret; +} + +static int sahara_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1; + rctx->digest_size = SHA1_DIGEST_SIZE; + break; + case SHA256_DIGEST_SIZE: + rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256; + rctx->digest_size = SHA256_DIGEST_SIZE; + break; + default: + return -EINVAL; + } + + rctx->context_size = rctx->digest_size + 4; + rctx->active = 0; + + mutex_init(&rctx->mutex); + + return 0; +} + +static int sahara_sha_update(struct ahash_request *req) +{ + return sahara_sha_enqueue(req, 0); +} + +static int sahara_sha_final(struct ahash_request *req) +{ + req->nbytes = 0; + return sahara_sha_enqueue(req, 1); +} + +static int sahara_sha_finup(struct ahash_request *req) +{ + return sahara_sha_enqueue(req, 1); +} + +static int sahara_sha_digest(struct ahash_request *req) +{ + sahara_sha_init(req); + + return sahara_sha_finup(req); +} + +static int sahara_sha_export(struct ahash_request *req, void *out) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct sahara_ctx *ctx = crypto_ahash_ctx(ahash); + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + + memcpy(out, ctx, sizeof(struct sahara_ctx)); + memcpy(out + sizeof(struct sahara_sha_reqctx), rctx, + sizeof(struct sahara_sha_reqctx)); + + return 0; +} + +static int sahara_sha_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct sahara_ctx *ctx = crypto_ahash_ctx(ahash); + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + + memcpy(ctx, in, sizeof(struct sahara_ctx)); + memcpy(rctx, in + sizeof(struct sahara_sha_reqctx), + sizeof(struct sahara_sha_reqctx)); + + return 0; +} + +static int sahara_sha_cra_init(struct crypto_tfm *tfm) +{ + const char *name = crypto_tfm_alg_name(tfm); + struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->shash_fallback = crypto_alloc_shash(name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->shash_fallback)) { + pr_err("Error allocating fallback algo %s\n", name); + return PTR_ERR(ctx->shash_fallback); + } + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct sahara_sha_reqctx) + + SHA_BUFFER_LEN + SHA256_BLOCK_SIZE); + + return 0; +} + +static void sahara_sha_cra_exit(struct crypto_tfm *tfm) +{ + struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_shash(ctx->shash_fallback); + ctx->shash_fallback = NULL; +} + static struct crypto_alg aes_algs[] = { { .cra_name = "ecb(aes)", @@ -797,14 +1250,66 @@ } }; +static struct ahash_alg sha_v3_algs[] = { +{ + .init = sahara_sha_init, + .update = sahara_sha_update, + .final = sahara_sha_final, + .finup = sahara_sha_finup, + .digest = sahara_sha_digest, + .export = sahara_sha_export, + .import = sahara_sha_import, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "sahara-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sahara_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = sahara_sha_cra_init, + .cra_exit = sahara_sha_cra_exit, + } +}, +}; + +static struct ahash_alg sha_v4_algs[] = { +{ + .init = sahara_sha_init, + .update = sahara_sha_update, + .final = sahara_sha_final, + .finup = sahara_sha_finup, + .digest = sahara_sha_digest, + .export = sahara_sha_export, + .import = sahara_sha_import, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "sahara-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sahara_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = sahara_sha_cra_init, + .cra_exit = sahara_sha_cra_exit, + } +}, +}; + static irqreturn_t sahara_irq_handler(int irq, void *data) { struct sahara_dev *dev = (struct sahara_dev *)data; unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS); unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS); - del_timer(&dev->watchdog); - sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR, SAHARA_REG_CMD); @@ -819,7 +1324,7 @@ dev->error = -EINVAL; } - tasklet_schedule(&dev->done_task); + complete(&dev->dma_completion); return IRQ_HANDLED; } @@ -827,7 +1332,8 @@ static int sahara_register_algs(struct sahara_dev *dev) { - int err, i, j; + int err; + unsigned int i, j, k, l; for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { INIT_LIST_HEAD(&aes_algs[i].cra_list); @@ -836,8 +1342,29 @@ goto err_aes_algs; } + for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) { + err = crypto_register_ahash(&sha_v3_algs[k]); + if (err) + goto err_sha_v3_algs; + } + + if (dev->version > SAHARA_VERSION_3) + for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) { + err = crypto_register_ahash(&sha_v4_algs[l]); + if (err) + goto err_sha_v4_algs; + } + return 0; +err_sha_v4_algs: + for (j = 0; j < l; j++) + crypto_unregister_ahash(&sha_v4_algs[j]); + +err_sha_v3_algs: + for (j = 0; j < k; j++) + crypto_unregister_ahash(&sha_v4_algs[j]); + err_aes_algs: for (j = 0; j < i; j++) crypto_unregister_alg(&aes_algs[j]); @@ -847,10 +1374,17 @@ static void sahara_unregister_algs(struct sahara_dev *dev) { - int i; + unsigned int i; for (i = 0; i < ARRAY_SIZE(aes_algs); i++) crypto_unregister_alg(&aes_algs[i]); + + for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++) + crypto_unregister_ahash(&sha_v3_algs[i]); + + if (dev->version > SAHARA_VERSION_3) + for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++) + crypto_unregister_ahash(&sha_v4_algs[i]); } static struct platform_device_id sahara_platform_ids[] = { @@ -860,6 +1394,7 @@ MODULE_DEVICE_TABLE(platform, sahara_platform_ids); static struct of_device_id sahara_dt_ids[] = { + { .compatible = "fsl,imx53-sahara" }, { .compatible = "fsl,imx27-sahara" }, { /* sentinel */ } }; @@ -885,22 +1420,9 @@ /* Get the base address */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_err(&pdev->dev, "failed to get memory region resource\n"); - return -ENODEV; - } - - if (devm_request_mem_region(&pdev->dev, res->start, - resource_size(res), SAHARA_NAME) == NULL) { - dev_err(&pdev->dev, "failed to request memory region\n"); - return -ENOENT; - } - dev->regs_base = devm_ioremap(&pdev->dev, res->start, - resource_size(res)); - if (!dev->regs_base) { - dev_err(&pdev->dev, "failed to ioremap address region\n"); - return -ENOENT; - } + dev->regs_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(dev->regs_base)) + return PTR_ERR(dev->regs_base); /* Get the IRQ */ irq = platform_get_irq(pdev, 0); @@ -909,10 +1431,11 @@ return irq; } - if (devm_request_irq(&pdev->dev, irq, sahara_irq_handler, - 0, SAHARA_NAME, dev) < 0) { + err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler, + 0, dev_name(&pdev->dev), dev); + if (err) { dev_err(&pdev->dev, "failed to request irq\n"); - return -ENOENT; + return err; } /* clocks */ @@ -929,7 +1452,7 @@ } /* Allocate HW descriptors */ - dev->hw_desc[0] = dma_alloc_coherent(&pdev->dev, + dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev, SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), &dev->hw_phys_desc[0], GFP_KERNEL); if (!dev->hw_desc[0]) { @@ -941,24 +1464,31 @@ sizeof(struct sahara_hw_desc); /* Allocate space for iv and key */ - dev->key_base = dma_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, + dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, &dev->key_phys_base, GFP_KERNEL); if (!dev->key_base) { dev_err(&pdev->dev, "Could not allocate memory for key\n"); - err = -ENOMEM; - goto err_key; + return -ENOMEM; } dev->iv_base = dev->key_base + AES_KEYSIZE_128; dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128; + /* Allocate space for context: largest digest + message length field */ + dev->context_base = dmam_alloc_coherent(&pdev->dev, + SHA256_DIGEST_SIZE + 4, + &dev->context_phys_base, GFP_KERNEL); + if (!dev->context_base) { + dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n"); + return -ENOMEM; + } + /* Allocate space for HW links */ - dev->hw_link[0] = dma_alloc_coherent(&pdev->dev, + dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev, SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), &dev->hw_phys_link[0], GFP_KERNEL); - if (!dev->hw_link) { + if (!dev->hw_link[0]) { dev_err(&pdev->dev, "Could not allocate hw links\n"); - err = -ENOMEM; - goto err_link; + return -ENOMEM; } for (i = 1; i < SAHARA_MAX_HW_LINK; i++) { dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] + @@ -968,28 +1498,43 @@ crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH); + spin_lock_init(&dev->lock); + mutex_init(&dev->queue_mutex); + dev_ptr = dev; - tasklet_init(&dev->queue_task, sahara_aes_queue_task, - (unsigned long)dev); - tasklet_init(&dev->done_task, sahara_aes_done_task, - (unsigned long)dev); - - init_timer(&dev->watchdog); - dev->watchdog.function = &sahara_watchdog; - dev->watchdog.data = (unsigned long)dev; + dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto"); + if (IS_ERR(dev->kthread)) { + return PTR_ERR(dev->kthread); + } - clk_prepare_enable(dev->clk_ipg); - clk_prepare_enable(dev->clk_ahb); + init_completion(&dev->dma_completion); + + err = clk_prepare_enable(dev->clk_ipg); + if (err) + return err; + err = clk_prepare_enable(dev->clk_ahb); + if (err) + goto clk_ipg_disable; version = sahara_read(dev, SAHARA_REG_VERSION); - if (version != SAHARA_VERSION_3) { + if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) { + if (version != SAHARA_VERSION_3) + err = -ENODEV; + } else if (of_device_is_compatible(pdev->dev.of_node, + "fsl,imx53-sahara")) { + if (((version >> 8) & 0xff) != SAHARA_VERSION_4) + err = -ENODEV; + version = (version >> 8) & 0xff; + } + if (err == -ENODEV) { dev_err(&pdev->dev, "SAHARA version %d not supported\n", - version); - err = -ENODEV; + version); goto err_algs; } + dev->version = version; + sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH, SAHARA_REG_CMD); sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) | @@ -1007,20 +1552,11 @@ return 0; err_algs: - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), - dev->hw_link[0], dev->hw_phys_link[0]); - clk_disable_unprepare(dev->clk_ipg); - clk_disable_unprepare(dev->clk_ahb); + kthread_stop(dev->kthread); dev_ptr = NULL; -err_link: - dma_free_coherent(&pdev->dev, - 2 * AES_KEYSIZE_128, - dev->key_base, dev->key_phys_base); -err_key: - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), - dev->hw_desc[0], dev->hw_phys_desc[0]); + clk_disable_unprepare(dev->clk_ahb); +clk_ipg_disable: + clk_disable_unprepare(dev->clk_ipg); return err; } @@ -1029,18 +1565,7 @@ { struct sahara_dev *dev = platform_get_drvdata(pdev); - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), - dev->hw_link[0], dev->hw_phys_link[0]); - dma_free_coherent(&pdev->dev, - 2 * AES_KEYSIZE_128, - dev->key_base, dev->key_phys_base); - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), - dev->hw_desc[0], dev->hw_phys_desc[0]); - - tasklet_kill(&dev->done_task); - tasklet_kill(&dev->queue_task); + kthread_stop(dev->kthread); sahara_unregister_algs(dev); @@ -1057,8 +1582,7 @@ .remove = sahara_remove, .driver = { .name = SAHARA_NAME, - .owner = THIS_MODULE, - .of_match_table = of_match_ptr(sahara_dt_ids), + .of_match_table = sahara_dt_ids, }, .id_table = sahara_platform_ids, }; @@ -1067,4 +1591,5 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Javier Martin "); +MODULE_AUTHOR("Steffen Trumtrar "); MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");