/* Qualcomm Crypto Engine driver. * * Copyright (c) 2012-2017, 2021 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and * only version 2 as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. */ #define pr_fmt(fmt) "QCE50: %s: " fmt, __func__ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "qce.h" #include "qce50.h" #include "qcryptohw_50.h" #include "qce_ota.h" #define CRYPTO_CONFIG_RESET 0xE01EF #define MAX_SPS_DESC_FIFO_SIZE 0xfff0 #define QCE_MAX_NUM_DSCR 0x200 #define QCE_SECTOR_SIZE 0x200 #define CE_CLK_100MHZ 100000000 #define CE_CLK_DIV 1000000 #define CRYPTO_CORE_MAJOR_VER_NUM 0x05 #define CRYPTO_CORE_MINOR_VER_NUM 0x03 #define CRYPTO_CORE_STEP_VER_NUM 0x1 #define CRYPTO_REQ_USER_PAT 0xdead0000 static DEFINE_MUTEX(bam_register_lock); static DEFINE_MUTEX(qce_iomap_mutex); struct bam_registration_info { struct list_head qlist; unsigned long handle; uint32_t cnt; uint32_t bam_mem; void __iomem *bam_iobase; bool support_cmd_dscr; }; static LIST_HEAD(qce50_bam_list); /* Used to determine the mode */ #define MAX_BUNCH_MODE_REQ 2 /* Max number of request supported */ #define MAX_QCE_BAM_REQ 8 /* Interrupt flag will be set for every SET_INTR_AT_REQ request */ #define SET_INTR_AT_REQ (MAX_QCE_BAM_REQ / 2) /* To create extra request space to hold dummy request */ #define MAX_QCE_BAM_REQ_WITH_DUMMY_REQ (MAX_QCE_BAM_REQ + 1) /* Allocate the memory for MAX_QCE_BAM_REQ + 1 (for dummy request) */ #define MAX_QCE_ALLOC_BAM_REQ MAX_QCE_BAM_REQ_WITH_DUMMY_REQ /* QCE driver modes */ #define IN_INTERRUPT_MODE 0 #define IN_BUNCH_MODE 1 /* Dummy request data length */ #define DUMMY_REQ_DATA_LEN 64 /* Delay timer to expire when in bunch mode */ #define DELAY_IN_JIFFIES 5 /* Index to point the dummy request */ #define DUMMY_REQ_INDEX MAX_QCE_BAM_REQ #define TOTAL_IOVEC_SPACE_PER_PIPE (QCE_MAX_NUM_DSCR * sizeof(struct sps_iovec)) enum qce_owner { QCE_OWNER_NONE = 0, QCE_OWNER_CLIENT = 1, QCE_OWNER_TIMEOUT = 2 }; struct dummy_request { struct qce_sha_req sreq; struct scatterlist sg; struct ahash_request areq; }; /* * CE HW device structure. * Each engine has an instance of the structure. * Each engine can only handle one crypto operation at one time. It is up to * the sw above to ensure single threading of operation on an engine. */ struct qce_device { struct device *pdev; /* Handle to platform_device structure */ struct bam_registration_info *pbam; unsigned char *coh_vmem; /* Allocated coherent virtual memory */ dma_addr_t coh_pmem; /* Allocated coherent physical memory */ int memsize; /* Memory allocated */ unsigned char *iovec_vmem; /* Allocate iovec virtual memory */ int iovec_memsize; /* Memory allocated */ uint32_t bam_mem; /* bam physical address, from DT */ uint32_t bam_mem_size; /* bam io size, from DT */ int is_shared; /* CE HW is shared */ bool support_cmd_dscr; bool support_hw_key; bool support_clk_mgmt_sus_res; bool support_only_core_src_clk; void __iomem *iobase; /* Virtual io base of CE HW */ unsigned int phy_iobase; /* Physical io base of CE HW */ struct clk *ce_core_src_clk; /* Handle to CE src clk*/ struct clk *ce_core_clk; /* Handle to CE clk */ struct clk *ce_clk; /* Handle to CE clk */ struct clk *ce_bus_clk; /* Handle to CE AXI clk*/ bool no_get_around; bool no_ccm_mac_status_get_around; unsigned int ce_opp_freq_hz; bool use_sw_aes_cbc_ecb_ctr_algo; bool use_sw_aead_algo; bool use_sw_aes_xts_algo; bool use_sw_ahash_algo; bool use_sw_hmac_algo; bool use_sw_aes_ccm_algo; uint32_t engines_avail; struct qce_ce_cfg_reg_setting reg; struct ce_bam_info ce_bam_info; struct ce_request_info ce_request_info[MAX_QCE_ALLOC_BAM_REQ]; unsigned int ce_request_index; enum qce_owner owner; atomic_t no_of_queued_req; struct timer_list timer; struct dummy_request dummyreq; unsigned int mode; unsigned int intr_cadence; unsigned int dev_no; struct qce_driver_stats qce_stats; atomic_t bunch_cmd_seq; atomic_t last_intr_seq; bool cadence_flag; uint8_t *dummyreq_in_buf; }; static void print_notify_debug(struct sps_event_notify *notify); static void _sps_producer_callback(struct sps_event_notify *notify); static int qce_dummy_req(struct qce_device *pce_dev); static int _qce50_disp_stats; /* Standard initialization vector for SHA-1, source: FIPS 180-2 */ static uint32_t _std_init_vector_sha1[] = { 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0 }; /* Standard initialization vector for SHA-256, source: FIPS 180-2 */ static uint32_t _std_init_vector_sha256[] = { 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 }; static void _byte_stream_to_net_words(uint32_t *iv, unsigned char *b, unsigned int len) { unsigned n; n = len / sizeof(uint32_t); for (; n > 0; n--) { *iv = ((*b << 24) & 0xff000000) | (((*(b+1)) << 16) & 0xff0000) | (((*(b+2)) << 8) & 0xff00) | (*(b+3) & 0xff); b += sizeof(uint32_t); iv++; } n = len % sizeof(uint32_t); if (n == 3) { *iv = ((*b << 24) & 0xff000000) | (((*(b+1)) << 16) & 0xff0000) | (((*(b+2)) << 8) & 0xff00); } else if (n == 2) { *iv = ((*b << 24) & 0xff000000) | (((*(b+1)) << 16) & 0xff0000); } else if (n == 1) { *iv = ((*b << 24) & 0xff000000); } } static void _byte_stream_swap_to_net_words(uint32_t *iv, unsigned char *b, unsigned int len) { unsigned i, j; unsigned char swap_iv[AES_IV_LENGTH]; memset(swap_iv, 0, AES_IV_LENGTH); for (i = (AES_IV_LENGTH-len), j = len-1; i < AES_IV_LENGTH; i++, j--) swap_iv[i] = b[j]; _byte_stream_to_net_words(iv, swap_iv, AES_IV_LENGTH); } static int count_sg(struct scatterlist *sg, int nbytes) { int i; for (i = 0; nbytes > 0 && sg != NULL; i++, sg = sg_next(sg)) { nbytes -= sg->length; } return i; } static int qce_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction) { int i; for (i = 0; ((i < nents) && (sg != NULL)); ++i) { dma_map_sg(dev, sg, 1, direction); sg = sg_next(sg); } return nents; } static int qce_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction) { int i; for (i = 0; ((i < nents) && (sg != NULL)); ++i) { dma_unmap_sg(dev, sg, 1, direction); sg = sg_next(sg); } return nents; } static int _probe_ce_engine(struct qce_device *pce_dev) { unsigned int rev; unsigned int maj_rev, min_rev, step_rev; rev = readl_relaxed(pce_dev->iobase + CRYPTO_VERSION_REG); mb(); maj_rev = (rev & CRYPTO_CORE_MAJOR_REV_MASK) >> CRYPTO_CORE_MAJOR_REV; min_rev = (rev & CRYPTO_CORE_MINOR_REV_MASK) >> CRYPTO_CORE_MINOR_REV; step_rev = (rev & CRYPTO_CORE_STEP_REV_MASK) >> CRYPTO_CORE_STEP_REV; if (maj_rev != CRYPTO_CORE_MAJOR_VER_NUM) { pr_err("Unsupported Qualcomm crypto device at 0x%x, rev %d.%d.%d\n", pce_dev->phy_iobase, maj_rev, min_rev, step_rev); return -EIO; } else { /* * The majority of crypto HW bugs have been fixed in 5.3.0 and * above. That allows a single sps transfer of consumer * pipe, and a single sps transfer of producer pipe * for a crypto request. no_get_around flag indicates this. * * In 5.3.1, the CCM MAC_FAILED in result dump issue is * fixed. no_ccm_mac_status_get_around flag indicates this. */ pce_dev->no_get_around = (min_rev >= CRYPTO_CORE_MINOR_VER_NUM) ? true : false; if (min_rev > CRYPTO_CORE_MINOR_VER_NUM) pce_dev->no_ccm_mac_status_get_around = true; else if ((min_rev == CRYPTO_CORE_MINOR_VER_NUM) && (step_rev >= CRYPTO_CORE_STEP_VER_NUM)) pce_dev->no_ccm_mac_status_get_around = true; else pce_dev->no_ccm_mac_status_get_around = false; } pce_dev->ce_bam_info.minor_version = min_rev; pce_dev->engines_avail = readl_relaxed(pce_dev->iobase + CRYPTO_ENGINES_AVAIL); dev_info(pce_dev->pdev, "Qualcomm Crypto %d.%d.%d device found @0x%x\n", maj_rev, min_rev, step_rev, pce_dev->phy_iobase); pce_dev->ce_bam_info.ce_burst_size = MAX_CE_BAM_BURST_SIZE; dev_info(pce_dev->pdev, "CE device = 0x%x\n" "IO base, CE = 0x%pK\n" "Consumer (IN) PIPE %d, " "Producer (OUT) PIPE %d\n" "IO base BAM = 0x%pK\n" "BAM IRQ %d\n" "Engines Availability = 0x%x\n", pce_dev->ce_bam_info.ce_device, pce_dev->iobase, pce_dev->ce_bam_info.dest_pipe_index, pce_dev->ce_bam_info.src_pipe_index, pce_dev->ce_bam_info.bam_iobase, pce_dev->ce_bam_info.bam_irq, pce_dev->engines_avail); return 0; }; static struct qce_cmdlist_info *_ce_get_hash_cmdlistinfo( struct qce_device *pce_dev, int req_info, struct qce_sha_req *sreq) { struct ce_sps_data *pce_sps_data; struct qce_cmdlistptr_ops *cmdlistptr; pce_sps_data = &pce_dev->ce_request_info[req_info].ce_sps; cmdlistptr = &pce_sps_data->cmdlistptr; switch (sreq->alg) { case QCE_HASH_SHA1: return &cmdlistptr->auth_sha1; case QCE_HASH_SHA256: return &cmdlistptr->auth_sha256; case QCE_HASH_SHA1_HMAC: return &cmdlistptr->auth_sha1_hmac; case QCE_HASH_SHA256_HMAC: return &cmdlistptr->auth_sha256_hmac; case QCE_HASH_AES_CMAC: if (sreq->authklen == AES128_KEY_SIZE) return &cmdlistptr->auth_aes_128_cmac; return &cmdlistptr->auth_aes_256_cmac; default: return NULL; } return NULL; } static int _ce_setup_hash(struct qce_device *pce_dev, struct qce_sha_req *sreq, struct qce_cmdlist_info *cmdlistinfo) { uint32_t auth32[SHA256_DIGEST_SIZE / sizeof(uint32_t)]; uint32_t diglen; int i; uint32_t mackey32[SHA_HMAC_KEY_SIZE/sizeof(uint32_t)] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; bool sha1 = false; struct sps_command_element *pce = NULL; bool use_hw_key = false; bool use_pipe_key = false; uint32_t authk_size_in_word = sreq->authklen/sizeof(uint32_t); uint32_t auth_cfg; if ((sreq->alg == QCE_HASH_SHA1_HMAC) || (sreq->alg == QCE_HASH_SHA256_HMAC) || (sreq->alg == QCE_HASH_AES_CMAC)) { /* no more check for null key. use flag */ if ((sreq->flags & QCRYPTO_CTX_USE_HW_KEY) == QCRYPTO_CTX_USE_HW_KEY) use_hw_key = true; else if ((sreq->flags & QCRYPTO_CTX_USE_PIPE_KEY) == QCRYPTO_CTX_USE_PIPE_KEY) use_pipe_key = true; pce = cmdlistinfo->go_proc; if (use_hw_key == true) { pce->addr = (uint32_t)(CRYPTO_GOPROC_QC_KEY_REG + pce_dev->phy_iobase); } else { pce->addr = (uint32_t)(CRYPTO_GOPROC_REG + pce_dev->phy_iobase); pce = cmdlistinfo->auth_key; if (use_pipe_key == false) { _byte_stream_to_net_words(mackey32, sreq->authkey, sreq->authklen); for (i = 0; i < authk_size_in_word; i++, pce++) pce->data = mackey32[i]; } } } if (sreq->alg == QCE_HASH_AES_CMAC) goto go_proc; /* if not the last, the size has to be on the block boundary */ if (sreq->last_blk == 0 && (sreq->size % SHA256_BLOCK_SIZE)) return -EIO; switch (sreq->alg) { case QCE_HASH_SHA1: case QCE_HASH_SHA1_HMAC: diglen = SHA1_DIGEST_SIZE; sha1 = true; break; case QCE_HASH_SHA256: case QCE_HASH_SHA256_HMAC: diglen = SHA256_DIGEST_SIZE; break; default: return -EINVAL; } /* write 20/32 bytes, 5/8 words into auth_iv for SHA1/SHA256 */ if (sreq->first_blk) { if (sha1) { for (i = 0; i < 5; i++) auth32[i] = _std_init_vector_sha1[i]; } else { for (i = 0; i < 8; i++) auth32[i] = _std_init_vector_sha256[i]; } } else { _byte_stream_to_net_words(auth32, sreq->digest, diglen); } pce = cmdlistinfo->auth_iv; for (i = 0; i < 5; i++, pce++) pce->data = auth32[i]; if ((sreq->alg == QCE_HASH_SHA256) || (sreq->alg == QCE_HASH_SHA256_HMAC)) { for (i = 5; i < 8; i++, pce++) pce->data = auth32[i]; } /* write auth_bytecnt 0/1, start with 0 */ pce = cmdlistinfo->auth_bytecount; for (i = 0; i < 2; i++, pce++) pce->data = sreq->auth_data[i]; /* Set/reset last bit in CFG register */ pce = cmdlistinfo->auth_seg_cfg; auth_cfg = pce->data & ~(1 << CRYPTO_LAST | 1 << CRYPTO_FIRST | 1 << CRYPTO_USE_PIPE_KEY_AUTH | 1 << CRYPTO_USE_HW_KEY_AUTH); if (sreq->last_blk) auth_cfg |= 1 << CRYPTO_LAST; if (sreq->first_blk) auth_cfg |= 1 << CRYPTO_FIRST; if (use_hw_key) auth_cfg |= 1 << CRYPTO_USE_HW_KEY_AUTH; if (use_pipe_key) auth_cfg |= 1 << CRYPTO_USE_PIPE_KEY_AUTH; pce->data = auth_cfg; go_proc: /* write auth seg size */ pce = cmdlistinfo->auth_seg_size; pce->data = sreq->size; pce = cmdlistinfo->encr_seg_cfg; pce->data = 0; /* write auth seg size start*/ pce = cmdlistinfo->auth_seg_start; pce->data = 0; /* write seg size */ pce = cmdlistinfo->seg_size; /* always ensure there is input data. ZLT does not work for bam-ndp */ if (sreq->size) pce->data = sreq->size; else pce->data = pce_dev->ce_bam_info.ce_burst_size; return 0; } static struct qce_cmdlist_info *_ce_get_aead_cmdlistinfo( struct qce_device *pce_dev, int req_info, struct qce_req *creq) { struct ce_sps_data *pce_sps_data; struct qce_cmdlistptr_ops *cmdlistptr; pce_sps_data = &pce_dev->ce_request_info[req_info].ce_sps; cmdlistptr = &pce_sps_data->cmdlistptr; switch (creq->alg) { case CIPHER_ALG_DES: switch (creq->mode) { case QCE_MODE_CBC: if (creq->auth_alg == QCE_HASH_SHA1_HMAC) return &cmdlistptr->aead_hmac_sha1_cbc_des; else if (creq->auth_alg == QCE_HASH_SHA256_HMAC) return &cmdlistptr->aead_hmac_sha256_cbc_des; else return NULL; break; default: return NULL; } break; case CIPHER_ALG_3DES: switch (creq->mode) { case QCE_MODE_CBC: if (creq->auth_alg == QCE_HASH_SHA1_HMAC) return &cmdlistptr->aead_hmac_sha1_cbc_3des; else if (creq->auth_alg == QCE_HASH_SHA256_HMAC) return &cmdlistptr->aead_hmac_sha256_cbc_3des; else return NULL; break; default: return NULL; } break; case CIPHER_ALG_AES: switch (creq->mode) { case QCE_MODE_CBC: if (creq->encklen == AES128_KEY_SIZE) { if (creq->auth_alg == QCE_HASH_SHA1_HMAC) return &cmdlistptr-> aead_hmac_sha1_cbc_aes_128; else if (creq->auth_alg == QCE_HASH_SHA256_HMAC) return &cmdlistptr-> aead_hmac_sha256_cbc_aes_128; else return NULL; } else if (creq->encklen == AES256_KEY_SIZE) { if (creq->auth_alg == QCE_HASH_SHA1_HMAC) return &cmdlistptr-> aead_hmac_sha1_cbc_aes_256; else if (creq->auth_alg == QCE_HASH_SHA256_HMAC) return &cmdlistptr-> aead_hmac_sha256_cbc_aes_256; else return NULL; } else return NULL; break; default: return NULL; } break; default: return NULL; } return NULL; } static int _ce_setup_aead(struct qce_device *pce_dev, struct qce_req *q_req, uint32_t totallen_in, uint32_t coffset, struct qce_cmdlist_info *cmdlistinfo) { int32_t authk_size_in_word = SHA_HMAC_KEY_SIZE/sizeof(uint32_t); int i; uint32_t mackey32[SHA_HMAC_KEY_SIZE/sizeof(uint32_t)] = {0}; struct sps_command_element *pce; uint32_t a_cfg; uint32_t enckey32[(MAX_CIPHER_KEY_SIZE*2)/sizeof(uint32_t)] = {0}; uint32_t enciv32[MAX_IV_LENGTH/sizeof(uint32_t)] = {0}; uint32_t enck_size_in_word = 0; uint32_t enciv_in_word; uint32_t key_size; uint32_t encr_cfg = 0; uint32_t ivsize = q_req->ivsize; key_size = q_req->encklen; enck_size_in_word = key_size/sizeof(uint32_t); switch (q_req->alg) { case CIPHER_ALG_DES: enciv_in_word = 2; break; case CIPHER_ALG_3DES: enciv_in_word = 2; break; case CIPHER_ALG_AES: if ((key_size != AES128_KEY_SIZE) && (key_size != AES256_KEY_SIZE)) return -EINVAL; enciv_in_word = 4; break; default: return -EINVAL; } /* only support cbc mode */ if (q_req->mode != QCE_MODE_CBC) return -EINVAL; _byte_stream_to_net_words(enciv32, q_req->iv, ivsize); pce = cmdlistinfo->encr_cntr_iv; for (i = 0; i < enciv_in_word; i++, pce++) pce->data = enciv32[i]; /* * write encr key * do not use hw key or pipe key */ _byte_stream_to_net_words(enckey32, q_req->enckey, key_size); pce = cmdlistinfo->encr_key; for (i = 0; i < enck_size_in_word; i++, pce++) pce->data = enckey32[i]; /* write encr seg cfg */ pce = cmdlistinfo->encr_seg_cfg; encr_cfg = pce->data; if (q_req->dir == QCE_ENCRYPT) encr_cfg |= (1 << CRYPTO_ENCODE); else encr_cfg &= ~(1 << CRYPTO_ENCODE); pce->data = encr_cfg; /* we only support sha1-hmac and sha256-hmac at this point */ _byte_stream_to_net_words(mackey32, q_req->authkey, q_req->authklen); pce = cmdlistinfo->auth_key; for (i = 0; i < authk_size_in_word; i++, pce++) pce->data = mackey32[i]; pce = cmdlistinfo->auth_iv; if (q_req->auth_alg == QCE_HASH_SHA1_HMAC) for (i = 0; i < 5; i++, pce++) pce->data = _std_init_vector_sha1[i]; else for (i = 0; i < 8; i++, pce++) pce->data = _std_init_vector_sha256[i]; /* write auth_bytecnt 0/1, start with 0 */ pce = cmdlistinfo->auth_bytecount; for (i = 0; i < 2; i++, pce++) pce->data = 0; pce = cmdlistinfo->auth_seg_cfg; a_cfg = pce->data; a_cfg &= ~(CRYPTO_AUTH_POS_MASK); if (q_req->dir == QCE_ENCRYPT) a_cfg |= (CRYPTO_AUTH_POS_AFTER << CRYPTO_AUTH_POS); else a_cfg |= (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS); pce->data = a_cfg; /* write auth seg size */ pce = cmdlistinfo->auth_seg_size; pce->data = totallen_in; /* write auth seg size start*/ pce = cmdlistinfo->auth_seg_start; pce->data = 0; /* write seg size */ pce = cmdlistinfo->seg_size; pce->data = totallen_in; /* write encr seg size */ pce = cmdlistinfo->encr_seg_size; pce->data = q_req->cryptlen; /* write encr seg start */ pce = cmdlistinfo->encr_seg_start; pce->data = (coffset & 0xffff); return 0; } static struct qce_cmdlist_info *_ce_get_cipher_cmdlistinfo( struct qce_device *pce_dev, int req_info, struct qce_req *creq) { struct ce_request_info *preq_info; struct ce_sps_data *pce_sps_data; struct qce_cmdlistptr_ops *cmdlistptr; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; cmdlistptr = &pce_sps_data->cmdlistptr; if (creq->alg != CIPHER_ALG_AES) { switch (creq->alg) { case CIPHER_ALG_DES: if (creq->mode == QCE_MODE_ECB) return &cmdlistptr->cipher_des_ecb; return &cmdlistptr->cipher_des_cbc; case CIPHER_ALG_3DES: if (creq->mode == QCE_MODE_ECB) return &cmdlistptr->cipher_3des_ecb; return &cmdlistptr->cipher_3des_cbc; default: return NULL; } } else { switch (creq->mode) { case QCE_MODE_ECB: if (creq->encklen == AES128_KEY_SIZE) return &cmdlistptr->cipher_aes_128_ecb; return &cmdlistptr->cipher_aes_256_ecb; case QCE_MODE_CBC: case QCE_MODE_CTR: if (creq->encklen == AES128_KEY_SIZE) return &cmdlistptr->cipher_aes_128_cbc_ctr; return &cmdlistptr->cipher_aes_256_cbc_ctr; case QCE_MODE_XTS: if (creq->encklen/2 == AES128_KEY_SIZE) return &cmdlistptr->cipher_aes_128_xts; return &cmdlistptr->cipher_aes_256_xts; case QCE_MODE_CCM: if (creq->encklen == AES128_KEY_SIZE) return &cmdlistptr->aead_aes_128_ccm; return &cmdlistptr->aead_aes_256_ccm; default: return NULL; } } return NULL; } static int _ce_setup_cipher(struct qce_device *pce_dev, struct qce_req *creq, uint32_t totallen_in, uint32_t coffset, struct qce_cmdlist_info *cmdlistinfo) { uint32_t enckey32[(MAX_CIPHER_KEY_SIZE * 2)/sizeof(uint32_t)] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; uint32_t enciv32[MAX_IV_LENGTH / sizeof(uint32_t)] = { 0, 0, 0, 0}; uint32_t enck_size_in_word = 0; uint32_t key_size; bool use_hw_key = false; bool use_pipe_key = false; uint32_t encr_cfg = 0; uint32_t ivsize = creq->ivsize; int i; struct sps_command_element *pce = NULL; if (creq->mode == QCE_MODE_XTS) key_size = creq->encklen/2; else key_size = creq->encklen; pce = cmdlistinfo->go_proc; if ((creq->flags & QCRYPTO_CTX_USE_HW_KEY) == QCRYPTO_CTX_USE_HW_KEY) { use_hw_key = true; } else { if ((creq->flags & QCRYPTO_CTX_USE_PIPE_KEY) == QCRYPTO_CTX_USE_PIPE_KEY) use_pipe_key = true; } pce = cmdlistinfo->go_proc; if (use_hw_key == true) pce->addr = (uint32_t)(CRYPTO_GOPROC_QC_KEY_REG + pce_dev->phy_iobase); else pce->addr = (uint32_t)(CRYPTO_GOPROC_REG + pce_dev->phy_iobase); if ((use_pipe_key == false) && (use_hw_key == false)) { _byte_stream_to_net_words(enckey32, creq->enckey, key_size); enck_size_in_word = key_size/sizeof(uint32_t); } if ((creq->op == QCE_REQ_AEAD) && (creq->mode == QCE_MODE_CCM)) { uint32_t authklen32 = creq->encklen/sizeof(uint32_t); uint32_t noncelen32 = MAX_NONCE/sizeof(uint32_t); uint32_t nonce32[MAX_NONCE/sizeof(uint32_t)] = {0, 0, 0, 0}; uint32_t auth_cfg = 0; /* write nonce */ _byte_stream_to_net_words(nonce32, creq->nonce, MAX_NONCE); pce = cmdlistinfo->auth_nonce_info; for (i = 0; i < noncelen32; i++, pce++) pce->data = nonce32[i]; if (creq->authklen == AES128_KEY_SIZE) auth_cfg = pce_dev->reg.auth_cfg_aes_ccm_128; else { if (creq->authklen == AES256_KEY_SIZE) auth_cfg = pce_dev->reg.auth_cfg_aes_ccm_256; } if (creq->dir == QCE_ENCRYPT) auth_cfg |= (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS); else auth_cfg |= (CRYPTO_AUTH_POS_AFTER << CRYPTO_AUTH_POS); auth_cfg |= ((creq->authsize - 1) << CRYPTO_AUTH_SIZE); if (use_hw_key == true) { auth_cfg |= (1 << CRYPTO_USE_HW_KEY_AUTH); } else { auth_cfg &= ~(1 << CRYPTO_USE_HW_KEY_AUTH); /* write auth key */ pce = cmdlistinfo->auth_key; for (i = 0; i < authklen32; i++, pce++) pce->data = enckey32[i]; } pce = cmdlistinfo->auth_seg_cfg; pce->data = auth_cfg; pce = cmdlistinfo->auth_seg_size; if (creq->dir == QCE_ENCRYPT) pce->data = totallen_in; else pce->data = totallen_in - creq->authsize; pce = cmdlistinfo->auth_seg_start; pce->data = 0; } else { if (creq->op != QCE_REQ_AEAD) { pce = cmdlistinfo->auth_seg_cfg; pce->data = 0; } } switch (creq->mode) { case QCE_MODE_ECB: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_ecb_128; else encr_cfg = pce_dev->reg.encr_cfg_aes_ecb_256; break; case QCE_MODE_CBC: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_128; else encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_256; break; case QCE_MODE_XTS: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_xts_128; else encr_cfg = pce_dev->reg.encr_cfg_aes_xts_256; break; case QCE_MODE_CCM: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_ccm_128; else encr_cfg = pce_dev->reg.encr_cfg_aes_ccm_256; encr_cfg |= (CRYPTO_ENCR_MODE_CCM << CRYPTO_ENCR_MODE) | (CRYPTO_LAST_CCM_XFR << CRYPTO_LAST_CCM); break; case QCE_MODE_CTR: default: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_ctr_128; else encr_cfg = pce_dev->reg.encr_cfg_aes_ctr_256; break; } switch (creq->alg) { case CIPHER_ALG_DES: if (creq->mode != QCE_MODE_ECB) { if (ivsize > MAX_IV_LENGTH) { pr_err("%s: error: Invalid length parameter\n", __func__); return -EINVAL; } _byte_stream_to_net_words(enciv32, creq->iv, ivsize); pce = cmdlistinfo->encr_cntr_iv; pce->data = enciv32[0]; pce++; pce->data = enciv32[1]; } if (use_hw_key == false) { pce = cmdlistinfo->encr_key; pce->data = enckey32[0]; pce++; pce->data = enckey32[1]; } break; case CIPHER_ALG_3DES: if (creq->mode != QCE_MODE_ECB) { if (ivsize > MAX_IV_LENGTH) { pr_err("%s: error: Invalid length parameter\n", __func__); return -EINVAL; } _byte_stream_to_net_words(enciv32, creq->iv, ivsize); pce = cmdlistinfo->encr_cntr_iv; pce->data = enciv32[0]; pce++; pce->data = enciv32[1]; } if (use_hw_key == false) { /* write encr key */ pce = cmdlistinfo->encr_key; for (i = 0; i < 6; i++, pce++) pce->data = enckey32[i]; } break; case CIPHER_ALG_AES: default: if (creq->mode == QCE_MODE_XTS) { uint32_t xtskey32[MAX_CIPHER_KEY_SIZE/sizeof(uint32_t)] = {0, 0, 0, 0, 0, 0, 0, 0}; uint32_t xtsklen = creq->encklen/(2 * sizeof(uint32_t)); if ((use_hw_key == false) && (use_pipe_key == false)) { _byte_stream_to_net_words(xtskey32, (creq->enckey + creq->encklen/2), creq->encklen/2); /* write xts encr key */ pce = cmdlistinfo->encr_xts_key; for (i = 0; i < xtsklen; i++, pce++) pce->data = xtskey32[i]; } /* write xts du size */ pce = cmdlistinfo->encr_xts_du_size; switch (creq->flags & QCRYPTO_CTX_XTS_MASK) { case QCRYPTO_CTX_XTS_DU_SIZE_512B: pce->data = min((unsigned int)QCE_SECTOR_SIZE, creq->cryptlen); break; case QCRYPTO_CTX_XTS_DU_SIZE_1KB: pce->data = min((unsigned int)QCE_SECTOR_SIZE * 2, creq->cryptlen); break; default: pce->data = creq->cryptlen; break; } } if (creq->mode != QCE_MODE_ECB) { if (ivsize > MAX_IV_LENGTH) { pr_err("%s: error: Invalid length parameter\n", __func__); return -EINVAL; } if (creq->mode == QCE_MODE_XTS) _byte_stream_swap_to_net_words(enciv32, creq->iv, ivsize); else _byte_stream_to_net_words(enciv32, creq->iv, ivsize); /* write encr cntr iv */ pce = cmdlistinfo->encr_cntr_iv; for (i = 0; i < 4; i++, pce++) pce->data = enciv32[i]; if (creq->mode == QCE_MODE_CCM) { /* write cntr iv for ccm */ pce = cmdlistinfo->encr_ccm_cntr_iv; for (i = 0; i < 4; i++, pce++) pce->data = enciv32[i]; /* update cntr_iv[3] by one */ pce = cmdlistinfo->encr_cntr_iv; pce += 3; pce->data += 1; } } if (creq->op == QCE_REQ_ABLK_CIPHER_NO_KEY) { encr_cfg |= (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ); } else { if (use_hw_key == false) { /* write encr key */ pce = cmdlistinfo->encr_key; for (i = 0; i < enck_size_in_word; i++, pce++) pce->data = enckey32[i]; } } /* else of if (creq->op == QCE_REQ_ABLK_CIPHER_NO_KEY) */ break; } /* end of switch (creq->mode) */ if (use_pipe_key) encr_cfg |= (CRYPTO_USE_PIPE_KEY_ENCR_ENABLED << CRYPTO_USE_PIPE_KEY_ENCR); /* write encr seg cfg */ pce = cmdlistinfo->encr_seg_cfg; if ((creq->alg == CIPHER_ALG_DES) || (creq->alg == CIPHER_ALG_3DES)) { if (creq->dir == QCE_ENCRYPT) pce->data |= (1 << CRYPTO_ENCODE); else pce->data &= ~(1 << CRYPTO_ENCODE); encr_cfg = pce->data; } else { encr_cfg |= ((creq->dir == QCE_ENCRYPT) ? 1 : 0) << CRYPTO_ENCODE; } if (use_hw_key == true) encr_cfg |= (CRYPTO_USE_HW_KEY << CRYPTO_USE_HW_KEY_ENCR); else encr_cfg &= ~(CRYPTO_USE_HW_KEY << CRYPTO_USE_HW_KEY_ENCR); pce->data = encr_cfg; /* write encr seg size */ pce = cmdlistinfo->encr_seg_size; if ((creq->mode == QCE_MODE_CCM) && (creq->dir == QCE_DECRYPT)) pce->data = (creq->cryptlen + creq->authsize); else pce->data = creq->cryptlen; /* write encr seg start */ pce = cmdlistinfo->encr_seg_start; pce->data = (coffset & 0xffff); /* write seg size */ pce = cmdlistinfo->seg_size; pce->data = totallen_in; return 0; }; static int _ce_f9_setup(struct qce_device *pce_dev, struct qce_f9_req *req, struct qce_cmdlist_info *cmdlistinfo) { uint32_t ikey32[OTA_KEY_SIZE/sizeof(uint32_t)]; uint32_t key_size_in_word = OTA_KEY_SIZE/sizeof(uint32_t); uint32_t cfg; struct sps_command_element *pce; int i; switch (req->algorithm) { case QCE_OTA_ALGO_KASUMI: cfg = pce_dev->reg.auth_cfg_kasumi; break; case QCE_OTA_ALGO_SNOW3G: default: cfg = pce_dev->reg.auth_cfg_snow3g; break; }; /* write key in CRYPTO_AUTH_IV0-3_REG */ _byte_stream_to_net_words(ikey32, &req->ikey[0], OTA_KEY_SIZE); pce = cmdlistinfo->auth_iv; for (i = 0; i < key_size_in_word; i++, pce++) pce->data = ikey32[i]; /* write last bits in CRYPTO_AUTH_IV4_REG */ pce->data = req->last_bits; /* write fresh to CRYPTO_AUTH_BYTECNT0_REG */ pce = cmdlistinfo->auth_bytecount; pce->data = req->fresh; /* write count-i to CRYPTO_AUTH_BYTECNT1_REG */ pce++; pce->data = req->count_i; /* write auth seg cfg */ pce = cmdlistinfo->auth_seg_cfg; if (req->direction == QCE_OTA_DIR_DOWNLINK) cfg |= BIT(CRYPTO_F9_DIRECTION); pce->data = cfg; /* write auth seg size */ pce = cmdlistinfo->auth_seg_size; pce->data = req->msize; /* write auth seg start*/ pce = cmdlistinfo->auth_seg_start; pce->data = 0; /* write seg size */ pce = cmdlistinfo->seg_size; pce->data = req->msize; /* write go */ pce = cmdlistinfo->go_proc; pce->addr = (uint32_t)(CRYPTO_GOPROC_REG + pce_dev->phy_iobase); return 0; } static int _ce_f8_setup(struct qce_device *pce_dev, struct qce_f8_req *req, bool key_stream_mode, uint16_t npkts, uint16_t cipher_offset, uint16_t cipher_size, struct qce_cmdlist_info *cmdlistinfo) { uint32_t ckey32[OTA_KEY_SIZE/sizeof(uint32_t)]; uint32_t key_size_in_word = OTA_KEY_SIZE/sizeof(uint32_t); uint32_t cfg; struct sps_command_element *pce; int i; switch (req->algorithm) { case QCE_OTA_ALGO_KASUMI: cfg = pce_dev->reg.encr_cfg_kasumi; break; case QCE_OTA_ALGO_SNOW3G: default: cfg = pce_dev->reg.encr_cfg_snow3g; break; }; /* write key */ _byte_stream_to_net_words(ckey32, &req->ckey[0], OTA_KEY_SIZE); pce = cmdlistinfo->encr_key; for (i = 0; i < key_size_in_word; i++, pce++) pce->data = ckey32[i]; /* write encr seg cfg */ pce = cmdlistinfo->encr_seg_cfg; if (key_stream_mode) cfg |= BIT(CRYPTO_F8_KEYSTREAM_ENABLE); if (req->direction == QCE_OTA_DIR_DOWNLINK) cfg |= BIT(CRYPTO_F8_DIRECTION); pce->data = cfg; /* write encr seg start */ pce = cmdlistinfo->encr_seg_start; pce->data = (cipher_offset & 0xffff); /* write encr seg size */ pce = cmdlistinfo->encr_seg_size; pce->data = cipher_size; /* write seg size */ pce = cmdlistinfo->seg_size; pce->data = req->data_len; /* write cntr0_iv0 for countC */ pce = cmdlistinfo->encr_cntr_iv; pce->data = req->count_c; /* write cntr1_iv1 for nPkts, and bearer */ pce++; if (npkts == 1) npkts = 0; pce->data = req->bearer << CRYPTO_CNTR1_IV1_REG_F8_BEARER | npkts << CRYPTO_CNTR1_IV1_REG_F8_PKT_CNT; /* write go */ pce = cmdlistinfo->go_proc; pce->addr = (uint32_t)(CRYPTO_GOPROC_REG + pce_dev->phy_iobase); return 0; } static void _qce_dump_descr_fifos(struct qce_device *pce_dev, int req_info) { int i, j, ents; struct ce_sps_data *pce_sps_data; struct sps_iovec *iovec; uint32_t cmd_flags = SPS_IOVEC_FLAG_CMD; pce_sps_data = &pce_dev->ce_request_info[req_info].ce_sps; iovec = pce_sps_data->in_transfer.iovec; pr_info("==============================================\n"); pr_info("CONSUMER (TX/IN/DEST) PIPE DESCRIPTOR\n"); pr_info("==============================================\n"); for (i = 0; i < pce_sps_data->in_transfer.iovec_count; i++) { pr_info(" [%d] addr=0x%x size=0x%x flags=0x%x\n", i, iovec->addr, iovec->size, iovec->flags); if (iovec->flags & cmd_flags) { struct sps_command_element *pced; pced = (struct sps_command_element *) (GET_VIRT_ADDR(iovec->addr)); ents = iovec->size/(sizeof(struct sps_command_element)); for (j = 0; j < ents; j++) { pr_info(" [%d] [0x%x] 0x%x\n", j, pced->addr, pced->data); pced++; } } iovec++; } pr_info("==============================================\n"); pr_info("PRODUCER (RX/OUT/SRC) PIPE DESCRIPTOR\n"); pr_info("==============================================\n"); iovec = pce_sps_data->out_transfer.iovec; for (i = 0; i < pce_sps_data->out_transfer.iovec_count; i++) { pr_info(" [%d] addr=0x%x size=0x%x flags=0x%x\n", i, iovec->addr, iovec->size, iovec->flags); iovec++; } } #ifdef QCE_DEBUG static void _qce_dump_descr_fifos_dbg(struct qce_device *pce_dev, int req_info) { _qce_dump_descr_fifos(pce_dev, req_info); } #define QCE_WRITE_REG(val, addr) \ { \ pr_info(" [0x%pK] 0x%x\n", addr, (uint32_t)val); \ writel_relaxed(val, addr); \ } #else static void _qce_dump_descr_fifos_dbg(struct qce_device *pce_dev, int req_info) { } #define QCE_WRITE_REG(val, addr) \ writel_relaxed(val, addr) #endif static int _ce_setup_hash_direct(struct qce_device *pce_dev, struct qce_sha_req *sreq) { uint32_t auth32[SHA256_DIGEST_SIZE / sizeof(uint32_t)]; uint32_t diglen; bool use_hw_key = false; bool use_pipe_key = false; int i; uint32_t mackey32[SHA_HMAC_KEY_SIZE/sizeof(uint32_t)] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; uint32_t authk_size_in_word = sreq->authklen/sizeof(uint32_t); bool sha1 = false; uint32_t auth_cfg = 0; /* clear status */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_STATUS_REG); QCE_WRITE_REG(pce_dev->reg.crypto_cfg_be, (pce_dev->iobase + CRYPTO_CONFIG_REG)); /* * Ensure previous instructions (setting the CONFIG register) * was completed before issuing starting to set other config register * This is to ensure the configurations are done in correct endian-ness * as set in the CONFIG registers */ mb(); if (sreq->alg == QCE_HASH_AES_CMAC) { /* write seg_cfg */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); /* write seg_cfg */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG); /* write seg_cfg */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_ENCR_SEG_SIZE_REG); /* Clear auth_ivn, auth_keyn registers */ for (i = 0; i < 16; i++) { QCE_WRITE_REG(0, (pce_dev->iobase + (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)))); QCE_WRITE_REG(0, (pce_dev->iobase + (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)))); } /* write auth_bytecnt 0/1/2/3, start with 0 */ for (i = 0; i < 4; i++) QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_BYTECNT0_REG + i * sizeof(uint32_t)); if (sreq->authklen == AES128_KEY_SIZE) auth_cfg = pce_dev->reg.auth_cfg_cmac_128; else auth_cfg = pce_dev->reg.auth_cfg_cmac_256; } if ((sreq->alg == QCE_HASH_SHA1_HMAC) || (sreq->alg == QCE_HASH_SHA256_HMAC) || (sreq->alg == QCE_HASH_AES_CMAC)) { _byte_stream_to_net_words(mackey32, sreq->authkey, sreq->authklen); /* no more check for null key. use flag to check*/ if ((sreq->flags & QCRYPTO_CTX_USE_HW_KEY) == QCRYPTO_CTX_USE_HW_KEY) { use_hw_key = true; } else if ((sreq->flags & QCRYPTO_CTX_USE_PIPE_KEY) == QCRYPTO_CTX_USE_PIPE_KEY) { use_pipe_key = true; } else { /* setup key */ for (i = 0; i < authk_size_in_word; i++) QCE_WRITE_REG(mackey32[i], (pce_dev->iobase + (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)))); } } if (sreq->alg == QCE_HASH_AES_CMAC) goto go_proc; /* if not the last, the size has to be on the block boundary */ if (sreq->last_blk == 0 && (sreq->size % SHA256_BLOCK_SIZE)) return -EIO; switch (sreq->alg) { case QCE_HASH_SHA1: auth_cfg = pce_dev->reg.auth_cfg_sha1; diglen = SHA1_DIGEST_SIZE; sha1 = true; break; case QCE_HASH_SHA1_HMAC: auth_cfg = pce_dev->reg.auth_cfg_hmac_sha1; diglen = SHA1_DIGEST_SIZE; sha1 = true; break; case QCE_HASH_SHA256: auth_cfg = pce_dev->reg.auth_cfg_sha256; diglen = SHA256_DIGEST_SIZE; break; case QCE_HASH_SHA256_HMAC: auth_cfg = pce_dev->reg.auth_cfg_hmac_sha256; diglen = SHA256_DIGEST_SIZE; break; default: return -EINVAL; } /* write 20/32 bytes, 5/8 words into auth_iv for SHA1/SHA256 */ if (sreq->first_blk) { if (sha1) { for (i = 0; i < 5; i++) auth32[i] = _std_init_vector_sha1[i]; } else { for (i = 0; i < 8; i++) auth32[i] = _std_init_vector_sha256[i]; } } else { _byte_stream_to_net_words(auth32, sreq->digest, diglen); } /* Set auth_ivn, auth_keyn registers */ for (i = 0; i < 5; i++) QCE_WRITE_REG(auth32[i], (pce_dev->iobase + (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)))); if ((sreq->alg == QCE_HASH_SHA256) || (sreq->alg == QCE_HASH_SHA256_HMAC)) { for (i = 5; i < 8; i++) QCE_WRITE_REG(auth32[i], (pce_dev->iobase + (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)))); } /* write auth_bytecnt 0/1/2/3, start with 0 */ for (i = 0; i < 2; i++) QCE_WRITE_REG(sreq->auth_data[i], pce_dev->iobase + CRYPTO_AUTH_BYTECNT0_REG + i * sizeof(uint32_t)); /* Set/reset last bit in CFG register */ if (sreq->last_blk) auth_cfg |= 1 << CRYPTO_LAST; else auth_cfg &= ~(1 << CRYPTO_LAST); if (sreq->first_blk) auth_cfg |= 1 << CRYPTO_FIRST; else auth_cfg &= ~(1 << CRYPTO_FIRST); if (use_hw_key) auth_cfg |= 1 << CRYPTO_USE_HW_KEY_AUTH; if (use_pipe_key) auth_cfg |= 1 << CRYPTO_USE_PIPE_KEY_AUTH; go_proc: /* write seg_cfg */ QCE_WRITE_REG(auth_cfg, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); /* write auth seg_size */ QCE_WRITE_REG(sreq->size, pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG); /* write auth_seg_start */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_START_REG); /* reset encr seg_cfg */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG); /* write seg_size */ QCE_WRITE_REG(sreq->size, pce_dev->iobase + CRYPTO_SEG_SIZE_REG); QCE_WRITE_REG(pce_dev->reg.crypto_cfg_le, (pce_dev->iobase + CRYPTO_CONFIG_REG)); /* issue go to crypto */ if (use_hw_key == false) { QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), pce_dev->iobase + CRYPTO_GOPROC_REG); } else { QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP)), pce_dev->iobase + CRYPTO_GOPROC_QC_KEY_REG); } /* * Ensure previous instructions (setting the GO register) * was completed before issuing a DMA transfer request */ mb(); return 0; } static int _ce_setup_aead_direct(struct qce_device *pce_dev, struct qce_req *q_req, uint32_t totallen_in, uint32_t coffset) { int32_t authk_size_in_word = SHA_HMAC_KEY_SIZE/sizeof(uint32_t); int i; uint32_t mackey32[SHA_HMAC_KEY_SIZE/sizeof(uint32_t)] = {0}; uint32_t a_cfg; uint32_t enckey32[(MAX_CIPHER_KEY_SIZE*2)/sizeof(uint32_t)] = {0}; uint32_t enciv32[MAX_IV_LENGTH/sizeof(uint32_t)] = {0}; uint32_t enck_size_in_word = 0; uint32_t enciv_in_word; uint32_t key_size; uint32_t ivsize = q_req->ivsize; uint32_t encr_cfg; /* clear status */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_STATUS_REG); QCE_WRITE_REG(pce_dev->reg.crypto_cfg_be, (pce_dev->iobase + CRYPTO_CONFIG_REG)); /* * Ensure previous instructions (setting the CONFIG register) * was completed before issuing starting to set other config register * This is to ensure the configurations are done in correct endian-ness * as set in the CONFIG registers */ mb(); key_size = q_req->encklen; enck_size_in_word = key_size/sizeof(uint32_t); switch (q_req->alg) { case CIPHER_ALG_DES: switch (q_req->mode) { case QCE_MODE_CBC: encr_cfg = pce_dev->reg.encr_cfg_des_cbc; break; default: return -EINVAL; } enciv_in_word = 2; break; case CIPHER_ALG_3DES: switch (q_req->mode) { case QCE_MODE_CBC: encr_cfg = pce_dev->reg.encr_cfg_3des_cbc; break; default: return -EINVAL; } enciv_in_word = 2; break; case CIPHER_ALG_AES: switch (q_req->mode) { case QCE_MODE_CBC: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_128; else if (key_size == AES256_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_256; else return -EINVAL; break; default: return -EINVAL; } enciv_in_word = 4; break; default: return -EINVAL; } /* write CNTR0_IV0_REG */ if (q_req->mode != QCE_MODE_ECB) { _byte_stream_to_net_words(enciv32, q_req->iv, ivsize); for (i = 0; i < enciv_in_word; i++) QCE_WRITE_REG(enciv32[i], pce_dev->iobase + (CRYPTO_CNTR0_IV0_REG + i * sizeof(uint32_t))); } /* * write encr key * do not use hw key or pipe key */ _byte_stream_to_net_words(enckey32, q_req->enckey, key_size); for (i = 0; i < enck_size_in_word; i++) QCE_WRITE_REG(enckey32[i], pce_dev->iobase + (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t))); /* write encr seg cfg */ if (q_req->dir == QCE_ENCRYPT) encr_cfg |= (1 << CRYPTO_ENCODE); QCE_WRITE_REG(encr_cfg, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG); /* we only support sha1-hmac and sha256-hmac at this point */ _byte_stream_to_net_words(mackey32, q_req->authkey, q_req->authklen); for (i = 0; i < authk_size_in_word; i++) QCE_WRITE_REG(mackey32[i], pce_dev->iobase + (CRYPTO_AUTH_KEY0_REG + i * sizeof(uint32_t))); if (q_req->auth_alg == QCE_HASH_SHA1_HMAC) { for (i = 0; i < 5; i++) QCE_WRITE_REG(_std_init_vector_sha1[i], pce_dev->iobase + (CRYPTO_AUTH_IV0_REG + i * sizeof(uint32_t))); } else { for (i = 0; i < 8; i++) QCE_WRITE_REG(_std_init_vector_sha256[i], pce_dev->iobase + (CRYPTO_AUTH_IV0_REG + i * sizeof(uint32_t))); } /* write auth_bytecnt 0/1, start with 0 */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_BYTECNT0_REG); QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_BYTECNT1_REG); /* write encr seg size */ QCE_WRITE_REG(q_req->cryptlen, pce_dev->iobase + CRYPTO_ENCR_SEG_SIZE_REG); /* write encr start */ QCE_WRITE_REG(coffset & 0xffff, pce_dev->iobase + CRYPTO_ENCR_SEG_START_REG); if (q_req->auth_alg == QCE_HASH_SHA1_HMAC) a_cfg = pce_dev->reg.auth_cfg_aead_sha1_hmac; else a_cfg = pce_dev->reg.auth_cfg_aead_sha256_hmac; if (q_req->dir == QCE_ENCRYPT) a_cfg |= (CRYPTO_AUTH_POS_AFTER << CRYPTO_AUTH_POS); else a_cfg |= (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS); /* write auth seg_cfg */ QCE_WRITE_REG(a_cfg, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); /* write auth seg_size */ QCE_WRITE_REG(totallen_in, pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG); /* write auth_seg_start */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_START_REG); /* write seg_size */ QCE_WRITE_REG(totallen_in, pce_dev->iobase + CRYPTO_SEG_SIZE_REG); QCE_WRITE_REG(pce_dev->reg.crypto_cfg_le, (pce_dev->iobase + CRYPTO_CONFIG_REG)); /* issue go to crypto */ QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), pce_dev->iobase + CRYPTO_GOPROC_REG); /* * Ensure previous instructions (setting the GO register) * was completed before issuing a DMA transfer request */ mb(); return 0; }; static int _ce_setup_cipher_direct(struct qce_device *pce_dev, struct qce_req *creq, uint32_t totallen_in, uint32_t coffset) { uint32_t enckey32[(MAX_CIPHER_KEY_SIZE * 2)/sizeof(uint32_t)] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; uint32_t enciv32[MAX_IV_LENGTH / sizeof(uint32_t)] = { 0, 0, 0, 0}; uint32_t enck_size_in_word = 0; uint32_t key_size; bool use_hw_key = false; bool use_pipe_key = false; uint32_t encr_cfg = 0; uint32_t ivsize = creq->ivsize; int i; /* clear status */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_STATUS_REG); QCE_WRITE_REG(pce_dev->reg.crypto_cfg_be, (pce_dev->iobase + CRYPTO_CONFIG_REG)); /* * Ensure previous instructions (setting the CONFIG register) * was completed before issuing starting to set other config register * This is to ensure the configurations are done in correct endian-ness * as set in the CONFIG registers */ mb(); if (creq->mode == QCE_MODE_XTS) key_size = creq->encklen/2; else key_size = creq->encklen; if ((creq->flags & QCRYPTO_CTX_USE_HW_KEY) == QCRYPTO_CTX_USE_HW_KEY) { use_hw_key = true; } else { if ((creq->flags & QCRYPTO_CTX_USE_PIPE_KEY) == QCRYPTO_CTX_USE_PIPE_KEY) use_pipe_key = true; } if ((use_pipe_key == false) && (use_hw_key == false)) { _byte_stream_to_net_words(enckey32, creq->enckey, key_size); enck_size_in_word = key_size/sizeof(uint32_t); } if ((creq->op == QCE_REQ_AEAD) && (creq->mode == QCE_MODE_CCM)) { uint32_t authklen32 = creq->encklen/sizeof(uint32_t); uint32_t noncelen32 = MAX_NONCE/sizeof(uint32_t); uint32_t nonce32[MAX_NONCE/sizeof(uint32_t)] = {0, 0, 0, 0}; uint32_t auth_cfg = 0; /* Clear auth_ivn, auth_keyn registers */ for (i = 0; i < 16; i++) { QCE_WRITE_REG(0, (pce_dev->iobase + (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)))); QCE_WRITE_REG(0, (pce_dev->iobase + (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)))); } /* write auth_bytecnt 0/1/2/3, start with 0 */ for (i = 0; i < 4; i++) QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_BYTECNT0_REG + i * sizeof(uint32_t)); /* write nonce */ _byte_stream_to_net_words(nonce32, creq->nonce, MAX_NONCE); for (i = 0; i < noncelen32; i++) QCE_WRITE_REG(nonce32[i], pce_dev->iobase + CRYPTO_AUTH_INFO_NONCE0_REG + (i*sizeof(uint32_t))); if (creq->authklen == AES128_KEY_SIZE) auth_cfg = pce_dev->reg.auth_cfg_aes_ccm_128; else { if (creq->authklen == AES256_KEY_SIZE) auth_cfg = pce_dev->reg.auth_cfg_aes_ccm_256; } if (creq->dir == QCE_ENCRYPT) auth_cfg |= (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS); else auth_cfg |= (CRYPTO_AUTH_POS_AFTER << CRYPTO_AUTH_POS); auth_cfg |= ((creq->authsize - 1) << CRYPTO_AUTH_SIZE); if (use_hw_key == true) { auth_cfg |= (1 << CRYPTO_USE_HW_KEY_AUTH); } else { auth_cfg &= ~(1 << CRYPTO_USE_HW_KEY_AUTH); /* write auth key */ for (i = 0; i < authklen32; i++) QCE_WRITE_REG(enckey32[i], pce_dev->iobase + CRYPTO_AUTH_KEY0_REG + (i*sizeof(uint32_t))); } QCE_WRITE_REG(auth_cfg, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); if (creq->dir == QCE_ENCRYPT) { QCE_WRITE_REG(totallen_in, pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG); } else { QCE_WRITE_REG((totallen_in - creq->authsize), pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG); } QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_START_REG); } else { if (creq->op != QCE_REQ_AEAD) QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); } /* * Ensure previous instructions (write to all AUTH registers) * was completed before accessing a register that is not in * in the same 1K range. */ mb(); switch (creq->mode) { case QCE_MODE_ECB: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_ecb_128; else encr_cfg = pce_dev->reg.encr_cfg_aes_ecb_256; break; case QCE_MODE_CBC: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_128; else encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_256; break; case QCE_MODE_XTS: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_xts_128; else encr_cfg = pce_dev->reg.encr_cfg_aes_xts_256; break; case QCE_MODE_CCM: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_ccm_128; else encr_cfg = pce_dev->reg.encr_cfg_aes_ccm_256; break; case QCE_MODE_CTR: default: if (key_size == AES128_KEY_SIZE) encr_cfg = pce_dev->reg.encr_cfg_aes_ctr_128; else encr_cfg = pce_dev->reg.encr_cfg_aes_ctr_256; break; } switch (creq->alg) { case CIPHER_ALG_DES: if (creq->mode != QCE_MODE_ECB) { encr_cfg = pce_dev->reg.encr_cfg_des_cbc; _byte_stream_to_net_words(enciv32, creq->iv, ivsize); QCE_WRITE_REG(enciv32[0], pce_dev->iobase + CRYPTO_CNTR0_IV0_REG); QCE_WRITE_REG(enciv32[1], pce_dev->iobase + CRYPTO_CNTR1_IV1_REG); } else { encr_cfg = pce_dev->reg.encr_cfg_des_ecb; } if (use_hw_key == false) { QCE_WRITE_REG(enckey32[0], pce_dev->iobase + CRYPTO_ENCR_KEY0_REG); QCE_WRITE_REG(enckey32[1], pce_dev->iobase + CRYPTO_ENCR_KEY1_REG); } break; case CIPHER_ALG_3DES: if (creq->mode != QCE_MODE_ECB) { _byte_stream_to_net_words(enciv32, creq->iv, ivsize); QCE_WRITE_REG(enciv32[0], pce_dev->iobase + CRYPTO_CNTR0_IV0_REG); QCE_WRITE_REG(enciv32[1], pce_dev->iobase + CRYPTO_CNTR1_IV1_REG); encr_cfg = pce_dev->reg.encr_cfg_3des_cbc; } else { encr_cfg = pce_dev->reg.encr_cfg_3des_ecb; } if (use_hw_key == false) { /* write encr key */ for (i = 0; i < 6; i++) QCE_WRITE_REG(enckey32[0], (pce_dev->iobase + (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)))); } break; case CIPHER_ALG_AES: default: if (creq->mode == QCE_MODE_XTS) { uint32_t xtskey32[MAX_CIPHER_KEY_SIZE/sizeof(uint32_t)] = {0, 0, 0, 0, 0, 0, 0, 0}; uint32_t xtsklen = creq->encklen/(2 * sizeof(uint32_t)); if ((use_hw_key == false) && (use_pipe_key == false)) { _byte_stream_to_net_words(xtskey32, (creq->enckey + creq->encklen/2), creq->encklen/2); /* write xts encr key */ for (i = 0; i < xtsklen; i++) QCE_WRITE_REG(xtskey32[i], pce_dev->iobase + CRYPTO_ENCR_XTS_KEY0_REG + (i * sizeof(uint32_t))); } /* write xts du size */ switch (creq->flags & QCRYPTO_CTX_XTS_MASK) { case QCRYPTO_CTX_XTS_DU_SIZE_512B: QCE_WRITE_REG( min((uint32_t)QCE_SECTOR_SIZE, creq->cryptlen), pce_dev->iobase + CRYPTO_ENCR_XTS_DU_SIZE_REG); break; case QCRYPTO_CTX_XTS_DU_SIZE_1KB: QCE_WRITE_REG( min((uint32_t)(QCE_SECTOR_SIZE * 2), creq->cryptlen), pce_dev->iobase + CRYPTO_ENCR_XTS_DU_SIZE_REG); break; default: QCE_WRITE_REG(creq->cryptlen, pce_dev->iobase + CRYPTO_ENCR_XTS_DU_SIZE_REG); break; } } if (creq->mode != QCE_MODE_ECB) { if (creq->mode == QCE_MODE_XTS) _byte_stream_swap_to_net_words(enciv32, creq->iv, ivsize); else _byte_stream_to_net_words(enciv32, creq->iv, ivsize); /* write encr cntr iv */ for (i = 0; i <= 3; i++) QCE_WRITE_REG(enciv32[i], pce_dev->iobase + CRYPTO_CNTR0_IV0_REG + (i * sizeof(uint32_t))); if (creq->mode == QCE_MODE_CCM) { /* write cntr iv for ccm */ for (i = 0; i <= 3; i++) QCE_WRITE_REG(enciv32[i], pce_dev->iobase + CRYPTO_ENCR_CCM_INT_CNTR0_REG + (i * sizeof(uint32_t))); /* update cntr_iv[3] by one */ QCE_WRITE_REG((enciv32[3] + 1), pce_dev->iobase + CRYPTO_CNTR0_IV0_REG + (3 * sizeof(uint32_t))); } } if (creq->op == QCE_REQ_ABLK_CIPHER_NO_KEY) { encr_cfg |= (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ); } else { if ((use_hw_key == false) && (use_pipe_key == false)) { for (i = 0; i < enck_size_in_word; i++) QCE_WRITE_REG(enckey32[i], pce_dev->iobase + CRYPTO_ENCR_KEY0_REG + (i * sizeof(uint32_t))); } } /* else of if (creq->op == QCE_REQ_ABLK_CIPHER_NO_KEY) */ break; } /* end of switch (creq->mode) */ if (use_pipe_key) encr_cfg |= (CRYPTO_USE_PIPE_KEY_ENCR_ENABLED << CRYPTO_USE_PIPE_KEY_ENCR); /* write encr seg cfg */ encr_cfg |= ((creq->dir == QCE_ENCRYPT) ? 1 : 0) << CRYPTO_ENCODE; if (use_hw_key == true) encr_cfg |= (CRYPTO_USE_HW_KEY << CRYPTO_USE_HW_KEY_ENCR); else encr_cfg &= ~(CRYPTO_USE_HW_KEY << CRYPTO_USE_HW_KEY_ENCR); /* write encr seg cfg */ QCE_WRITE_REG(encr_cfg, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG); /* write encr seg size */ if ((creq->mode == QCE_MODE_CCM) && (creq->dir == QCE_DECRYPT)) { QCE_WRITE_REG((creq->cryptlen + creq->authsize), pce_dev->iobase + CRYPTO_ENCR_SEG_SIZE_REG); } else { QCE_WRITE_REG(creq->cryptlen, pce_dev->iobase + CRYPTO_ENCR_SEG_SIZE_REG); } /* write encr seg start */ QCE_WRITE_REG((coffset & 0xffff), pce_dev->iobase + CRYPTO_ENCR_SEG_START_REG); /* write encr counter mask */ QCE_WRITE_REG(0xffffffff, pce_dev->iobase + CRYPTO_CNTR_MASK_REG); QCE_WRITE_REG(0xffffffff, pce_dev->iobase + CRYPTO_CNTR_MASK_REG0); QCE_WRITE_REG(0xffffffff, pce_dev->iobase + CRYPTO_CNTR_MASK_REG1); QCE_WRITE_REG(0xffffffff, pce_dev->iobase + CRYPTO_CNTR_MASK_REG2); /* write seg size */ QCE_WRITE_REG(totallen_in, pce_dev->iobase + CRYPTO_SEG_SIZE_REG); QCE_WRITE_REG(pce_dev->reg.crypto_cfg_le, (pce_dev->iobase + CRYPTO_CONFIG_REG)); /* issue go to crypto */ if (use_hw_key == false) { QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), pce_dev->iobase + CRYPTO_GOPROC_REG); } else { QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP)), pce_dev->iobase + CRYPTO_GOPROC_QC_KEY_REG); } /* * Ensure previous instructions (setting the GO register) * was completed before issuing a DMA transfer request */ mb(); return 0; }; static int _ce_f9_setup_direct(struct qce_device *pce_dev, struct qce_f9_req *req) { uint32_t ikey32[OTA_KEY_SIZE/sizeof(uint32_t)]; uint32_t key_size_in_word = OTA_KEY_SIZE/sizeof(uint32_t); uint32_t auth_cfg; int i; switch (req->algorithm) { case QCE_OTA_ALGO_KASUMI: auth_cfg = pce_dev->reg.auth_cfg_kasumi; break; case QCE_OTA_ALGO_SNOW3G: default: auth_cfg = pce_dev->reg.auth_cfg_snow3g; break; }; /* clear status */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_STATUS_REG); /* set big endian configuration */ QCE_WRITE_REG(pce_dev->reg.crypto_cfg_be, (pce_dev->iobase + CRYPTO_CONFIG_REG)); /* * Ensure previous instructions (setting the CONFIG register) * was completed before issuing starting to set other config register * This is to ensure the configurations are done in correct endian-ness * as set in the CONFIG registers */ mb(); /* write enc_seg_cfg */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG); /* write ecn_seg_size */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_ENCR_SEG_SIZE_REG); /* write key in CRYPTO_AUTH_IV0-3_REG */ _byte_stream_to_net_words(ikey32, &req->ikey[0], OTA_KEY_SIZE); for (i = 0; i < key_size_in_word; i++) QCE_WRITE_REG(ikey32[i], (pce_dev->iobase + (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)))); /* write last bits in CRYPTO_AUTH_IV4_REG */ QCE_WRITE_REG(req->last_bits, (pce_dev->iobase + CRYPTO_AUTH_IV4_REG)); /* write fresh to CRYPTO_AUTH_BYTECNT0_REG */ QCE_WRITE_REG(req->fresh, (pce_dev->iobase + CRYPTO_AUTH_BYTECNT0_REG)); /* write count-i to CRYPTO_AUTH_BYTECNT1_REG */ QCE_WRITE_REG(req->count_i, (pce_dev->iobase + CRYPTO_AUTH_BYTECNT1_REG)); /* write auth seg cfg */ if (req->direction == QCE_OTA_DIR_DOWNLINK) auth_cfg |= BIT(CRYPTO_F9_DIRECTION); QCE_WRITE_REG(auth_cfg, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); /* write auth seg size */ QCE_WRITE_REG(req->msize, pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG); /* write auth seg start*/ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_START_REG); /* write seg size */ QCE_WRITE_REG(req->msize, pce_dev->iobase + CRYPTO_SEG_SIZE_REG); /* set little endian configuration before go*/ QCE_WRITE_REG(pce_dev->reg.crypto_cfg_le, (pce_dev->iobase + CRYPTO_CONFIG_REG)); /* write go */ QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), pce_dev->iobase + CRYPTO_GOPROC_REG); /* * Ensure previous instructions (setting the GO register) * was completed before issuing a DMA transfer request */ mb(); return 0; } static int _ce_f8_setup_direct(struct qce_device *pce_dev, struct qce_f8_req *req, bool key_stream_mode, uint16_t npkts, uint16_t cipher_offset, uint16_t cipher_size) { int i = 0; uint32_t encr_cfg = 0; uint32_t ckey32[OTA_KEY_SIZE/sizeof(uint32_t)]; uint32_t key_size_in_word = OTA_KEY_SIZE/sizeof(uint32_t); switch (req->algorithm) { case QCE_OTA_ALGO_KASUMI: encr_cfg = pce_dev->reg.encr_cfg_kasumi; break; case QCE_OTA_ALGO_SNOW3G: default: encr_cfg = pce_dev->reg.encr_cfg_snow3g; break; }; /* clear status */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_STATUS_REG); /* set big endian configuration */ QCE_WRITE_REG(pce_dev->reg.crypto_cfg_be, (pce_dev->iobase + CRYPTO_CONFIG_REG)); /* write auth seg configuration */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG); /* write auth seg size */ QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG); /* write key */ _byte_stream_to_net_words(ckey32, &req->ckey[0], OTA_KEY_SIZE); for (i = 0; i < key_size_in_word; i++) QCE_WRITE_REG(ckey32[i], (pce_dev->iobase + (CRYPTO_ENCR_KEY0_REG + i*sizeof(uint32_t)))); /* write encr seg cfg */ if (key_stream_mode) encr_cfg |= BIT(CRYPTO_F8_KEYSTREAM_ENABLE); if (req->direction == QCE_OTA_DIR_DOWNLINK) encr_cfg |= BIT(CRYPTO_F8_DIRECTION); QCE_WRITE_REG(encr_cfg, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG); /* write encr seg start */ QCE_WRITE_REG((cipher_offset & 0xffff), pce_dev->iobase + CRYPTO_ENCR_SEG_START_REG); /* write encr seg size */ QCE_WRITE_REG(cipher_size, pce_dev->iobase + CRYPTO_ENCR_SEG_SIZE_REG); /* write seg size */ QCE_WRITE_REG(req->data_len, pce_dev->iobase + CRYPTO_SEG_SIZE_REG); /* write cntr0_iv0 for countC */ QCE_WRITE_REG(req->count_c, pce_dev->iobase + CRYPTO_CNTR0_IV0_REG); /* write cntr1_iv1 for nPkts, and bearer */ if (npkts == 1) npkts = 0; QCE_WRITE_REG(req->bearer << CRYPTO_CNTR1_IV1_REG_F8_BEARER | npkts << CRYPTO_CNTR1_IV1_REG_F8_PKT_CNT, pce_dev->iobase + CRYPTO_CNTR1_IV1_REG); /* set little endian configuration before go*/ QCE_WRITE_REG(pce_dev->reg.crypto_cfg_le, (pce_dev->iobase + CRYPTO_CONFIG_REG)); /* write go */ QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), pce_dev->iobase + CRYPTO_GOPROC_REG); /* * Ensure previous instructions (setting the GO register) * was completed before issuing a DMA transfer request */ mb(); return 0; } static int _qce_unlock_other_pipes(struct qce_device *pce_dev, int req_info) { int rc = 0; struct ce_sps_data *pce_sps_data = &pce_dev->ce_request_info [req_info].ce_sps; if (pce_dev->no_get_around || pce_dev->support_cmd_dscr == false) return rc; rc = sps_transfer_one(pce_dev->ce_bam_info.consumer.pipe, GET_PHYS_ADDR(pce_sps_data-> cmdlistptr.unlock_all_pipes.cmdlist), 0, NULL, (SPS_IOVEC_FLAG_CMD | SPS_IOVEC_FLAG_UNLOCK)); if (rc) { pr_err("sps_xfr_one() fail rc=%d", rc); rc = -EINVAL; } return rc; } static inline void qce_free_req_info(struct qce_device *pce_dev, int req_info, bool is_complete); static int _aead_complete(struct qce_device *pce_dev, int req_info) { struct aead_request *areq; unsigned char mac[SHA256_DIGEST_SIZE]; uint32_t ccm_fail_status = 0; uint32_t result_dump_status; int32_t result_status = 0; struct ce_request_info *preq_info; struct ce_sps_data *pce_sps_data; qce_comp_func_ptr_t qce_callback; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; qce_callback = preq_info->qce_cb; areq = (struct aead_request *) preq_info->areq; if (areq->src != areq->dst) { qce_dma_unmap_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents, DMA_FROM_DEVICE); } qce_dma_unmap_sg(pce_dev->pdev, areq->src, preq_info->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); if (preq_info->asg) qce_dma_unmap_sg(pce_dev->pdev, preq_info->asg, preq_info->assoc_nents, DMA_TO_DEVICE); /* check MAC */ memcpy(mac, (char *)(&pce_sps_data->result->auth_iv[0]), SHA256_DIGEST_SIZE); /* read status before unlock */ if (preq_info->dir == QCE_DECRYPT) { if (pce_dev->no_get_around) if (pce_dev->no_ccm_mac_status_get_around) ccm_fail_status = be32_to_cpu(pce_sps_data-> result->status); else ccm_fail_status = be32_to_cpu(pce_sps_data-> result_null->status); else ccm_fail_status = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS_REG); } if (_qce_unlock_other_pipes(pce_dev, req_info)) { qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, mac, NULL, -ENXIO); return -ENXIO; } result_dump_status = be32_to_cpu(pce_sps_data->result->status); pce_sps_data->result->status = 0; if (result_dump_status & ((1 << CRYPTO_SW_ERR) | (1 << CRYPTO_AXI_ERR) | (1 << CRYPTO_HSD_ERR))) { pr_err("aead operation error. Status %x\n", result_dump_status); result_status = -ENXIO; } else if (pce_sps_data->consumer_status | pce_sps_data->producer_status) { pr_err("aead sps operation error. sps status %x %x\n", pce_sps_data->consumer_status, pce_sps_data->producer_status); result_status = -ENXIO; } if (preq_info->mode == QCE_MODE_CCM) { /* * Not from result dump, instead, use the status we just * read of device for MAC_FAILED. */ if (result_status == 0 && (preq_info->dir == QCE_DECRYPT) && (ccm_fail_status & (1 << CRYPTO_MAC_FAILED))) result_status = -EBADMSG; qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, mac, NULL, result_status); } else { uint32_t ivsize = 0; struct crypto_aead *aead; unsigned char iv[NUM_OF_CRYPTO_CNTR_IV_REG * CRYPTO_REG_SIZE]; aead = crypto_aead_reqtfm(areq); ivsize = crypto_aead_ivsize(aead); memcpy(iv, (char *)(pce_sps_data->result->encr_cntr_iv), sizeof(iv)); qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, mac, iv, result_status); } return 0; }; static int _sha_complete(struct qce_device *pce_dev, int req_info) { struct ahash_request *areq; unsigned char digest[SHA256_DIGEST_SIZE]; uint32_t bytecount32[2]; int32_t result_status = 0; uint32_t result_dump_status; struct ce_request_info *preq_info; struct ce_sps_data *pce_sps_data; qce_comp_func_ptr_t qce_callback; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; qce_callback = preq_info->qce_cb; areq = (struct ahash_request *) preq_info->areq; if (!areq) { pr_err("sha operation error. areq is NULL\n"); return -ENXIO; } qce_dma_unmap_sg(pce_dev->pdev, areq->src, preq_info->src_nents, DMA_TO_DEVICE); memcpy(digest, (char *)(&pce_sps_data->result->auth_iv[0]), SHA256_DIGEST_SIZE); _byte_stream_to_net_words(bytecount32, (unsigned char *)pce_sps_data->result->auth_byte_count, 2 * CRYPTO_REG_SIZE); if (_qce_unlock_other_pipes(pce_dev, req_info)) { qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, digest, (char *)bytecount32, -ENXIO); return -ENXIO; } result_dump_status = be32_to_cpu(pce_sps_data->result->status); pce_sps_data->result->status = 0; if (result_dump_status & ((1 << CRYPTO_SW_ERR) | (1 << CRYPTO_AXI_ERR) | (1 << CRYPTO_HSD_ERR))) { pr_err("sha operation error. Status %x\n", result_dump_status); result_status = -ENXIO; } else if (pce_sps_data->consumer_status) { pr_err("sha sps operation error. sps status %x\n", pce_sps_data->consumer_status); result_status = -ENXIO; } qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, digest, (char *)bytecount32, result_status); return 0; } static int _f9_complete(struct qce_device *pce_dev, int req_info) { uint32_t mac_i; int32_t result_status = 0; uint32_t result_dump_status; struct ce_request_info *preq_info; struct ce_sps_data *pce_sps_data; qce_comp_func_ptr_t qce_callback; void *areq; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; qce_callback = preq_info->qce_cb; areq = preq_info->areq; dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_src, preq_info->ota_size, DMA_TO_DEVICE); _byte_stream_to_net_words(&mac_i, (char *)(&pce_sps_data->result->auth_iv[0]), CRYPTO_REG_SIZE); if (_qce_unlock_other_pipes(pce_dev, req_info)) { qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, NULL, NULL, -ENXIO); return -ENXIO; } result_dump_status = be32_to_cpu(pce_sps_data->result->status); pce_sps_data->result->status = 0; if (result_dump_status & ((1 << CRYPTO_SW_ERR) | (1 << CRYPTO_AXI_ERR) | (1 << CRYPTO_HSD_ERR))) { pr_err("f9 operation error. Status %x\n", result_dump_status); result_status = -ENXIO; } else if (pce_sps_data->consumer_status | pce_sps_data->producer_status) { pr_err("f9 sps operation error. sps status %x %x\n", pce_sps_data->consumer_status, pce_sps_data->producer_status); result_status = -ENXIO; } qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, (char *)&mac_i, NULL, result_status); return 0; } static int _ablk_cipher_complete(struct qce_device *pce_dev, int req_info) { struct ablkcipher_request *areq; unsigned char iv[NUM_OF_CRYPTO_CNTR_IV_REG * CRYPTO_REG_SIZE]; int32_t result_status = 0; uint32_t result_dump_status; struct ce_request_info *preq_info; struct ce_sps_data *pce_sps_data; qce_comp_func_ptr_t qce_callback; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; qce_callback = preq_info->qce_cb; areq = (struct ablkcipher_request *) preq_info->areq; if (areq->src != areq->dst) { qce_dma_unmap_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents, DMA_FROM_DEVICE); } qce_dma_unmap_sg(pce_dev->pdev, areq->src, preq_info->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); if (_qce_unlock_other_pipes(pce_dev, req_info)) { qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, NULL, NULL, -ENXIO); return -ENXIO; } result_dump_status = be32_to_cpu(pce_sps_data->result->status); pce_sps_data->result->status = 0; if (result_dump_status & ((1 << CRYPTO_SW_ERR) | (1 << CRYPTO_AXI_ERR) | (1 << CRYPTO_HSD_ERR))) { pr_err("ablk_cipher operation error. Status %x\n", result_dump_status); result_status = -ENXIO; } else if (pce_sps_data->consumer_status | pce_sps_data->producer_status) { pr_err("ablk_cipher sps operation error. sps status %x %x\n", pce_sps_data->consumer_status, pce_sps_data->producer_status); result_status = -ENXIO; } if (preq_info->mode == QCE_MODE_ECB) { qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, NULL, NULL, pce_sps_data->consumer_status | result_status); } else { if (pce_dev->ce_bam_info.minor_version == 0) { if (preq_info->mode == QCE_MODE_CBC) { if (preq_info->dir == QCE_DECRYPT) memcpy(iv, (char *)preq_info->dec_iv, sizeof(iv)); else memcpy(iv, (unsigned char *) (sg_virt(areq->src) + areq->src->length - 16), sizeof(iv)); } if ((preq_info->mode == QCE_MODE_CTR) || (preq_info->mode == QCE_MODE_XTS)) { uint32_t num_blk = 0; uint32_t cntr_iv3 = 0; unsigned long long cntr_iv64 = 0; unsigned char *b = (unsigned char *)(&cntr_iv3); memcpy(iv, areq->info, sizeof(iv)); if (preq_info->mode != QCE_MODE_XTS) num_blk = areq->nbytes/16; else num_blk = 1; cntr_iv3 = ((*(iv + 12) << 24) & 0xff000000) | (((*(iv + 13)) << 16) & 0xff0000) | (((*(iv + 14)) << 8) & 0xff00) | (*(iv + 15) & 0xff); cntr_iv64 = (((unsigned long long)cntr_iv3 & (unsigned long long)0xFFFFFFFFULL) + (unsigned long long)num_blk) % (unsigned long long)(0x100000000ULL); cntr_iv3 = (u32)(cntr_iv64 & 0xFFFFFFFF); *(iv + 15) = (char)(*b); *(iv + 14) = (char)(*(b + 1)); *(iv + 13) = (char)(*(b + 2)); *(iv + 12) = (char)(*(b + 3)); } } else { memcpy(iv, (char *)(pce_sps_data->result->encr_cntr_iv), sizeof(iv)); } qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, NULL, iv, result_status); } return 0; } static int _f8_complete(struct qce_device *pce_dev, int req_info) { int32_t result_status = 0; uint32_t result_dump_status; uint32_t result_dump_status2; struct ce_request_info *preq_info; struct ce_sps_data *pce_sps_data; qce_comp_func_ptr_t qce_callback; void *areq; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; qce_callback = preq_info->qce_cb; areq = preq_info->areq; if (preq_info->phy_ota_dst) dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_dst, preq_info->ota_size, DMA_FROM_DEVICE); if (preq_info->phy_ota_src) dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_src, preq_info->ota_size, (preq_info->phy_ota_dst) ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL); if (_qce_unlock_other_pipes(pce_dev, req_info)) { qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, NULL, NULL, -ENXIO); return -ENXIO; } result_dump_status = be32_to_cpu(pce_sps_data->result->status); result_dump_status2 = be32_to_cpu(pce_sps_data->result->status2); if ((result_dump_status & ((1 << CRYPTO_SW_ERR) | (1 << CRYPTO_AXI_ERR) | (1 << CRYPTO_HSD_ERR)))) { pr_err( "f8 oper error. Dump Sta %x Sta2 %x req %d\n", result_dump_status, result_dump_status2, req_info); result_status = -ENXIO; } else if (pce_sps_data->consumer_status | pce_sps_data->producer_status) { pr_err("f8 sps operation error. sps status %x %x\n", pce_sps_data->consumer_status, pce_sps_data->producer_status); result_status = -ENXIO; } pce_sps_data->result->status = 0; pce_sps_data->result->status2 = 0; qce_free_req_info(pce_dev, req_info, true); qce_callback(areq, NULL, NULL, result_status); return 0; } static void _qce_sps_iovec_count_init(struct qce_device *pce_dev, int req_info) { struct ce_sps_data *pce_sps_data = &pce_dev->ce_request_info[req_info] .ce_sps; pce_sps_data->in_transfer.iovec_count = 0; pce_sps_data->out_transfer.iovec_count = 0; } static void _qce_set_flag(struct sps_transfer *sps_bam_pipe, uint32_t flag) { struct sps_iovec *iovec; if (sps_bam_pipe->iovec_count == 0) return; iovec = sps_bam_pipe->iovec + (sps_bam_pipe->iovec_count - 1); iovec->flags |= flag; } static int _qce_sps_add_data(dma_addr_t paddr, uint32_t len, struct sps_transfer *sps_bam_pipe) { struct sps_iovec *iovec = sps_bam_pipe->iovec + sps_bam_pipe->iovec_count; uint32_t data_cnt; while (len > 0) { if (sps_bam_pipe->iovec_count == QCE_MAX_NUM_DSCR) { pr_err("Num of descrptor %d exceed max (%d)", sps_bam_pipe->iovec_count, (uint32_t)QCE_MAX_NUM_DSCR); return -ENOMEM; } if (len > SPS_MAX_PKT_SIZE) data_cnt = SPS_MAX_PKT_SIZE; else data_cnt = len; iovec->size = data_cnt; iovec->addr = SPS_GET_LOWER_ADDR(paddr); iovec->flags = SPS_GET_UPPER_ADDR(paddr); sps_bam_pipe->iovec_count++; iovec++; paddr += data_cnt; len -= data_cnt; } return 0; } static int _qce_sps_add_sg_data(struct qce_device *pce_dev, struct scatterlist *sg_src, uint32_t nbytes, struct sps_transfer *sps_bam_pipe) { uint32_t data_cnt, len; dma_addr_t addr; struct sps_iovec *iovec = sps_bam_pipe->iovec + sps_bam_pipe->iovec_count; while ((nbytes > 0) && (sg_src != NULL)) { len = min(nbytes, sg_dma_len(sg_src)); nbytes -= len; addr = sg_dma_address(sg_src); if (pce_dev->ce_bam_info.minor_version == 0) len = ALIGN(len, pce_dev->ce_bam_info.ce_burst_size); while (len > 0) { if (sps_bam_pipe->iovec_count == QCE_MAX_NUM_DSCR) { pr_err("Num of descrptor %d exceed max (%d)", sps_bam_pipe->iovec_count, (uint32_t)QCE_MAX_NUM_DSCR); return -ENOMEM; } if (len > SPS_MAX_PKT_SIZE) { data_cnt = SPS_MAX_PKT_SIZE; iovec->size = data_cnt; iovec->addr = SPS_GET_LOWER_ADDR(addr); iovec->flags = SPS_GET_UPPER_ADDR(addr); } else { data_cnt = len; iovec->size = data_cnt; iovec->addr = SPS_GET_LOWER_ADDR(addr); iovec->flags = SPS_GET_UPPER_ADDR(addr); } iovec++; sps_bam_pipe->iovec_count++; addr += data_cnt; len -= data_cnt; } sg_src = sg_next(sg_src); } return 0; } static int _qce_sps_add_sg_data_off(struct qce_device *pce_dev, struct scatterlist *sg_src, uint32_t nbytes, uint32_t off, struct sps_transfer *sps_bam_pipe) { uint32_t data_cnt, len; dma_addr_t addr; struct sps_iovec *iovec = sps_bam_pipe->iovec + sps_bam_pipe->iovec_count; unsigned int res_within_sg; if (!sg_src) return -ENOENT; res_within_sg = sg_dma_len(sg_src); while (off > 0) { if (!sg_src) { pr_err("broken sg list off %d nbytes %d\n", off, nbytes); return -ENOENT; } len = sg_dma_len(sg_src); if (off < len) { res_within_sg = len - off; break; } off -= len; sg_src = sg_next(sg_src); if (sg_src) res_within_sg = sg_dma_len(sg_src); } while (nbytes > 0 && sg_src) { len = min(nbytes, res_within_sg); nbytes -= len; addr = sg_dma_address(sg_src) + off; if (pce_dev->ce_bam_info.minor_version == 0) len = ALIGN(len, pce_dev->ce_bam_info.ce_burst_size); while (len > 0) { if (sps_bam_pipe->iovec_count == QCE_MAX_NUM_DSCR) { pr_err("Num of descrptor %d exceed max (%d)", sps_bam_pipe->iovec_count, (uint32_t)QCE_MAX_NUM_DSCR); return -ENOMEM; } if (len > SPS_MAX_PKT_SIZE) { data_cnt = SPS_MAX_PKT_SIZE; iovec->size = data_cnt; iovec->addr = SPS_GET_LOWER_ADDR(addr); iovec->flags = SPS_GET_UPPER_ADDR(addr); } else { data_cnt = len; iovec->size = data_cnt; iovec->addr = SPS_GET_LOWER_ADDR(addr); iovec->flags = SPS_GET_UPPER_ADDR(addr); } iovec++; sps_bam_pipe->iovec_count++; addr += data_cnt; len -= data_cnt; } if (nbytes) { sg_src = sg_next(sg_src); if (!sg_src) { pr_err("more data bytes %d\n", nbytes); return -ENOMEM; } res_within_sg = sg_dma_len(sg_src); off = 0; } } return 0; } static int _qce_sps_add_cmd(struct qce_device *pce_dev, uint32_t flag, struct qce_cmdlist_info *cmdptr, struct sps_transfer *sps_bam_pipe) { dma_addr_t paddr = GET_PHYS_ADDR(cmdptr->cmdlist); struct sps_iovec *iovec = sps_bam_pipe->iovec + sps_bam_pipe->iovec_count; iovec->size = cmdptr->size; iovec->addr = SPS_GET_LOWER_ADDR(paddr); iovec->flags = SPS_GET_UPPER_ADDR(paddr) | SPS_IOVEC_FLAG_CMD | flag; sps_bam_pipe->iovec_count++; if (sps_bam_pipe->iovec_count >= QCE_MAX_NUM_DSCR) { pr_err("Num of descrptor %d exceed max (%d)", sps_bam_pipe->iovec_count, (uint32_t)QCE_MAX_NUM_DSCR); return -ENOMEM; } return 0; } static int _qce_sps_transfer(struct qce_device *pce_dev, int req_info) { int rc = 0; struct ce_sps_data *pce_sps_data; pce_sps_data = &pce_dev->ce_request_info[req_info].ce_sps; pce_sps_data->out_transfer.user = (void *)((uintptr_t)(CRYPTO_REQ_USER_PAT | (unsigned int) req_info)); pce_sps_data->in_transfer.user = (void *)((uintptr_t)(CRYPTO_REQ_USER_PAT | (unsigned int) req_info)); _qce_dump_descr_fifos_dbg(pce_dev, req_info); if (pce_sps_data->in_transfer.iovec_count) { rc = sps_transfer(pce_dev->ce_bam_info.consumer.pipe, &pce_sps_data->in_transfer); if (rc) { pr_err("sps_xfr() fail (consumer pipe=0x%lx) rc = %d\n", (uintptr_t)pce_dev->ce_bam_info.consumer.pipe, rc); goto ret; } } rc = sps_transfer(pce_dev->ce_bam_info.producer.pipe, &pce_sps_data->out_transfer); if (rc) pr_err("sps_xfr() fail (producer pipe=0x%lx) rc = %d\n", (uintptr_t)pce_dev->ce_bam_info.producer.pipe, rc); ret: if (rc) _qce_dump_descr_fifos(pce_dev, req_info); return rc; } /** * Allocate and Connect a CE peripheral's SPS endpoint * * This function allocates endpoint context and * connect it with memory endpoint by calling * appropriate SPS driver APIs. * * Also registers a SPS callback function with * SPS driver * * This function should only be called once typically * during driver probe. * * @pce_dev - Pointer to qce_device structure * @ep - Pointer to sps endpoint data structure * @is_produce - 1 means Producer endpoint * 0 means Consumer endpoint * * @return - 0 if successful else negative value. * */ static int qce_sps_init_ep_conn(struct qce_device *pce_dev, struct qce_sps_ep_conn_data *ep, bool is_producer) { int rc = 0; struct sps_pipe *sps_pipe_info; struct sps_connect *sps_connect_info = &ep->connect; struct sps_register_event *sps_event = &ep->event; /* Allocate endpoint context */ sps_pipe_info = sps_alloc_endpoint(); if (!sps_pipe_info) { pr_err("sps_alloc_endpoint() failed!!! is_producer=%d", is_producer); rc = -ENOMEM; goto out; } /* Now save the sps pipe handle */ ep->pipe = sps_pipe_info; /* Get default connection configuration for an endpoint */ rc = sps_get_config(sps_pipe_info, sps_connect_info); if (rc) { pr_err("sps_get_config() fail pipe_handle=0x%lx, rc = %d\n", (uintptr_t)sps_pipe_info, rc); goto get_config_err; } /* Modify the default connection configuration */ if (is_producer) { /* * For CE producer transfer, source should be * CE peripheral where as destination should * be system memory. */ sps_connect_info->source = pce_dev->ce_bam_info.bam_handle; sps_connect_info->destination = SPS_DEV_HANDLE_MEM; /* Producer pipe will handle this connection */ sps_connect_info->mode = SPS_MODE_SRC; sps_connect_info->options = SPS_O_AUTO_ENABLE | SPS_O_DESC_DONE; } else { /* For CE consumer transfer, source should be * system memory where as destination should * CE peripheral */ sps_connect_info->source = SPS_DEV_HANDLE_MEM; sps_connect_info->destination = pce_dev->ce_bam_info.bam_handle; sps_connect_info->mode = SPS_MODE_DEST; sps_connect_info->options = SPS_O_AUTO_ENABLE; } /* Producer pipe index */ sps_connect_info->src_pipe_index = pce_dev->ce_bam_info.src_pipe_index; /* Consumer pipe index */ sps_connect_info->dest_pipe_index = pce_dev->ce_bam_info.dest_pipe_index; /* Set pipe group */ sps_connect_info->lock_group = pce_dev->ce_bam_info.pipe_pair_index; sps_connect_info->event_thresh = 0x10; /* * Max. no of scatter/gather buffers that can * be passed by block layer = 32 (NR_SG). * Each BAM descritor needs 64 bits (8 bytes). * One BAM descriptor is required per buffer transfer. * So we would require total 256 (32 * 8) bytes of descriptor FIFO. * But due to HW limitation we need to allocate atleast one extra * descriptor memory (256 bytes + 8 bytes). But in order to be * in power of 2, we are allocating 512 bytes of memory. */ sps_connect_info->desc.size = QCE_MAX_NUM_DSCR * MAX_QCE_ALLOC_BAM_REQ * sizeof(struct sps_iovec); if (sps_connect_info->desc.size > MAX_SPS_DESC_FIFO_SIZE) sps_connect_info->desc.size = MAX_SPS_DESC_FIFO_SIZE; sps_connect_info->desc.base = dma_alloc_coherent(pce_dev->pdev, sps_connect_info->desc.size, &sps_connect_info->desc.phys_base, GFP_KERNEL); if (sps_connect_info->desc.base == NULL) { rc = -ENOMEM; pr_err("Can not allocate coherent memory for sps data\n"); goto get_config_err; } memset(sps_connect_info->desc.base, 0x00, sps_connect_info->desc.size); /* Establish connection between peripheral and memory endpoint */ rc = sps_connect(sps_pipe_info, sps_connect_info); if (rc) { pr_err("sps_connect() fail pipe_handle=0x%lx, rc = %d\n", (uintptr_t)sps_pipe_info, rc); goto sps_connect_err; } sps_event->mode = SPS_TRIGGER_CALLBACK; sps_event->xfer_done = NULL; sps_event->user = (void *)pce_dev; if (is_producer) { sps_event->options = SPS_O_EOT | SPS_O_DESC_DONE; sps_event->callback = _sps_producer_callback; rc = sps_register_event(ep->pipe, sps_event); if (rc) { pr_err("Producer callback registration failed rc=%d\n", rc); goto sps_connect_err; } } else { sps_event->options = SPS_O_EOT; sps_event->callback = NULL; } pr_debug("success, %s : pipe_handle=0x%lx, desc fifo base (phy) = 0x%pK\n", is_producer ? "PRODUCER(RX/OUT)" : "CONSUMER(TX/IN)", (uintptr_t)sps_pipe_info, &sps_connect_info->desc.phys_base); goto out; sps_connect_err: dma_free_coherent(pce_dev->pdev, sps_connect_info->desc.size, sps_connect_info->desc.base, sps_connect_info->desc.phys_base); get_config_err: sps_free_endpoint(sps_pipe_info); out: return rc; } /** * Disconnect and Deallocate a CE peripheral's SPS endpoint * * This function disconnect endpoint and deallocates * endpoint context. * * This function should only be called once typically * during driver remove. * * @pce_dev - Pointer to qce_device structure * @ep - Pointer to sps endpoint data structure * */ static void qce_sps_exit_ep_conn(struct qce_device *pce_dev, struct qce_sps_ep_conn_data *ep) { struct sps_pipe *sps_pipe_info = ep->pipe; struct sps_connect *sps_connect_info = &ep->connect; sps_disconnect(sps_pipe_info); dma_free_coherent(pce_dev->pdev, sps_connect_info->desc.size, sps_connect_info->desc.base, sps_connect_info->desc.phys_base); sps_free_endpoint(sps_pipe_info); } static void qce_sps_release_bam(struct qce_device *pce_dev) { struct bam_registration_info *pbam; mutex_lock(&bam_register_lock); pbam = pce_dev->pbam; if (pbam == NULL) goto ret; pbam->cnt--; if (pbam->cnt > 0) goto ret; if (pce_dev->ce_bam_info.bam_handle) { sps_deregister_bam_device(pce_dev->ce_bam_info.bam_handle); pr_debug("deregister bam handle 0x%lx\n", pce_dev->ce_bam_info.bam_handle); pce_dev->ce_bam_info.bam_handle = 0; } iounmap(pbam->bam_iobase); pr_debug("delete bam 0x%x\n", pbam->bam_mem); list_del(&pbam->qlist); kfree(pbam); ret: pce_dev->pbam = NULL; mutex_unlock(&bam_register_lock); } static int qce_sps_get_bam(struct qce_device *pce_dev) { int rc = 0; struct sps_bam_props bam = {0}; struct bam_registration_info *pbam = NULL; struct bam_registration_info *p; uint32_t bam_cfg = 0; mutex_lock(&bam_register_lock); list_for_each_entry(p, &qce50_bam_list, qlist) { if (p->bam_mem == pce_dev->bam_mem) { pbam = p; /* found */ break; } } if (pbam) { pr_debug("found bam 0x%x\n", pbam->bam_mem); pbam->cnt++; pce_dev->ce_bam_info.bam_handle = pbam->handle; pce_dev->ce_bam_info.bam_mem = pbam->bam_mem; pce_dev->ce_bam_info.bam_iobase = pbam->bam_iobase; pce_dev->pbam = pbam; pce_dev->support_cmd_dscr = pbam->support_cmd_dscr; goto ret; } pbam = kzalloc(sizeof(struct bam_registration_info), GFP_KERNEL); if (!pbam) { pr_err("qce50 Memory allocation of bam FAIL, error %ld\n", PTR_ERR(pbam)); rc = -ENOMEM; goto ret; } pbam->cnt = 1; pbam->bam_mem = pce_dev->bam_mem; pbam->bam_iobase = ioremap_nocache(pce_dev->bam_mem, pce_dev->bam_mem_size); if (!pbam->bam_iobase) { kfree(pbam); rc = -ENOMEM; pr_err("Can not map BAM io memory\n"); goto ret; } pce_dev->ce_bam_info.bam_mem = pbam->bam_mem; pce_dev->ce_bam_info.bam_iobase = pbam->bam_iobase; pbam->handle = 0; pr_debug("allocate bam 0x%x\n", pbam->bam_mem); bam_cfg = readl_relaxed(pce_dev->ce_bam_info.bam_iobase + CRYPTO_BAM_CNFG_BITS_REG); pbam->support_cmd_dscr = (bam_cfg & CRYPTO_BAM_CD_ENABLE_MASK) ? true : false; if (pbam->support_cmd_dscr == false) { pr_info("qce50 don't support command descriptor. bam_cfg%x\n", bam_cfg); pce_dev->no_get_around = false; } pce_dev->support_cmd_dscr = pbam->support_cmd_dscr; bam.phys_addr = pce_dev->ce_bam_info.bam_mem; bam.virt_addr = pce_dev->ce_bam_info.bam_iobase; /* * This event thresold value is only significant for BAM-to-BAM * transfer. It's ignored for BAM-to-System mode transfer. */ bam.event_threshold = 0x10; /* Pipe event threshold */ /* * This threshold controls when the BAM publish * the descriptor size on the sideband interface. * SPS HW will only be used when * data transfer size > 64 bytes. */ bam.summing_threshold = 64; /* SPS driver wll handle the crypto BAM IRQ */ bam.irq = (u32)pce_dev->ce_bam_info.bam_irq; /* * Set flag to indicate BAM global device control is managed * remotely. */ if ((pce_dev->support_cmd_dscr == false) || (pce_dev->is_shared)) bam.manage = SPS_BAM_MGR_DEVICE_REMOTE; else bam.manage = SPS_BAM_MGR_LOCAL; bam.ee = pce_dev->ce_bam_info.bam_ee; bam.ipc_loglevel = QCE_BAM_DEFAULT_IPC_LOGLVL; bam.options |= SPS_BAM_CACHED_WP; pr_debug("bam physical base=0x%lx\n", (uintptr_t)bam.phys_addr); pr_debug("bam virtual base=0x%pK\n", bam.virt_addr); /* Register CE Peripheral BAM device to SPS driver */ rc = sps_register_bam_device(&bam, &pbam->handle); if (rc) { pr_err("sps_register_bam_device() failed! err=%d", rc); rc = -EIO; iounmap(pbam->bam_iobase); kfree(pbam); goto ret; } pce_dev->pbam = pbam; list_add_tail(&pbam->qlist, &qce50_bam_list); pce_dev->ce_bam_info.bam_handle = pbam->handle; ret: mutex_unlock(&bam_register_lock); return rc; } /** * Initialize SPS HW connected with CE core * * This function register BAM HW resources with * SPS driver and then initialize 2 SPS endpoints * * This function should only be called once typically * during driver probe. * * @pce_dev - Pointer to qce_device structure * * @return - 0 if successful else negative value. * */ static int qce_sps_init(struct qce_device *pce_dev) { int rc = 0; rc = qce_sps_get_bam(pce_dev); if (rc) return rc; pr_debug("BAM device registered. bam_handle=0x%lx\n", pce_dev->ce_bam_info.bam_handle); rc = qce_sps_init_ep_conn(pce_dev, &pce_dev->ce_bam_info.producer, true); if (rc) goto sps_connect_producer_err; rc = qce_sps_init_ep_conn(pce_dev, &pce_dev->ce_bam_info.consumer, false); if (rc) goto sps_connect_consumer_err; pr_info(" Qualcomm MSM CE-BAM at 0x%016llx irq %d\n", (unsigned long long)pce_dev->ce_bam_info.bam_mem, (unsigned int)pce_dev->ce_bam_info.bam_irq); return rc; sps_connect_consumer_err: qce_sps_exit_ep_conn(pce_dev, &pce_dev->ce_bam_info.producer); sps_connect_producer_err: qce_sps_release_bam(pce_dev); return rc; } static inline int qce_alloc_req_info(struct qce_device *pce_dev) { int i; int request_index = pce_dev->ce_request_index; for (i = 0; i < MAX_QCE_BAM_REQ; i++) { request_index++; if (request_index >= MAX_QCE_BAM_REQ) request_index = 0; if (atomic_xchg(&pce_dev->ce_request_info[request_index]. in_use, true) == false) { pce_dev->ce_request_index = request_index; return request_index; } } pr_warn("pcedev %d no reqs available no_of_queued_req %d\n", pce_dev->dev_no, atomic_read( &pce_dev->no_of_queued_req)); return -EBUSY; } static inline void qce_free_req_info(struct qce_device *pce_dev, int req_info, bool is_complete) { pce_dev->ce_request_info[req_info].xfer_type = QCE_XFER_TYPE_LAST; if (atomic_xchg(&pce_dev->ce_request_info[req_info].in_use, false) == true) { if (req_info < MAX_QCE_BAM_REQ && is_complete) atomic_dec(&pce_dev->no_of_queued_req); } else pr_warn("request info %d free already\n", req_info); } static void print_notify_debug(struct sps_event_notify *notify) { phys_addr_t addr = DESC_FULL_ADDR((phys_addr_t) notify->data.transfer.iovec.flags, notify->data.transfer.iovec.addr); pr_debug("sps ev_id=%d, addr=0x%pa, size=0x%x, flags=0x%x user=0x%pK\n", notify->event_id, &addr, notify->data.transfer.iovec.size, notify->data.transfer.iovec.flags, notify->data.transfer.user); } static void _qce_req_complete(struct qce_device *pce_dev, unsigned int req_info) { struct ce_request_info *preq_info; preq_info = &pce_dev->ce_request_info[req_info]; switch (preq_info->xfer_type) { case QCE_XFER_CIPHERING: _ablk_cipher_complete(pce_dev, req_info); break; case QCE_XFER_HASHING: _sha_complete(pce_dev, req_info); break; case QCE_XFER_AEAD: _aead_complete(pce_dev, req_info); break; case QCE_XFER_F8: _f8_complete(pce_dev, req_info); break; case QCE_XFER_F9: _f9_complete(pce_dev, req_info); break; default: qce_free_req_info(pce_dev, req_info, true); break; } } static void qce_multireq_timeout(unsigned long data) { struct qce_device *pce_dev = (struct qce_device *)data; int ret = 0; int last_seq; unsigned long flags; last_seq = atomic_read(&pce_dev->bunch_cmd_seq); if (last_seq == 0 || last_seq != atomic_read(&pce_dev->last_intr_seq)) { atomic_set(&pce_dev->last_intr_seq, last_seq); mod_timer(&(pce_dev->timer), (jiffies + DELAY_IN_JIFFIES)); return; } /* last bunch mode command time out */ /* * From here to dummy request finish sps request and set owner back * to none, we disable interrupt. * So it won't get preempted or interrupted. If bam inerrupts happen * between, and completion callback gets called from BAM, a new * request may be issued by the client driver. Deadlock may happen. */ local_irq_save(flags); if (cmpxchg(&pce_dev->owner, QCE_OWNER_NONE, QCE_OWNER_TIMEOUT) != QCE_OWNER_NONE) { local_irq_restore(flags); mod_timer(&(pce_dev->timer), (jiffies + DELAY_IN_JIFFIES)); return; } ret = qce_dummy_req(pce_dev); if (ret) pr_warn("pcedev %d: Failed to insert dummy req\n", pce_dev->dev_no); cmpxchg(&pce_dev->owner, QCE_OWNER_TIMEOUT, QCE_OWNER_NONE); pce_dev->mode = IN_INTERRUPT_MODE; local_irq_restore(flags); del_timer(&(pce_dev->timer)); pce_dev->qce_stats.no_of_timeouts++; pr_debug("pcedev %d mode switch to INTR\n", pce_dev->dev_no); } void qce_get_driver_stats(void *handle) { struct qce_device *pce_dev = (struct qce_device *) handle; if (!_qce50_disp_stats) return; pr_info("Engine %d timeout occuured %d\n", pce_dev->dev_no, pce_dev->qce_stats.no_of_timeouts); pr_info("Engine %d dummy request inserted %d\n", pce_dev->dev_no, pce_dev->qce_stats.no_of_dummy_reqs); if (pce_dev->mode) pr_info("Engine %d is in BUNCH MODE\n", pce_dev->dev_no); else pr_info("Engine %d is in INTERRUPT MODE\n", pce_dev->dev_no); pr_info("Engine %d outstanding request %d\n", pce_dev->dev_no, atomic_read(&pce_dev->no_of_queued_req)); } EXPORT_SYMBOL(qce_get_driver_stats); void qce_clear_driver_stats(void *handle) { struct qce_device *pce_dev = (struct qce_device *) handle; pce_dev->qce_stats.no_of_timeouts = 0; pce_dev->qce_stats.no_of_dummy_reqs = 0; } EXPORT_SYMBOL(qce_clear_driver_stats); static void _sps_producer_callback(struct sps_event_notify *notify) { struct qce_device *pce_dev = (struct qce_device *) ((struct sps_event_notify *)notify)->user; int rc = 0; unsigned int req_info; struct ce_sps_data *pce_sps_data; struct ce_request_info *preq_info; print_notify_debug(notify); req_info = (unsigned int)((uintptr_t)notify->data.transfer.user); if ((req_info & 0xffff0000) != CRYPTO_REQ_USER_PAT) { pr_warn("request information %d out of range\n", req_info); return; } req_info = req_info & 0x00ff; if (req_info < 0 || req_info >= MAX_QCE_ALLOC_BAM_REQ) { pr_warn("request information %d out of range\n", req_info); return; } preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; if ((preq_info->xfer_type == QCE_XFER_CIPHERING || preq_info->xfer_type == QCE_XFER_AEAD) && pce_sps_data->producer_state == QCE_PIPE_STATE_IDLE) { pce_sps_data->producer_state = QCE_PIPE_STATE_COMP; pce_sps_data->out_transfer.iovec_count = 0; _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump), CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer); _qce_set_flag(&pce_sps_data->out_transfer, SPS_IOVEC_FLAG_INT); rc = sps_transfer(pce_dev->ce_bam_info.producer.pipe, &pce_sps_data->out_transfer); if (rc) { pr_err("sps_xfr() fail (producer pipe=0x%lx) rc = %d\n", (uintptr_t)pce_dev->ce_bam_info.producer.pipe, rc); } return; } _qce_req_complete(pce_dev, req_info); } /** * De-initialize SPS HW connected with CE core * * This function deinitialize SPS endpoints and then * deregisters BAM resources from SPS driver. * * This function should only be called once typically * during driver remove. * * @pce_dev - Pointer to qce_device structure * */ static void qce_sps_exit(struct qce_device *pce_dev) { qce_sps_exit_ep_conn(pce_dev, &pce_dev->ce_bam_info.consumer); qce_sps_exit_ep_conn(pce_dev, &pce_dev->ce_bam_info.producer); qce_sps_release_bam(pce_dev); } static void qce_add_cmd_element(struct qce_device *pdev, struct sps_command_element **cmd_ptr, u32 addr, u32 data, struct sps_command_element **populate) { (*cmd_ptr)->addr = (uint32_t)(addr + pdev->phy_iobase); (*cmd_ptr)->command = 0; (*cmd_ptr)->data = data; (*cmd_ptr)->mask = 0xFFFFFFFF; (*cmd_ptr)->reserved = 0; if (populate != NULL) *populate = *cmd_ptr; (*cmd_ptr)++; } static int _setup_cipher_aes_cmdlistptrs(struct qce_device *pdev, int cri_index, unsigned char **pvaddr, enum qce_cipher_mode_enum mode, bool key_128) { struct sps_command_element *ce_vaddr; uintptr_t ce_vaddr_start; struct qce_cmdlistptr_ops *cmdlistptr; struct qce_cmdlist_info *pcl_info = NULL; int i = 0; uint32_t encr_cfg = 0; uint32_t key_reg = 0; uint32_t xts_key_reg = 0; uint32_t iv_reg = 0; cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr; *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)), pdev->ce_bam_info.ce_burst_size); ce_vaddr = (struct sps_command_element *)(*pvaddr); ce_vaddr_start = (uintptr_t)(*pvaddr); /* * Designate chunks of the allocated memory to various * command list pointers related to AES cipher operations defined * in ce_cmdlistptrs_ops structure. */ switch (mode) { case QCE_MODE_CBC: case QCE_MODE_CTR: if (key_128 == true) { cmdlistptr->cipher_aes_128_cbc_ctr.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_aes_128_cbc_ctr); if (mode == QCE_MODE_CBC) encr_cfg = pdev->reg.encr_cfg_aes_cbc_128; else encr_cfg = pdev->reg.encr_cfg_aes_ctr_128; iv_reg = 4; key_reg = 4; xts_key_reg = 0; } else { cmdlistptr->cipher_aes_256_cbc_ctr.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_aes_256_cbc_ctr); if (mode == QCE_MODE_CBC) encr_cfg = pdev->reg.encr_cfg_aes_cbc_256; else encr_cfg = pdev->reg.encr_cfg_aes_ctr_256; iv_reg = 4; key_reg = 8; xts_key_reg = 0; } break; case QCE_MODE_ECB: if (key_128 == true) { cmdlistptr->cipher_aes_128_ecb.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_aes_128_ecb); encr_cfg = pdev->reg.encr_cfg_aes_ecb_128; iv_reg = 0; key_reg = 4; xts_key_reg = 0; } else { cmdlistptr->cipher_aes_256_ecb.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_aes_256_ecb); encr_cfg = pdev->reg.encr_cfg_aes_ecb_256; iv_reg = 0; key_reg = 8; xts_key_reg = 0; } break; case QCE_MODE_XTS: if (key_128 == true) { cmdlistptr->cipher_aes_128_xts.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_aes_128_xts); encr_cfg = pdev->reg.encr_cfg_aes_xts_128; iv_reg = 4; key_reg = 4; xts_key_reg = 4; } else { cmdlistptr->cipher_aes_256_xts.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_aes_256_xts); encr_cfg = pdev->reg.encr_cfg_aes_xts_256; iv_reg = 4; key_reg = 8; xts_key_reg = 8; } break; default: pr_err("Unknown mode of operation %d received, exiting now\n", mode); return -EINVAL; break; } /* clear status register */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0, &pcl_info->seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg, &pcl_info->encr_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0, &pcl_info->encr_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0, &pcl_info->encr_seg_start); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG, (uint32_t)0xffffffff, &pcl_info->encr_mask); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG0, (uint32_t)0xffffffff, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG1, (uint32_t)0xffffffff, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG2, (uint32_t)0xffffffff, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, 0, &pcl_info->auth_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0, &pcl_info->encr_key); for (i = 1; i < key_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)), 0, NULL); if (xts_key_reg) { qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_XTS_KEY0_REG, 0, &pcl_info->encr_xts_key); for (i = 1; i < xts_key_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_ENCR_XTS_KEY0_REG + i * sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_XTS_DU_SIZE_REG, 0, &pcl_info->encr_xts_du_size); } if (iv_reg) { qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0, &pcl_info->encr_cntr_iv); for (i = 1; i < iv_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_CNTR0_IV0_REG + i * sizeof(uint32_t)), 0, NULL); } /* Add dummy to align size to burst-size multiple */ if (mode == QCE_MODE_XTS) { qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0, &pcl_info->auth_seg_size); } else { qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0, &pcl_info->auth_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0, &pcl_info->auth_seg_size); } qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_le, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG, ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc); pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start; *pvaddr = (unsigned char *) ce_vaddr; return 0; } static int _setup_cipher_des_cmdlistptrs(struct qce_device *pdev, int cri_index, unsigned char **pvaddr, enum qce_cipher_alg_enum alg, bool mode_cbc) { struct sps_command_element *ce_vaddr; uintptr_t ce_vaddr_start; struct qce_cmdlistptr_ops *cmdlistptr; struct qce_cmdlist_info *pcl_info = NULL; int i = 0; uint32_t encr_cfg = 0; uint32_t key_reg = 0; uint32_t iv_reg = 0; cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr; *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)), pdev->ce_bam_info.ce_burst_size); ce_vaddr = (struct sps_command_element *)(*pvaddr); ce_vaddr_start = (uintptr_t)(*pvaddr); /* * Designate chunks of the allocated memory to various * command list pointers related to cipher operations defined * in ce_cmdlistptrs_ops structure. */ switch (alg) { case CIPHER_ALG_DES: if (mode_cbc) { cmdlistptr->cipher_des_cbc.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_des_cbc); encr_cfg = pdev->reg.encr_cfg_des_cbc; iv_reg = 2; key_reg = 2; } else { cmdlistptr->cipher_des_ecb.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_des_ecb); encr_cfg = pdev->reg.encr_cfg_des_ecb; iv_reg = 0; key_reg = 2; } break; case CIPHER_ALG_3DES: if (mode_cbc) { cmdlistptr->cipher_3des_cbc.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_3des_cbc); encr_cfg = pdev->reg.encr_cfg_3des_cbc; iv_reg = 2; key_reg = 6; } else { cmdlistptr->cipher_3des_ecb.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_3des_ecb); encr_cfg = pdev->reg.encr_cfg_3des_ecb; iv_reg = 0; key_reg = 6; } break; default: pr_err("Unknown algorithms %d received, exiting now\n", alg); return -EINVAL; break; } /* clear status register */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0, &pcl_info->seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg, &pcl_info->encr_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0, &pcl_info->encr_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0, &pcl_info->encr_seg_start); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, 0, &pcl_info->auth_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0, &pcl_info->encr_key); for (i = 1; i < key_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)), 0, NULL); if (iv_reg) { qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0, &pcl_info->encr_cntr_iv); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR1_IV1_REG, 0, NULL); } qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_le, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG, ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc); pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start; *pvaddr = (unsigned char *) ce_vaddr; return 0; } static int _setup_cipher_null_cmdlistptrs(struct qce_device *pdev, int cri_index, unsigned char **pvaddr) { struct sps_command_element *ce_vaddr; uintptr_t ce_vaddr_start; struct qce_cmdlistptr_ops *cmdlistptr = &pdev->ce_request_info [cri_index].ce_sps.cmdlistptr; struct qce_cmdlist_info *pcl_info = NULL; *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)), pdev->ce_bam_info.ce_burst_size); ce_vaddr_start = (uintptr_t)(*pvaddr); ce_vaddr = (struct sps_command_element *)(*pvaddr); cmdlistptr->cipher_null.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->cipher_null); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, pdev->ce_bam_info.ce_burst_size, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, pdev->reg.encr_cfg_aes_ecb_128, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG, ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc); pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start; *pvaddr = (unsigned char *) ce_vaddr; return 0; } static int _setup_auth_cmdlistptrs(struct qce_device *pdev, int cri_index, unsigned char **pvaddr, enum qce_hash_alg_enum alg, bool key_128) { struct sps_command_element *ce_vaddr; uintptr_t ce_vaddr_start; struct qce_cmdlistptr_ops *cmdlistptr; struct qce_cmdlist_info *pcl_info = NULL; int i = 0; uint32_t key_reg = 0; uint32_t auth_cfg = 0; uint32_t iv_reg = 0; cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr; *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)), pdev->ce_bam_info.ce_burst_size); ce_vaddr_start = (uintptr_t)(*pvaddr); ce_vaddr = (struct sps_command_element *)(*pvaddr); /* * Designate chunks of the allocated memory to various * command list pointers related to authentication operations * defined in ce_cmdlistptrs_ops structure. */ switch (alg) { case QCE_HASH_SHA1: cmdlistptr->auth_sha1.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->auth_sha1); auth_cfg = pdev->reg.auth_cfg_sha1; iv_reg = 5; /* clear status register */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); break; case QCE_HASH_SHA256: cmdlistptr->auth_sha256.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->auth_sha256); auth_cfg = pdev->reg.auth_cfg_sha256; iv_reg = 8; /* clear status register */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); /* 1 dummy write */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0, NULL); break; case QCE_HASH_SHA1_HMAC: cmdlistptr->auth_sha1_hmac.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->auth_sha1_hmac); auth_cfg = pdev->reg.auth_cfg_hmac_sha1; key_reg = 16; iv_reg = 5; /* clear status register */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); break; case QCE_HASH_SHA256_HMAC: cmdlistptr->auth_sha256_hmac.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->auth_sha256_hmac); auth_cfg = pdev->reg.auth_cfg_hmac_sha256; key_reg = 16; iv_reg = 8; /* clear status register */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); /* 1 dummy write */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0, NULL); break; case QCE_HASH_AES_CMAC: if (key_128 == true) { cmdlistptr->auth_aes_128_cmac.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->auth_aes_128_cmac); auth_cfg = pdev->reg.auth_cfg_cmac_128; key_reg = 4; } else { cmdlistptr->auth_aes_256_cmac.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->auth_aes_256_cmac); auth_cfg = pdev->reg.auth_cfg_cmac_256; key_reg = 8; } /* clear status register */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); /* 1 dummy write */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0, NULL); break; default: pr_err("Unknown algorithms %d received, exiting now\n", alg); return -EINVAL; break; } qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0, &pcl_info->seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, 0, &pcl_info->encr_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, auth_cfg, &pcl_info->auth_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0, &pcl_info->auth_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0, &pcl_info->auth_seg_start); if (alg == QCE_HASH_AES_CMAC) { /* reset auth iv, bytecount and key registers */ for (i = 0; i < 16; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_IV0_REG + i * sizeof(uint32_t)), 0, NULL); for (i = 0; i < 16; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG, 0, NULL); } else { qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_IV0_REG, 0, &pcl_info->auth_iv); for (i = 1; i < iv_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG, 0, &pcl_info->auth_bytecount); } qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT1_REG, 0, NULL); if (key_reg) { qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_KEY0_REG, 0, &pcl_info->auth_key); for (i = 1; i < key_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)), 0, NULL); } qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_le, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG, ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc); pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start; *pvaddr = (unsigned char *) ce_vaddr; return 0; } static int _setup_aead_cmdlistptrs(struct qce_device *pdev, int cri_index, unsigned char **pvaddr, uint32_t alg, uint32_t mode, uint32_t key_size, bool sha1) { struct sps_command_element *ce_vaddr; uintptr_t ce_vaddr_start; struct qce_cmdlistptr_ops *cmdlistptr; struct qce_cmdlist_info *pcl_info = NULL; uint32_t key_reg; uint32_t iv_reg; uint32_t i; uint32_t enciv_in_word; uint32_t encr_cfg; cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr; *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)), pdev->ce_bam_info.ce_burst_size); ce_vaddr_start = (uintptr_t)(*pvaddr); ce_vaddr = (struct sps_command_element *)(*pvaddr); switch (alg) { case CIPHER_ALG_DES: switch (mode) { case QCE_MODE_CBC: if (sha1) { cmdlistptr->aead_hmac_sha1_cbc_des.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr-> aead_hmac_sha1_cbc_des); } else { cmdlistptr->aead_hmac_sha256_cbc_des.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr-> aead_hmac_sha256_cbc_des); } encr_cfg = pdev->reg.encr_cfg_des_cbc; break; default: return -EINVAL; }; enciv_in_word = 2; break; case CIPHER_ALG_3DES: switch (mode) { case QCE_MODE_CBC: if (sha1) { cmdlistptr->aead_hmac_sha1_cbc_3des.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr-> aead_hmac_sha1_cbc_3des); } else { cmdlistptr->aead_hmac_sha256_cbc_3des.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr-> aead_hmac_sha256_cbc_3des); } encr_cfg = pdev->reg.encr_cfg_3des_cbc; break; default: return -EINVAL; }; enciv_in_word = 2; break; case CIPHER_ALG_AES: switch (mode) { case QCE_MODE_CBC: if (key_size == AES128_KEY_SIZE) { if (sha1) { cmdlistptr-> aead_hmac_sha1_cbc_aes_128. cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr-> aead_hmac_sha1_cbc_aes_128); } else { cmdlistptr-> aead_hmac_sha256_cbc_aes_128. cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr-> aead_hmac_sha256_cbc_aes_128); } encr_cfg = pdev->reg.encr_cfg_aes_cbc_128; } else if (key_size == AES256_KEY_SIZE) { if (sha1) { cmdlistptr-> aead_hmac_sha1_cbc_aes_256. cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr-> aead_hmac_sha1_cbc_aes_256); } else { cmdlistptr-> aead_hmac_sha256_cbc_aes_256. cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr-> aead_hmac_sha256_cbc_aes_256); } encr_cfg = pdev->reg.encr_cfg_aes_cbc_256; } else { return -EINVAL; } break; default: return -EINVAL; }; enciv_in_word = 4; break; default: return -EINVAL; }; qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); key_reg = key_size/sizeof(uint32_t); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0, &pcl_info->encr_key); for (i = 1; i < key_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)), 0, NULL); if (mode != QCE_MODE_ECB) { qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0, &pcl_info->encr_cntr_iv); for (i = 1; i < enciv_in_word; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_CNTR0_IV0_REG + i * sizeof(uint32_t)), 0, NULL); }; if (sha1) iv_reg = 5; else iv_reg = 8; qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_IV0_REG, 0, &pcl_info->auth_iv); for (i = 1; i < iv_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG, 0, &pcl_info->auth_bytecount); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT1_REG, 0, NULL); key_reg = SHA_HMAC_KEY_SIZE/sizeof(uint32_t); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_KEY0_REG, 0, &pcl_info->auth_key); for (i = 1; i < key_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0, &pcl_info->seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg, &pcl_info->encr_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0, &pcl_info->encr_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0, &pcl_info->encr_seg_start); if (sha1) qce_add_cmd_element( pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, pdev->reg.auth_cfg_aead_sha1_hmac, &pcl_info->auth_seg_cfg); else qce_add_cmd_element( pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, pdev->reg.auth_cfg_aead_sha256_hmac, &pcl_info->auth_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0, &pcl_info->auth_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0, &pcl_info->auth_seg_start); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_le, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG, ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc); pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start; *pvaddr = (unsigned char *) ce_vaddr; return 0; } static int _setup_aead_ccm_cmdlistptrs(struct qce_device *pdev, int cri_index, unsigned char **pvaddr, bool key_128) { struct sps_command_element *ce_vaddr; uintptr_t ce_vaddr_start; struct qce_cmdlistptr_ops *cmdlistptr = &pdev->ce_request_info [cri_index].ce_sps.cmdlistptr; struct qce_cmdlist_info *pcl_info = NULL; int i = 0; uint32_t encr_cfg = 0; uint32_t auth_cfg = 0; uint32_t key_reg = 0; *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)), pdev->ce_bam_info.ce_burst_size); ce_vaddr_start = (uintptr_t)(*pvaddr); ce_vaddr = (struct sps_command_element *)(*pvaddr); /* * Designate chunks of the allocated memory to various * command list pointers related to aead operations * defined in ce_cmdlistptrs_ops structure. */ if (key_128 == true) { cmdlistptr->aead_aes_128_ccm.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->aead_aes_128_ccm); auth_cfg = pdev->reg.auth_cfg_aes_ccm_128; encr_cfg = pdev->reg.encr_cfg_aes_ccm_128; key_reg = 4; } else { cmdlistptr->aead_aes_256_ccm.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->aead_aes_256_ccm); auth_cfg = pdev->reg.auth_cfg_aes_ccm_256; encr_cfg = pdev->reg.encr_cfg_aes_ccm_256; key_reg = 8; } /* clear status register */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0, &pcl_info->seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg, &pcl_info->encr_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0, &pcl_info->encr_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0, &pcl_info->encr_seg_start); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG, (uint32_t)0xffffffff, &pcl_info->encr_mask); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG0, (uint32_t)0xffffffff, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG1, (uint32_t)0xffffffff, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG2, (uint32_t)0xffffffff, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, auth_cfg, &pcl_info->auth_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0, &pcl_info->auth_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0, &pcl_info->auth_seg_start); /* reset auth iv, bytecount and key registers */ for (i = 0; i < 8; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_IV0_REG + i * sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT1_REG, 0, NULL); for (i = 0; i < 16; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_KEY0_REG + i * sizeof(uint32_t)), 0, NULL); /* set auth key */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_KEY0_REG, 0, &pcl_info->auth_key); for (i = 1; i < key_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_KEY0_REG + i * sizeof(uint32_t)), 0, NULL); /* set NONCE info */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_INFO_NONCE0_REG, 0, &pcl_info->auth_nonce_info); for (i = 1; i < 4; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_INFO_NONCE0_REG + i * sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0, &pcl_info->encr_key); for (i = 1; i < key_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0, &pcl_info->encr_cntr_iv); for (i = 1; i < 4; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_CNTR0_IV0_REG + i * sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_CCM_INT_CNTR0_REG, 0, &pcl_info->encr_ccm_cntr_iv); for (i = 1; i < 4; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_ENCR_CCM_INT_CNTR0_REG + i * sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_le, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG, ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc); pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start; *pvaddr = (unsigned char *) ce_vaddr; return 0; } static int _setup_f8_cmdlistptrs(struct qce_device *pdev, int cri_index, unsigned char **pvaddr, enum qce_ota_algo_enum alg) { struct sps_command_element *ce_vaddr; uintptr_t ce_vaddr_start; struct qce_cmdlistptr_ops *cmdlistptr; struct qce_cmdlist_info *pcl_info = NULL; int i = 0; uint32_t encr_cfg = 0; uint32_t key_reg = 4; cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr; *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)), pdev->ce_bam_info.ce_burst_size); ce_vaddr = (struct sps_command_element *)(*pvaddr); ce_vaddr_start = (uintptr_t)(*pvaddr); /* * Designate chunks of the allocated memory to various * command list pointers related to f8 cipher algorithm defined * in ce_cmdlistptrs_ops structure. */ switch (alg) { case QCE_OTA_ALGO_KASUMI: cmdlistptr->f8_kasumi.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->f8_kasumi); encr_cfg = pdev->reg.encr_cfg_kasumi; break; case QCE_OTA_ALGO_SNOW3G: default: cmdlistptr->f8_snow3g.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->f8_snow3g); encr_cfg = pdev->reg.encr_cfg_snow3g; break; } /* clear status register */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); /* set config to big endian */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0, &pcl_info->seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg, &pcl_info->encr_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0, &pcl_info->encr_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0, &pcl_info->encr_seg_start); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, 0, &pcl_info->auth_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0, &pcl_info->auth_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0, &pcl_info->auth_seg_start); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0, &pcl_info->encr_key); for (i = 1; i < key_reg; i++) qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)), 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0, &pcl_info->encr_cntr_iv); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR1_IV1_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_le, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG, ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc); pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start; *pvaddr = (unsigned char *) ce_vaddr; return 0; } static int _setup_f9_cmdlistptrs(struct qce_device *pdev, int cri_index, unsigned char **pvaddr, enum qce_ota_algo_enum alg) { struct sps_command_element *ce_vaddr; uintptr_t ce_vaddr_start; struct qce_cmdlistptr_ops *cmdlistptr; struct qce_cmdlist_info *pcl_info = NULL; int i = 0; uint32_t auth_cfg = 0; uint32_t iv_reg = 0; cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr; *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)), pdev->ce_bam_info.ce_burst_size); ce_vaddr_start = (uintptr_t)(*pvaddr); ce_vaddr = (struct sps_command_element *)(*pvaddr); /* * Designate chunks of the allocated memory to various * command list pointers related to authentication operations * defined in ce_cmdlistptrs_ops structure. */ switch (alg) { case QCE_OTA_ALGO_KASUMI: cmdlistptr->f9_kasumi.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->f9_kasumi); auth_cfg = pdev->reg.auth_cfg_kasumi; break; case QCE_OTA_ALGO_SNOW3G: default: cmdlistptr->f9_snow3g.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->f9_snow3g); auth_cfg = pdev->reg.auth_cfg_snow3g; }; /* clear status register */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL); /* set config to big endian */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg); iv_reg = 5; qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0, &pcl_info->seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, 0, &pcl_info->encr_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, auth_cfg, &pcl_info->auth_seg_cfg); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0, &pcl_info->auth_seg_size); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0, &pcl_info->auth_seg_start); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_IV0_REG, 0, &pcl_info->auth_iv); for (i = 1; i < iv_reg; i++) { qce_add_cmd_element(pdev, &ce_vaddr, (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)), 0, NULL); } qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG, 0, &pcl_info->auth_bytecount); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT1_REG, 0, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, pdev->reg.crypto_cfg_le, NULL); qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG, ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) | (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc); pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start; *pvaddr = (unsigned char *) ce_vaddr; return 0; } static int _setup_unlock_pipe_cmdlistptrs(struct qce_device *pdev, int cri_index, unsigned char **pvaddr) { struct sps_command_element *ce_vaddr; uintptr_t ce_vaddr_start = (uintptr_t)(*pvaddr); struct qce_cmdlistptr_ops *cmdlistptr; struct qce_cmdlist_info *pcl_info = NULL; cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr; *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)), pdev->ce_bam_info.ce_burst_size); ce_vaddr = (struct sps_command_element *)(*pvaddr); cmdlistptr->unlock_all_pipes.cmdlist = (uintptr_t)ce_vaddr; pcl_info = &(cmdlistptr->unlock_all_pipes); /* * Designate chunks of the allocated memory to command list * to unlock pipes. */ qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, CRYPTO_CONFIG_RESET, NULL); pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start; *pvaddr = (unsigned char *) ce_vaddr; return 0; } static int qce_setup_cmdlistptrs(struct qce_device *pdev, int cri_index, unsigned char **pvaddr) { struct sps_command_element *ce_vaddr = (struct sps_command_element *)(*pvaddr); /* * Designate chunks of the allocated memory to various * command list pointers related to operations defined * in ce_cmdlistptrs_ops structure. */ ce_vaddr = (struct sps_command_element *)ALIGN(((uintptr_t) ce_vaddr), pdev->ce_bam_info.ce_burst_size); *pvaddr = (unsigned char *) ce_vaddr; _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_CBC, true); _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_CTR, true); _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_ECB, true); _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_XTS, true); _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_CBC, false); _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_CTR, false); _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_ECB, false); _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_XTS, false); _setup_cipher_des_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_DES, true); _setup_cipher_des_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_DES, false); _setup_cipher_des_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_3DES, true); _setup_cipher_des_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_3DES, false); _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_SHA1, false); _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_SHA256, false); _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_SHA1_HMAC, false); _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_SHA256_HMAC, false); _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_AES_CMAC, true); _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_AES_CMAC, false); _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_DES, QCE_MODE_CBC, DES_KEY_SIZE, true); _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_3DES, QCE_MODE_CBC, DES3_EDE_KEY_SIZE, true); _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_AES, QCE_MODE_CBC, AES128_KEY_SIZE, true); _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_AES, QCE_MODE_CBC, AES256_KEY_SIZE, true); _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_DES, QCE_MODE_CBC, DES_KEY_SIZE, false); _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_3DES, QCE_MODE_CBC, DES3_EDE_KEY_SIZE, false); _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_AES, QCE_MODE_CBC, AES128_KEY_SIZE, false); _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_AES, QCE_MODE_CBC, AES256_KEY_SIZE, false); _setup_cipher_null_cmdlistptrs(pdev, cri_index, pvaddr); _setup_aead_ccm_cmdlistptrs(pdev, cri_index, pvaddr, true); _setup_aead_ccm_cmdlistptrs(pdev, cri_index, pvaddr, false); _setup_f8_cmdlistptrs(pdev, cri_index, pvaddr, QCE_OTA_ALGO_KASUMI); _setup_f8_cmdlistptrs(pdev, cri_index, pvaddr, QCE_OTA_ALGO_SNOW3G); _setup_f9_cmdlistptrs(pdev, cri_index, pvaddr, QCE_OTA_ALGO_KASUMI); _setup_f9_cmdlistptrs(pdev, cri_index, pvaddr, QCE_OTA_ALGO_SNOW3G); _setup_unlock_pipe_cmdlistptrs(pdev, cri_index, pvaddr); return 0; } static int qce_setup_ce_sps_data(struct qce_device *pce_dev) { unsigned char *vaddr; int i; unsigned char *iovec_vaddr; int iovec_memsize; vaddr = pce_dev->coh_vmem; vaddr = (unsigned char *)ALIGN(((uintptr_t)vaddr), pce_dev->ce_bam_info.ce_burst_size); iovec_vaddr = pce_dev->iovec_vmem; iovec_memsize = pce_dev->iovec_memsize; for (i = 0; i < MAX_QCE_ALLOC_BAM_REQ; i++) { /* Allow for 256 descriptor (cmd and data) entries per pipe */ pce_dev->ce_request_info[i].ce_sps.in_transfer.iovec = (struct sps_iovec *)iovec_vaddr; pce_dev->ce_request_info[i].ce_sps.in_transfer.iovec_phys = virt_to_phys(pce_dev->ce_request_info[i]. ce_sps.in_transfer.iovec); iovec_vaddr += TOTAL_IOVEC_SPACE_PER_PIPE; iovec_memsize -= TOTAL_IOVEC_SPACE_PER_PIPE; pce_dev->ce_request_info[i].ce_sps.out_transfer.iovec = (struct sps_iovec *)iovec_vaddr; pce_dev->ce_request_info[i].ce_sps.out_transfer.iovec_phys = virt_to_phys(pce_dev->ce_request_info[i]. ce_sps.out_transfer.iovec); iovec_vaddr += TOTAL_IOVEC_SPACE_PER_PIPE; iovec_memsize -= TOTAL_IOVEC_SPACE_PER_PIPE; if (pce_dev->support_cmd_dscr) qce_setup_cmdlistptrs(pce_dev, i, &vaddr); vaddr = (unsigned char *)ALIGN(((uintptr_t)vaddr), pce_dev->ce_bam_info.ce_burst_size); pce_dev->ce_request_info[i].ce_sps.result_dump = (uintptr_t)vaddr; pce_dev->ce_request_info[i].ce_sps.result_dump_phy = GET_PHYS_ADDR((uintptr_t)vaddr); pce_dev->ce_request_info[i].ce_sps.result = (struct ce_result_dump_format *)vaddr; vaddr += CRYPTO_RESULT_DUMP_SIZE; pce_dev->ce_request_info[i].ce_sps.result_dump_null = (uintptr_t)vaddr; pce_dev->ce_request_info[i].ce_sps.result_dump_null_phy = GET_PHYS_ADDR((uintptr_t)vaddr); pce_dev->ce_request_info[i].ce_sps.result_null = (struct ce_result_dump_format *)vaddr; vaddr += CRYPTO_RESULT_DUMP_SIZE; pce_dev->ce_request_info[i].ce_sps.ignore_buffer = (uintptr_t)vaddr; vaddr += pce_dev->ce_bam_info.ce_burst_size * 2; } if ((vaddr - pce_dev->coh_vmem) > pce_dev->memsize || iovec_memsize < 0) panic("qce50: Not enough coherent memory. Allocate %x , need %lx\n", pce_dev->memsize, (uintptr_t)vaddr - (uintptr_t)pce_dev->coh_vmem); return 0; } static int qce_init_ce_cfg_val(struct qce_device *pce_dev) { uint32_t beats = (pce_dev->ce_bam_info.ce_burst_size >> 3) - 1; uint32_t pipe_pair = pce_dev->ce_bam_info.pipe_pair_index; pce_dev->reg.crypto_cfg_be = (beats << CRYPTO_REQ_SIZE) | BIT(CRYPTO_MASK_DOUT_INTR) | BIT(CRYPTO_MASK_DIN_INTR) | BIT(CRYPTO_MASK_OP_DONE_INTR) | (0 << CRYPTO_HIGH_SPD_EN_N) | (pipe_pair << CRYPTO_PIPE_SET_SELECT); pce_dev->reg.crypto_cfg_le = (pce_dev->reg.crypto_cfg_be | CRYPTO_LITTLE_ENDIAN_MASK); /* Initialize encr_cfg register for AES alg */ pce_dev->reg.encr_cfg_aes_cbc_128 = (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_CBC << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_aes_cbc_256 = (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_CBC << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_aes_ctr_128 = (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_CTR << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_aes_ctr_256 = (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_CTR << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_aes_xts_128 = (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_XTS << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_aes_xts_256 = (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_XTS << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_aes_ecb_128 = (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_ECB << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_aes_ecb_256 = (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_ECB << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_aes_ccm_128 = (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_CCM << CRYPTO_ENCR_MODE)| (CRYPTO_LAST_CCM_XFR << CRYPTO_LAST_CCM); pce_dev->reg.encr_cfg_aes_ccm_256 = (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_CCM << CRYPTO_ENCR_MODE) | (CRYPTO_LAST_CCM_XFR << CRYPTO_LAST_CCM); /* Initialize encr_cfg register for DES alg */ pce_dev->reg.encr_cfg_des_ecb = (CRYPTO_ENCR_KEY_SZ_DES << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_DES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_ECB << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_des_cbc = (CRYPTO_ENCR_KEY_SZ_DES << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_DES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_CBC << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_3des_ecb = (CRYPTO_ENCR_KEY_SZ_3DES << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_DES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_ECB << CRYPTO_ENCR_MODE); pce_dev->reg.encr_cfg_3des_cbc = (CRYPTO_ENCR_KEY_SZ_3DES << CRYPTO_ENCR_KEY_SZ) | (CRYPTO_ENCR_ALG_DES << CRYPTO_ENCR_ALG) | (CRYPTO_ENCR_MODE_CBC << CRYPTO_ENCR_MODE); /* Initialize encr_cfg register for kasumi/snow3g alg */ pce_dev->reg.encr_cfg_kasumi = (CRYPTO_ENCR_ALG_KASUMI << CRYPTO_ENCR_ALG); pce_dev->reg.encr_cfg_snow3g = (CRYPTO_ENCR_ALG_SNOW_3G << CRYPTO_ENCR_ALG); /* Initialize auth_cfg register for CMAC alg */ pce_dev->reg.auth_cfg_cmac_128 = (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) | (CRYPTO_AUTH_MODE_CMAC << CRYPTO_AUTH_MODE)| (CRYPTO_AUTH_SIZE_ENUM_16_BYTES << CRYPTO_AUTH_SIZE) | (CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) | (CRYPTO_AUTH_KEY_SZ_AES128 << CRYPTO_AUTH_KEY_SIZE); pce_dev->reg.auth_cfg_cmac_256 = (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) | (CRYPTO_AUTH_MODE_CMAC << CRYPTO_AUTH_MODE)| (CRYPTO_AUTH_SIZE_ENUM_16_BYTES << CRYPTO_AUTH_SIZE) | (CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) | (CRYPTO_AUTH_KEY_SZ_AES256 << CRYPTO_AUTH_KEY_SIZE); /* Initialize auth_cfg register for HMAC alg */ pce_dev->reg.auth_cfg_hmac_sha1 = (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)| (CRYPTO_AUTH_SIZE_SHA1 << CRYPTO_AUTH_SIZE) | (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) | (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS); pce_dev->reg.auth_cfg_hmac_sha256 = (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)| (CRYPTO_AUTH_SIZE_SHA256 << CRYPTO_AUTH_SIZE) | (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) | (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS); /* Initialize auth_cfg register for SHA1/256 alg */ pce_dev->reg.auth_cfg_sha1 = (CRYPTO_AUTH_MODE_HASH << CRYPTO_AUTH_MODE)| (CRYPTO_AUTH_SIZE_SHA1 << CRYPTO_AUTH_SIZE) | (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) | (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS); pce_dev->reg.auth_cfg_sha256 = (CRYPTO_AUTH_MODE_HASH << CRYPTO_AUTH_MODE)| (CRYPTO_AUTH_SIZE_SHA256 << CRYPTO_AUTH_SIZE) | (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) | (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS); /* Initialize auth_cfg register for AEAD alg */ pce_dev->reg.auth_cfg_aead_sha1_hmac = (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)| (CRYPTO_AUTH_SIZE_SHA1 << CRYPTO_AUTH_SIZE) | (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) | (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST); pce_dev->reg.auth_cfg_aead_sha256_hmac = (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)| (CRYPTO_AUTH_SIZE_SHA256 << CRYPTO_AUTH_SIZE) | (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) | (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST); pce_dev->reg.auth_cfg_aes_ccm_128 = (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) | (CRYPTO_AUTH_MODE_CCM << CRYPTO_AUTH_MODE)| (CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) | (CRYPTO_AUTH_KEY_SZ_AES128 << CRYPTO_AUTH_KEY_SIZE) | ((MAX_NONCE/sizeof(uint32_t)) << CRYPTO_AUTH_NONCE_NUM_WORDS); pce_dev->reg.auth_cfg_aes_ccm_128 &= ~(1 << CRYPTO_USE_HW_KEY_AUTH); pce_dev->reg.auth_cfg_aes_ccm_256 = (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) | (CRYPTO_AUTH_MODE_CCM << CRYPTO_AUTH_MODE)| (CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) | (CRYPTO_AUTH_KEY_SZ_AES256 << CRYPTO_AUTH_KEY_SIZE) | ((MAX_NONCE/sizeof(uint32_t)) << CRYPTO_AUTH_NONCE_NUM_WORDS); pce_dev->reg.auth_cfg_aes_ccm_256 &= ~(1 << CRYPTO_USE_HW_KEY_AUTH); /* Initialize auth_cfg register for kasumi/snow3g */ pce_dev->reg.auth_cfg_kasumi = (CRYPTO_AUTH_ALG_KASUMI << CRYPTO_AUTH_ALG) | BIT(CRYPTO_FIRST) | BIT(CRYPTO_LAST); pce_dev->reg.auth_cfg_snow3g = (CRYPTO_AUTH_ALG_SNOW3G << CRYPTO_AUTH_ALG) | BIT(CRYPTO_FIRST) | BIT(CRYPTO_LAST); return 0; } static void _qce_ccm_get_around_input(struct qce_device *pce_dev, struct ce_request_info *preq_info, enum qce_cipher_dir_enum dir) { struct qce_cmdlist_info *cmdlistinfo; struct ce_sps_data *pce_sps_data; pce_sps_data = &preq_info->ce_sps; if ((dir == QCE_DECRYPT) && pce_dev->no_get_around && !(pce_dev->no_ccm_mac_status_get_around)) { cmdlistinfo = &pce_sps_data->cmdlistptr.cipher_null; _qce_sps_add_cmd(pce_dev, 0, cmdlistinfo, &pce_sps_data->in_transfer); _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->ignore_buffer), pce_dev->ce_bam_info.ce_burst_size, &pce_sps_data->in_transfer); _qce_set_flag(&pce_sps_data->in_transfer, SPS_IOVEC_FLAG_EOT | SPS_IOVEC_FLAG_NWD); } } static void _qce_ccm_get_around_output(struct qce_device *pce_dev, struct ce_request_info *preq_info, enum qce_cipher_dir_enum dir) { struct ce_sps_data *pce_sps_data; pce_sps_data = &preq_info->ce_sps; if ((dir == QCE_DECRYPT) && pce_dev->no_get_around && !(pce_dev->no_ccm_mac_status_get_around)) { _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->ignore_buffer), pce_dev->ce_bam_info.ce_burst_size, &pce_sps_data->out_transfer); _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump_null), CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer); } } /* QCE_DUMMY_REQ */ static void qce_dummy_complete(void *cookie, unsigned char *digest, unsigned char *authdata, int ret) { if (!cookie) pr_err("invalid cookie\n"); } static int qce_dummy_req(struct qce_device *pce_dev) { int ret = 0; if (!(atomic_xchg(&pce_dev->ce_request_info[DUMMY_REQ_INDEX]. in_use, true) == false)) return -EBUSY; ret = qce_process_sha_req(pce_dev, NULL); pce_dev->qce_stats.no_of_dummy_reqs++; return ret; } static int select_mode(struct qce_device *pce_dev, struct ce_request_info *preq_info) { struct ce_sps_data *pce_sps_data = &preq_info->ce_sps; unsigned int no_of_queued_req; unsigned int cadence; if (!pce_dev->no_get_around) { _qce_set_flag(&pce_sps_data->out_transfer, SPS_IOVEC_FLAG_INT); return 0; } /* * claim ownership of device */ again: if (cmpxchg(&pce_dev->owner, QCE_OWNER_NONE, QCE_OWNER_CLIENT) != QCE_OWNER_NONE) { ndelay(40); goto again; } no_of_queued_req = atomic_inc_return(&pce_dev->no_of_queued_req); if (pce_dev->mode == IN_INTERRUPT_MODE) { if (no_of_queued_req >= MAX_BUNCH_MODE_REQ) { pce_dev->mode = IN_BUNCH_MODE; pr_debug("pcedev %d mode switch to BUNCH\n", pce_dev->dev_no); _qce_set_flag(&pce_sps_data->out_transfer, SPS_IOVEC_FLAG_INT); pce_dev->intr_cadence = 0; atomic_set(&pce_dev->bunch_cmd_seq, 1); atomic_set(&pce_dev->last_intr_seq, 1); mod_timer(&(pce_dev->timer), (jiffies + DELAY_IN_JIFFIES)); } else { _qce_set_flag(&pce_sps_data->out_transfer, SPS_IOVEC_FLAG_INT); } } else { pce_dev->intr_cadence++; cadence = (preq_info->req_len >> 7) + 1; if (cadence > SET_INTR_AT_REQ) cadence = SET_INTR_AT_REQ; if (pce_dev->intr_cadence < cadence || ((pce_dev->intr_cadence == cadence) && pce_dev->cadence_flag)) atomic_inc(&pce_dev->bunch_cmd_seq); else { _qce_set_flag(&pce_sps_data->out_transfer, SPS_IOVEC_FLAG_INT); pce_dev->intr_cadence = 0; atomic_set(&pce_dev->bunch_cmd_seq, 0); atomic_set(&pce_dev->last_intr_seq, 0); pce_dev->cadence_flag = !pce_dev->cadence_flag; } } return 0; } static int _qce_aead_ccm_req(void *handle, struct qce_req *q_req) { int rc = 0; struct qce_device *pce_dev = (struct qce_device *) handle; struct aead_request *areq = (struct aead_request *) q_req->areq; uint32_t authsize = q_req->authsize; uint32_t totallen_in, out_len; uint32_t hw_pad_out = 0; int ce_burst_size; struct qce_cmdlist_info *cmdlistinfo = NULL; int req_info = -1; struct ce_request_info *preq_info; struct ce_sps_data *pce_sps_data; req_info = qce_alloc_req_info(pce_dev); if (req_info < 0) return -EBUSY; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; ce_burst_size = pce_dev->ce_bam_info.ce_burst_size; totallen_in = areq->cryptlen + q_req->assoclen; if (q_req->dir == QCE_ENCRYPT) { q_req->cryptlen = areq->cryptlen; out_len = areq->cryptlen + authsize; hw_pad_out = ALIGN(authsize, ce_burst_size) - authsize; } else { q_req->cryptlen = areq->cryptlen - authsize; out_len = q_req->cryptlen; hw_pad_out = authsize; } /* * For crypto 5.0 that has burst size alignment requirement * for data descritpor, * the agent above(qcrypto) prepares the src scatter list with * memory starting with associated data, followed by * data stream to be ciphered. * The destination scatter list is pointing to the same * data area as source. */ if (pce_dev->ce_bam_info.minor_version == 0) preq_info->src_nents = count_sg(areq->src, totallen_in); else preq_info->src_nents = count_sg(areq->src, areq->cryptlen + areq->assoclen); if (q_req->assoclen) { preq_info->assoc_nents = count_sg(q_req->asg, q_req->assoclen); /* formatted associated data input */ qce_dma_map_sg(pce_dev->pdev, q_req->asg, preq_info->assoc_nents, DMA_TO_DEVICE); preq_info->asg = q_req->asg; } else { preq_info->assoc_nents = 0; preq_info->asg = NULL; } /* cipher input */ qce_dma_map_sg(pce_dev->pdev, areq->src, preq_info->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); /* cipher + mac output for encryption */ if (areq->src != areq->dst) { if (pce_dev->ce_bam_info.minor_version == 0) /* * The destination scatter list is pointing to the same * data area as src. * Note, the associated data will be pass-through * at the begining of destination area. */ preq_info->dst_nents = count_sg(areq->dst, out_len + areq->assoclen); else preq_info->dst_nents = count_sg(areq->dst, out_len + areq->assoclen); qce_dma_map_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents, DMA_FROM_DEVICE); } else { preq_info->dst_nents = preq_info->src_nents; } if (pce_dev->support_cmd_dscr) { cmdlistinfo = _ce_get_cipher_cmdlistinfo(pce_dev, req_info, q_req); if (cmdlistinfo == NULL) { pr_err("Unsupported cipher algorithm %d, mode %d\n", q_req->alg, q_req->mode); qce_free_req_info(pce_dev, req_info, false); return -EINVAL; } /* set up crypto device */ rc = _ce_setup_cipher(pce_dev, q_req, totallen_in, q_req->assoclen, cmdlistinfo); } else { /* set up crypto device */ rc = _ce_setup_cipher_direct(pce_dev, q_req, totallen_in, q_req->assoclen); } if (rc < 0) goto bad; preq_info->mode = q_req->mode; /* setup for callback, and issue command to bam */ preq_info->areq = q_req->areq; preq_info->qce_cb = q_req->qce_cb; preq_info->dir = q_req->dir; /* setup xfer type for producer callback handling */ preq_info->xfer_type = QCE_XFER_AEAD; preq_info->req_len = totallen_in; _qce_sps_iovec_count_init(pce_dev, req_info); if (pce_dev->support_cmd_dscr) _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK, cmdlistinfo, &pce_sps_data->in_transfer); if (pce_dev->ce_bam_info.minor_version == 0) { goto bad; } else { if (q_req->assoclen && (_qce_sps_add_sg_data( pce_dev, q_req->asg, q_req->assoclen, &pce_sps_data->in_transfer))) goto bad; if (_qce_sps_add_sg_data_off(pce_dev, areq->src, areq->cryptlen, areq->assoclen, &pce_sps_data->in_transfer)) goto bad; _qce_set_flag(&pce_sps_data->in_transfer, SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD); _qce_ccm_get_around_input(pce_dev, preq_info, q_req->dir); if (pce_dev->no_get_around) _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK, &pce_sps_data->cmdlistptr.unlock_all_pipes, &pce_sps_data->in_transfer); /* Pass through to ignore associated data*/ if (_qce_sps_add_data( GET_PHYS_ADDR(pce_sps_data->ignore_buffer), q_req->assoclen, &pce_sps_data->out_transfer)) goto bad; if (_qce_sps_add_sg_data_off(pce_dev, areq->dst, out_len, areq->assoclen, &pce_sps_data->out_transfer)) goto bad; /* Pass through to ignore hw_pad (padding of the MAC data) */ if (_qce_sps_add_data( GET_PHYS_ADDR(pce_sps_data->ignore_buffer), hw_pad_out, &pce_sps_data->out_transfer)) goto bad; if (pce_dev->no_get_around || totallen_in <= SPS_MAX_PKT_SIZE) { if (_qce_sps_add_data( GET_PHYS_ADDR(pce_sps_data->result_dump), CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer)) goto bad; pce_sps_data->producer_state = QCE_PIPE_STATE_COMP; } else { pce_sps_data->producer_state = QCE_PIPE_STATE_IDLE; } _qce_ccm_get_around_output(pce_dev, preq_info, q_req->dir); select_mode(pce_dev, preq_info); rc = _qce_sps_transfer(pce_dev, req_info); cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE); } if (rc) goto bad; return 0; bad: if (preq_info->assoc_nents) { qce_dma_unmap_sg(pce_dev->pdev, q_req->asg, preq_info->assoc_nents, DMA_TO_DEVICE); } if (preq_info->src_nents) { qce_dma_unmap_sg(pce_dev->pdev, areq->src, preq_info->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); } if (areq->src != areq->dst) { qce_dma_unmap_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents, DMA_FROM_DEVICE); } qce_free_req_info(pce_dev, req_info, false); return rc; } static int _qce_suspend(void *handle) { struct qce_device *pce_dev = (struct qce_device *)handle; struct sps_pipe *sps_pipe_info; if (handle == NULL) return -ENODEV; qce_enable_clk(pce_dev); sps_pipe_info = pce_dev->ce_bam_info.consumer.pipe; sps_disconnect(sps_pipe_info); sps_pipe_info = pce_dev->ce_bam_info.producer.pipe; sps_disconnect(sps_pipe_info); qce_disable_clk(pce_dev); return 0; } static int _qce_resume(void *handle) { struct qce_device *pce_dev = (struct qce_device *)handle; struct sps_pipe *sps_pipe_info; struct sps_connect *sps_connect_info; int rc; if (handle == NULL) return -ENODEV; qce_enable_clk(pce_dev); sps_pipe_info = pce_dev->ce_bam_info.consumer.pipe; sps_connect_info = &pce_dev->ce_bam_info.consumer.connect; memset(sps_connect_info->desc.base, 0x00, sps_connect_info->desc.size); rc = sps_connect(sps_pipe_info, sps_connect_info); if (rc) { pr_err("sps_connect() fail pipe_handle=0x%lx, rc = %d\n", (uintptr_t)sps_pipe_info, rc); return rc; } sps_pipe_info = pce_dev->ce_bam_info.producer.pipe; sps_connect_info = &pce_dev->ce_bam_info.producer.connect; memset(sps_connect_info->desc.base, 0x00, sps_connect_info->desc.size); rc = sps_connect(sps_pipe_info, sps_connect_info); if (rc) pr_err("sps_connect() fail pipe_handle=0x%lx, rc = %d\n", (uintptr_t)sps_pipe_info, rc); rc = sps_register_event(sps_pipe_info, &pce_dev->ce_bam_info.producer.event); if (rc) pr_err("Producer callback registration failed rc = %d\n", rc); qce_disable_clk(pce_dev); return rc; } struct qce_pm_table qce_pm_table = {_qce_suspend, _qce_resume}; EXPORT_SYMBOL(qce_pm_table); int qce_aead_req(void *handle, struct qce_req *q_req) { struct qce_device *pce_dev = (struct qce_device *)handle; struct aead_request *areq; uint32_t authsize; struct crypto_aead *aead; uint32_t ivsize; uint32_t totallen; int rc = 0; struct qce_cmdlist_info *cmdlistinfo = NULL; int req_info = -1; struct ce_sps_data *pce_sps_data; struct ce_request_info *preq_info; if (q_req->mode == QCE_MODE_CCM) return _qce_aead_ccm_req(handle, q_req); req_info = qce_alloc_req_info(pce_dev); if (req_info < 0) return -EBUSY; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; areq = (struct aead_request *) q_req->areq; aead = crypto_aead_reqtfm(areq); ivsize = crypto_aead_ivsize(aead); q_req->ivsize = ivsize; authsize = q_req->authsize; if (q_req->dir == QCE_ENCRYPT) q_req->cryptlen = areq->cryptlen; else q_req->cryptlen = areq->cryptlen - authsize; if (q_req->cryptlen > UINT_MAX - areq->assoclen) { pr_err("Integer overflow on total aead req length.\n"); return -EINVAL; } totallen = q_req->cryptlen + areq->assoclen; if (pce_dev->support_cmd_dscr) { cmdlistinfo = _ce_get_aead_cmdlistinfo(pce_dev, req_info, q_req); if (cmdlistinfo == NULL) { pr_err("Unsupported aead ciphering algorithm %d, mode %d, ciphering key length %d, auth digest size %d\n", q_req->alg, q_req->mode, q_req->encklen, q_req->authsize); qce_free_req_info(pce_dev, req_info, false); return -EINVAL; } /* set up crypto device */ rc = _ce_setup_aead(pce_dev, q_req, totallen, areq->assoclen, cmdlistinfo); if (rc < 0) { qce_free_req_info(pce_dev, req_info, false); return -EINVAL; } } /* * For crypto 5.0 that has burst size alignment requirement * for data descritpor, * the agent above(qcrypto) prepares the src scatter list with * memory starting with associated data, followed by * iv, and data stream to be ciphered. */ preq_info->src_nents = count_sg(areq->src, totallen); /* cipher input */ qce_dma_map_sg(pce_dev->pdev, areq->src, preq_info->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); /* cipher output for encryption */ if (areq->src != areq->dst) { preq_info->dst_nents = count_sg(areq->dst, totallen); qce_dma_map_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents, DMA_FROM_DEVICE); } /* setup for callback, and issue command to bam */ preq_info->areq = q_req->areq; preq_info->qce_cb = q_req->qce_cb; preq_info->dir = q_req->dir; preq_info->asg = NULL; /* setup xfer type for producer callback handling */ preq_info->xfer_type = QCE_XFER_AEAD; preq_info->req_len = totallen; _qce_sps_iovec_count_init(pce_dev, req_info); if (pce_dev->support_cmd_dscr) { _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK, cmdlistinfo, &pce_sps_data->in_transfer); } else { rc = _ce_setup_aead_direct(pce_dev, q_req, totallen, areq->assoclen); if (rc) goto bad; } preq_info->mode = q_req->mode; if (pce_dev->ce_bam_info.minor_version == 0) { if (_qce_sps_add_sg_data(pce_dev, areq->src, totallen, &pce_sps_data->in_transfer)) goto bad; _qce_set_flag(&pce_sps_data->in_transfer, SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD); if (_qce_sps_add_sg_data(pce_dev, areq->dst, totallen, &pce_sps_data->out_transfer)) goto bad; if (totallen > SPS_MAX_PKT_SIZE) { _qce_set_flag(&pce_sps_data->out_transfer, SPS_IOVEC_FLAG_INT); pce_sps_data->producer_state = QCE_PIPE_STATE_IDLE; } else { if (_qce_sps_add_data(GET_PHYS_ADDR( pce_sps_data->result_dump), CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer)) goto bad; _qce_set_flag(&pce_sps_data->out_transfer, SPS_IOVEC_FLAG_INT); pce_sps_data->producer_state = QCE_PIPE_STATE_COMP; } rc = _qce_sps_transfer(pce_dev, req_info); } else { if (_qce_sps_add_sg_data(pce_dev, areq->src, totallen, &pce_sps_data->in_transfer)) goto bad; _qce_set_flag(&pce_sps_data->in_transfer, SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD); if (pce_dev->no_get_around) _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK, &pce_sps_data->cmdlistptr.unlock_all_pipes, &pce_sps_data->in_transfer); if (_qce_sps_add_sg_data(pce_dev, areq->dst, totallen, &pce_sps_data->out_transfer)) goto bad; if (pce_dev->no_get_around || totallen <= SPS_MAX_PKT_SIZE) { if (_qce_sps_add_data( GET_PHYS_ADDR(pce_sps_data->result_dump), CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer)) goto bad; pce_sps_data->producer_state = QCE_PIPE_STATE_COMP; } else { pce_sps_data->producer_state = QCE_PIPE_STATE_IDLE; } select_mode(pce_dev, preq_info); rc = _qce_sps_transfer(pce_dev, req_info); cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE); } if (rc) goto bad; return 0; bad: if (preq_info->src_nents) qce_dma_unmap_sg(pce_dev->pdev, areq->src, preq_info->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); if (areq->src != areq->dst) qce_dma_unmap_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents, DMA_FROM_DEVICE); qce_free_req_info(pce_dev, req_info, false); return rc; } EXPORT_SYMBOL(qce_aead_req); int qce_ablk_cipher_req(void *handle, struct qce_req *c_req) { int rc = 0; struct qce_device *pce_dev = (struct qce_device *) handle; struct ablkcipher_request *areq = (struct ablkcipher_request *) c_req->areq; struct qce_cmdlist_info *cmdlistinfo = NULL; int req_info = -1; struct ce_sps_data *pce_sps_data; struct ce_request_info *preq_info; req_info = qce_alloc_req_info(pce_dev); if (req_info < 0) return -EBUSY; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; preq_info->src_nents = 0; preq_info->dst_nents = 0; /* cipher input */ preq_info->src_nents = count_sg(areq->src, areq->nbytes); qce_dma_map_sg(pce_dev->pdev, areq->src, preq_info->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); /* cipher output */ if (areq->src != areq->dst) { preq_info->dst_nents = count_sg(areq->dst, areq->nbytes); qce_dma_map_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents, DMA_FROM_DEVICE); } else { preq_info->dst_nents = preq_info->src_nents; } preq_info->dir = c_req->dir; if ((pce_dev->ce_bam_info.minor_version == 0) && (preq_info->dir == QCE_DECRYPT) && (c_req->mode == QCE_MODE_CBC)) { memcpy(preq_info->dec_iv, (unsigned char *) sg_virt(areq->src) + areq->src->length - 16, NUM_OF_CRYPTO_CNTR_IV_REG * CRYPTO_REG_SIZE); } /* set up crypto device */ if (pce_dev->support_cmd_dscr) { cmdlistinfo = _ce_get_cipher_cmdlistinfo(pce_dev, req_info, c_req); if (cmdlistinfo == NULL) { pr_err("Unsupported cipher algorithm %d, mode %d\n", c_req->alg, c_req->mode); qce_free_req_info(pce_dev, req_info, false); return -EINVAL; } rc = _ce_setup_cipher(pce_dev, c_req, areq->nbytes, 0, cmdlistinfo); } else { rc = _ce_setup_cipher_direct(pce_dev, c_req, areq->nbytes, 0); } if (rc < 0) goto bad; preq_info->mode = c_req->mode; /* setup for client callback, and issue command to BAM */ preq_info->areq = areq; preq_info->qce_cb = c_req->qce_cb; /* setup xfer type for producer callback handling */ preq_info->xfer_type = QCE_XFER_CIPHERING; preq_info->req_len = areq->nbytes; _qce_sps_iovec_count_init(pce_dev, req_info); if (pce_dev->support_cmd_dscr) _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK, cmdlistinfo, &pce_sps_data->in_transfer); if (_qce_sps_add_sg_data(pce_dev, areq->src, areq->nbytes, &pce_sps_data->in_transfer)) goto bad; _qce_set_flag(&pce_sps_data->in_transfer, SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD); if (pce_dev->no_get_around) _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK, &pce_sps_data->cmdlistptr.unlock_all_pipes, &pce_sps_data->in_transfer); if (_qce_sps_add_sg_data(pce_dev, areq->dst, areq->nbytes, &pce_sps_data->out_transfer)) goto bad; if (pce_dev->no_get_around || areq->nbytes <= SPS_MAX_PKT_SIZE) { pce_sps_data->producer_state = QCE_PIPE_STATE_COMP; if (_qce_sps_add_data( GET_PHYS_ADDR(pce_sps_data->result_dump), CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer)) goto bad; } else { pce_sps_data->producer_state = QCE_PIPE_STATE_IDLE; } select_mode(pce_dev, preq_info); rc = _qce_sps_transfer(pce_dev, req_info); cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE); if (rc) goto bad; return 0; bad: if (areq->src != areq->dst) { if (preq_info->dst_nents) { qce_dma_unmap_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents, DMA_FROM_DEVICE); } } if (preq_info->src_nents) { qce_dma_unmap_sg(pce_dev->pdev, areq->src, preq_info->src_nents, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); } qce_free_req_info(pce_dev, req_info, false); return rc; } EXPORT_SYMBOL(qce_ablk_cipher_req); int qce_process_sha_req(void *handle, struct qce_sha_req *sreq) { struct qce_device *pce_dev = (struct qce_device *) handle; int rc; struct ahash_request *areq; struct qce_cmdlist_info *cmdlistinfo = NULL; int req_info = -1; struct ce_sps_data *pce_sps_data; struct ce_request_info *preq_info; bool is_dummy = false; if (!sreq) { sreq = &(pce_dev->dummyreq.sreq); req_info = DUMMY_REQ_INDEX; is_dummy = true; } else { req_info = qce_alloc_req_info(pce_dev); if (req_info < 0) return -EBUSY; } areq = (struct ahash_request *)sreq->areq; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; preq_info->src_nents = count_sg(sreq->src, sreq->size); qce_dma_map_sg(pce_dev->pdev, sreq->src, preq_info->src_nents, DMA_TO_DEVICE); if (pce_dev->support_cmd_dscr) { cmdlistinfo = _ce_get_hash_cmdlistinfo(pce_dev, req_info, sreq); if (cmdlistinfo == NULL) { pr_err("Unsupported hash algorithm %d\n", sreq->alg); qce_free_req_info(pce_dev, req_info, false); return -EINVAL; } rc = _ce_setup_hash(pce_dev, sreq, cmdlistinfo); } else { rc = _ce_setup_hash_direct(pce_dev, sreq); } if (rc < 0) goto bad; preq_info->areq = areq; preq_info->qce_cb = sreq->qce_cb; /* setup xfer type for producer callback handling */ preq_info->xfer_type = QCE_XFER_HASHING; preq_info->req_len = sreq->size; _qce_sps_iovec_count_init(pce_dev, req_info); if (pce_dev->support_cmd_dscr) _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK, cmdlistinfo, &pce_sps_data->in_transfer); if (_qce_sps_add_sg_data(pce_dev, areq->src, areq->nbytes, &pce_sps_data->in_transfer)) goto bad; /* always ensure there is input data. ZLT does not work for bam-ndp */ if (!areq->nbytes) _qce_sps_add_data( GET_PHYS_ADDR(pce_sps_data->ignore_buffer), pce_dev->ce_bam_info.ce_burst_size, &pce_sps_data->in_transfer); _qce_set_flag(&pce_sps_data->in_transfer, SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD); if (pce_dev->no_get_around) _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK, &pce_sps_data->cmdlistptr.unlock_all_pipes, &pce_sps_data->in_transfer); if (_qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump), CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer)) goto bad; if (is_dummy) { _qce_set_flag(&pce_sps_data->out_transfer, SPS_IOVEC_FLAG_INT); rc = _qce_sps_transfer(pce_dev, req_info); } else { select_mode(pce_dev, preq_info); rc = _qce_sps_transfer(pce_dev, req_info); cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE); } if (rc) goto bad; return 0; bad: if (preq_info->src_nents) { qce_dma_unmap_sg(pce_dev->pdev, sreq->src, preq_info->src_nents, DMA_TO_DEVICE); } qce_free_req_info(pce_dev, req_info, false); return rc; } EXPORT_SYMBOL(qce_process_sha_req); int qce_f8_req(void *handle, struct qce_f8_req *req, void *cookie, qce_comp_func_ptr_t qce_cb) { struct qce_device *pce_dev = (struct qce_device *) handle; bool key_stream_mode; dma_addr_t dst; int rc; struct qce_cmdlist_info *cmdlistinfo; int req_info = -1; struct ce_request_info *preq_info; struct ce_sps_data *pce_sps_data; req_info = qce_alloc_req_info(pce_dev); if (req_info < 0) return -EBUSY; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; switch (req->algorithm) { case QCE_OTA_ALGO_KASUMI: cmdlistinfo = &pce_sps_data->cmdlistptr.f8_kasumi; break; case QCE_OTA_ALGO_SNOW3G: cmdlistinfo = &pce_sps_data->cmdlistptr.f8_snow3g; break; default: qce_free_req_info(pce_dev, req_info, false); return -EINVAL; }; key_stream_mode = (req->data_in == NULL); /* don't support key stream mode */ if (key_stream_mode || (req->bearer >= QCE_OTA_MAX_BEARER)) { qce_free_req_info(pce_dev, req_info, false); return -EINVAL; } /* F8 cipher input */ preq_info->phy_ota_src = dma_map_single(pce_dev->pdev, req->data_in, req->data_len, (req->data_in == req->data_out) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); /* F8 cipher output */ if (req->data_in != req->data_out) { dst = dma_map_single(pce_dev->pdev, req->data_out, req->data_len, DMA_FROM_DEVICE); preq_info->phy_ota_dst = dst; } else { /* in place ciphering */ dst = preq_info->phy_ota_src; preq_info->phy_ota_dst = 0; } preq_info->ota_size = req->data_len; /* set up crypto device */ if (pce_dev->support_cmd_dscr) rc = _ce_f8_setup(pce_dev, req, key_stream_mode, 1, 0, req->data_len, cmdlistinfo); else rc = _ce_f8_setup_direct(pce_dev, req, key_stream_mode, 1, 0, req->data_len); if (rc < 0) goto bad; /* setup for callback, and issue command to sps */ preq_info->areq = cookie; preq_info->qce_cb = qce_cb; /* setup xfer type for producer callback handling */ preq_info->xfer_type = QCE_XFER_F8; preq_info->req_len = req->data_len; _qce_sps_iovec_count_init(pce_dev, req_info); if (pce_dev->support_cmd_dscr) _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK, cmdlistinfo, &pce_sps_data->in_transfer); _qce_sps_add_data((uint32_t)preq_info->phy_ota_src, req->data_len, &pce_sps_data->in_transfer); _qce_set_flag(&pce_sps_data->in_transfer, SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD); _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK, &pce_sps_data->cmdlistptr.unlock_all_pipes, &pce_sps_data->in_transfer); _qce_sps_add_data((uint32_t)dst, req->data_len, &pce_sps_data->out_transfer); _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump), CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer); select_mode(pce_dev, preq_info); rc = _qce_sps_transfer(pce_dev, req_info); cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE); if (rc) goto bad; return 0; bad: if (preq_info->phy_ota_dst != 0) dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_dst, req->data_len, DMA_FROM_DEVICE); if (preq_info->phy_ota_src != 0) dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_src, req->data_len, (req->data_in == req->data_out) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); qce_free_req_info(pce_dev, req_info, false); return rc; } EXPORT_SYMBOL(qce_f8_req); int qce_f8_multi_pkt_req(void *handle, struct qce_f8_multi_pkt_req *mreq, void *cookie, qce_comp_func_ptr_t qce_cb) { struct qce_device *pce_dev = (struct qce_device *) handle; uint16_t num_pkt = mreq->num_pkt; uint16_t cipher_start = mreq->cipher_start; uint16_t cipher_size = mreq->cipher_size; struct qce_f8_req *req = &mreq->qce_f8_req; uint32_t total; dma_addr_t dst = 0; int rc = 0; struct qce_cmdlist_info *cmdlistinfo; int req_info = -1; struct ce_request_info *preq_info; struct ce_sps_data *pce_sps_data; req_info = qce_alloc_req_info(pce_dev); if (req_info < 0) return -EBUSY; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; switch (req->algorithm) { case QCE_OTA_ALGO_KASUMI: cmdlistinfo = &pce_sps_data->cmdlistptr.f8_kasumi; break; case QCE_OTA_ALGO_SNOW3G: cmdlistinfo = &pce_sps_data->cmdlistptr.f8_snow3g; break; default: qce_free_req_info(pce_dev, req_info, false); return -EINVAL; }; total = num_pkt * req->data_len; /* F8 cipher input */ preq_info->phy_ota_src = dma_map_single(pce_dev->pdev, req->data_in, total, (req->data_in == req->data_out) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); /* F8 cipher output */ if (req->data_in != req->data_out) { dst = dma_map_single(pce_dev->pdev, req->data_out, total, DMA_FROM_DEVICE); preq_info->phy_ota_dst = dst; } else { /* in place ciphering */ dst = preq_info->phy_ota_src; preq_info->phy_ota_dst = 0; } preq_info->ota_size = total; /* set up crypto device */ if (pce_dev->support_cmd_dscr) rc = _ce_f8_setup(pce_dev, req, false, num_pkt, cipher_start, cipher_size, cmdlistinfo); else rc = _ce_f8_setup_direct(pce_dev, req, false, num_pkt, cipher_start, cipher_size); if (rc) goto bad; /* setup for callback, and issue command to sps */ preq_info->areq = cookie; preq_info->qce_cb = qce_cb; /* setup xfer type for producer callback handling */ preq_info->xfer_type = QCE_XFER_F8; preq_info->req_len = total; _qce_sps_iovec_count_init(pce_dev, req_info); if (pce_dev->support_cmd_dscr) _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK, cmdlistinfo, &pce_sps_data->in_transfer); _qce_sps_add_data((uint32_t)preq_info->phy_ota_src, total, &pce_sps_data->in_transfer); _qce_set_flag(&pce_sps_data->in_transfer, SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD); _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK, &pce_sps_data->cmdlistptr.unlock_all_pipes, &pce_sps_data->in_transfer); _qce_sps_add_data((uint32_t)dst, total, &pce_sps_data->out_transfer); _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump), CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer); select_mode(pce_dev, preq_info); rc = _qce_sps_transfer(pce_dev, req_info); cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE); if (rc == 0) return 0; bad: if (preq_info->phy_ota_dst) dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_dst, total, DMA_FROM_DEVICE); dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_src, total, (req->data_in == req->data_out) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); qce_free_req_info(pce_dev, req_info, false); return rc; } EXPORT_SYMBOL(qce_f8_multi_pkt_req); int qce_f9_req(void *handle, struct qce_f9_req *req, void *cookie, qce_comp_func_ptr_t qce_cb) { struct qce_device *pce_dev = (struct qce_device *) handle; int rc; struct qce_cmdlist_info *cmdlistinfo; int req_info = -1; struct ce_sps_data *pce_sps_data; struct ce_request_info *preq_info; req_info = qce_alloc_req_info(pce_dev); if (req_info < 0) return -EBUSY; preq_info = &pce_dev->ce_request_info[req_info]; pce_sps_data = &preq_info->ce_sps; switch (req->algorithm) { case QCE_OTA_ALGO_KASUMI: cmdlistinfo = &pce_sps_data->cmdlistptr.f9_kasumi; break; case QCE_OTA_ALGO_SNOW3G: cmdlistinfo = &pce_sps_data->cmdlistptr.f9_snow3g; break; default: qce_free_req_info(pce_dev, req_info, false); return -EINVAL; }; preq_info->phy_ota_src = dma_map_single(pce_dev->pdev, req->message, req->msize, DMA_TO_DEVICE); preq_info->ota_size = req->msize; if (pce_dev->support_cmd_dscr) rc = _ce_f9_setup(pce_dev, req, cmdlistinfo); else rc = _ce_f9_setup_direct(pce_dev, req); if (rc < 0) goto bad; /* setup for callback, and issue command to sps */ preq_info->areq = cookie; preq_info->qce_cb = qce_cb; /* setup xfer type for producer callback handling */ preq_info->xfer_type = QCE_XFER_F9; preq_info->req_len = req->msize; _qce_sps_iovec_count_init(pce_dev, req_info); if (pce_dev->support_cmd_dscr) _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK, cmdlistinfo, &pce_sps_data->in_transfer); _qce_sps_add_data((uint32_t)preq_info->phy_ota_src, req->msize, &pce_sps_data->in_transfer); _qce_set_flag(&pce_sps_data->in_transfer, SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD); _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK, &pce_sps_data->cmdlistptr.unlock_all_pipes, &pce_sps_data->in_transfer); _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump), CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer); select_mode(pce_dev, preq_info); rc = _qce_sps_transfer(pce_dev, req_info); cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE); if (rc) goto bad; return 0; bad: dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_src, req->msize, DMA_TO_DEVICE); qce_free_req_info(pce_dev, req_info, false); return rc; } EXPORT_SYMBOL(qce_f9_req); static int __qce_get_device_tree_data(struct platform_device *pdev, struct qce_device *pce_dev) { struct resource *resource; int rc = 0; pce_dev->is_shared = of_property_read_bool((&pdev->dev)->of_node, "qcom,ce-hw-shared"); pce_dev->support_hw_key = of_property_read_bool((&pdev->dev)->of_node, "qcom,ce-hw-key"); pce_dev->use_sw_aes_cbc_ecb_ctr_algo = of_property_read_bool((&pdev->dev)->of_node, "qcom,use-sw-aes-cbc-ecb-ctr-algo"); pce_dev->use_sw_aead_algo = of_property_read_bool((&pdev->dev)->of_node, "qcom,use-sw-aead-algo"); pce_dev->use_sw_aes_xts_algo = of_property_read_bool((&pdev->dev)->of_node, "qcom,use-sw-aes-xts-algo"); pce_dev->use_sw_ahash_algo = of_property_read_bool((&pdev->dev)->of_node, "qcom,use-sw-ahash-algo"); pce_dev->use_sw_hmac_algo = of_property_read_bool((&pdev->dev)->of_node, "qcom,use-sw-hmac-algo"); pce_dev->use_sw_aes_ccm_algo = of_property_read_bool((&pdev->dev)->of_node, "qcom,use-sw-aes-ccm-algo"); pce_dev->support_clk_mgmt_sus_res = of_property_read_bool( (&pdev->dev)->of_node, "qcom,clk-mgmt-sus-res"); pce_dev->support_only_core_src_clk = of_property_read_bool( (&pdev->dev)->of_node, "qcom,support-core-clk-only"); if (of_property_read_u32((&pdev->dev)->of_node, "qcom,bam-pipe-pair", &pce_dev->ce_bam_info.pipe_pair_index)) { pr_err("Fail to get bam pipe pair information.\n"); return -EINVAL; } if (of_property_read_u32((&pdev->dev)->of_node, "qcom,ce-device", &pce_dev->ce_bam_info.ce_device)) { pr_err("Fail to get CE device information.\n"); return -EINVAL; } if (of_property_read_u32((&pdev->dev)->of_node, "qcom,ce-hw-instance", &pce_dev->ce_bam_info.ce_hw_instance)) { pr_err("Fail to get CE hw instance information.\n"); return -EINVAL; } if (of_property_read_u32((&pdev->dev)->of_node, "qcom,bam-ee", &pce_dev->ce_bam_info.bam_ee)) { pr_info("BAM Apps EE is not defined, setting to default 1\n"); pce_dev->ce_bam_info.bam_ee = 1; } if (of_property_read_u32((&pdev->dev)->of_node, "qcom,ce-opp-freq", &pce_dev->ce_opp_freq_hz)) { pr_info("CE operating frequency is not defined, setting to default 100MHZ\n"); pce_dev->ce_opp_freq_hz = CE_CLK_100MHZ; } pce_dev->ce_bam_info.dest_pipe_index = 2 * pce_dev->ce_bam_info.pipe_pair_index; pce_dev->ce_bam_info.src_pipe_index = pce_dev->ce_bam_info.dest_pipe_index + 1; resource = platform_get_resource_byname(pdev, IORESOURCE_MEM, "crypto-base"); if (resource) { pce_dev->phy_iobase = resource->start; pce_dev->iobase = ioremap_nocache(resource->start, resource_size(resource)); if (!pce_dev->iobase) { pr_err("Can not map CRYPTO io memory\n"); return -ENOMEM; } } else { pr_err("CRYPTO HW mem unavailable.\n"); return -ENODEV; } resource = platform_get_resource_byname(pdev, IORESOURCE_MEM, "crypto-bam-base"); if (resource) { pce_dev->bam_mem = resource->start; pce_dev->bam_mem_size = resource_size(resource); } else { pr_err("CRYPTO BAM mem unavailable.\n"); rc = -ENODEV; goto err_getting_bam_info; } resource = platform_get_resource(pdev, IORESOURCE_IRQ, 0); if (resource) { pce_dev->ce_bam_info.bam_irq = resource->start; } else { pr_err("CRYPTO BAM IRQ unavailable.\n"); goto err_dev; } return rc; err_dev: if (pce_dev->ce_bam_info.bam_iobase) iounmap(pce_dev->ce_bam_info.bam_iobase); err_getting_bam_info: if (pce_dev->iobase) iounmap(pce_dev->iobase); return rc; } static int __qce_init_clk(struct qce_device *pce_dev) { int rc = 0; pce_dev->ce_core_src_clk = clk_get(pce_dev->pdev, "core_clk_src"); if (!IS_ERR(pce_dev->ce_core_src_clk)) { rc = clk_set_rate(pce_dev->ce_core_src_clk, pce_dev->ce_opp_freq_hz); if (rc) { pr_err("Unable to set the core src clk @%uMhz.\n", pce_dev->ce_opp_freq_hz/CE_CLK_DIV); goto exit_put_core_src_clk; } } else { if (pce_dev->support_only_core_src_clk) { rc = PTR_ERR(pce_dev->ce_core_src_clk); pce_dev->ce_core_src_clk = NULL; pr_err("Unable to get CE core src clk\n"); return rc; } else { pr_warn("Unable to get CE core src clk, set to NULL\n"); pce_dev->ce_core_src_clk = NULL; } } if (pce_dev->support_only_core_src_clk) { pce_dev->ce_core_clk = NULL; pce_dev->ce_clk = NULL; pce_dev->ce_bus_clk = NULL; } else { pce_dev->ce_core_clk = clk_get(pce_dev->pdev, "core_clk"); if (IS_ERR(pce_dev->ce_core_clk)) { rc = PTR_ERR(pce_dev->ce_core_clk); pr_err("Unable to get CE core clk\n"); goto exit_put_core_src_clk; } pce_dev->ce_clk = clk_get(pce_dev->pdev, "iface_clk"); if (IS_ERR(pce_dev->ce_clk)) { rc = PTR_ERR(pce_dev->ce_clk); pr_err("Unable to get CE interface clk\n"); goto exit_put_core_clk; } pce_dev->ce_bus_clk = clk_get(pce_dev->pdev, "bus_clk"); if (IS_ERR(pce_dev->ce_bus_clk)) { rc = PTR_ERR(pce_dev->ce_bus_clk); pr_err("Unable to get CE BUS interface clk\n"); goto exit_put_iface_clk; } } return rc; exit_put_iface_clk: if (pce_dev->ce_clk) clk_put(pce_dev->ce_clk); exit_put_core_clk: if (pce_dev->ce_core_clk) clk_put(pce_dev->ce_core_clk); exit_put_core_src_clk: if (pce_dev->ce_core_src_clk) clk_put(pce_dev->ce_core_src_clk); pr_err("Unable to init CE clks, rc = %d\n", rc); return rc; } static void __qce_deinit_clk(struct qce_device *pce_dev) { if (pce_dev->ce_bus_clk) clk_put(pce_dev->ce_bus_clk); if (pce_dev->ce_clk) clk_put(pce_dev->ce_clk); if (pce_dev->ce_core_clk) clk_put(pce_dev->ce_core_clk); if (pce_dev->ce_core_src_clk) clk_put(pce_dev->ce_core_src_clk); } int qce_enable_clk(void *handle) { struct qce_device *pce_dev = (struct qce_device *)handle; int rc = 0; if (pce_dev->ce_core_src_clk) { rc = clk_prepare_enable(pce_dev->ce_core_src_clk); if (rc) { pr_err("Unable to enable/prepare CE core src clk\n"); return rc; } } if (pce_dev->support_only_core_src_clk) return rc; if (pce_dev->ce_core_clk) { rc = clk_prepare_enable(pce_dev->ce_core_clk); if (rc) { pr_err("Unable to enable/prepare CE core clk\n"); goto exit_disable_core_src_clk; } } if (pce_dev->ce_clk) { rc = clk_prepare_enable(pce_dev->ce_clk); if (rc) { pr_err("Unable to enable/prepare CE iface clk\n"); goto exit_disable_core_clk; } } if (pce_dev->ce_bus_clk) { rc = clk_prepare_enable(pce_dev->ce_bus_clk); if (rc) { pr_err("Unable to enable/prepare CE BUS clk\n"); goto exit_disable_ce_clk; } } return rc; exit_disable_ce_clk: if (pce_dev->ce_clk) clk_disable_unprepare(pce_dev->ce_clk); exit_disable_core_clk: if (pce_dev->ce_core_clk) clk_disable_unprepare(pce_dev->ce_core_clk); exit_disable_core_src_clk: if (pce_dev->ce_core_src_clk) clk_disable_unprepare(pce_dev->ce_core_src_clk); return rc; } EXPORT_SYMBOL(qce_enable_clk); int qce_disable_clk(void *handle) { struct qce_device *pce_dev = (struct qce_device *) handle; int rc = 0; if (pce_dev->ce_bus_clk) clk_disable_unprepare(pce_dev->ce_bus_clk); if (pce_dev->ce_clk) clk_disable_unprepare(pce_dev->ce_clk); if (pce_dev->ce_core_clk) clk_disable_unprepare(pce_dev->ce_core_clk); if (pce_dev->ce_core_src_clk) clk_disable_unprepare(pce_dev->ce_core_src_clk); return rc; } EXPORT_SYMBOL(qce_disable_clk); /* dummy req setup */ static int setup_dummy_req(struct qce_device *pce_dev) { char *input = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopqopqrpqrs"; int len = DUMMY_REQ_DATA_LEN; memcpy(pce_dev->dummyreq_in_buf, input, len); sg_set_buf(&pce_dev->dummyreq.sg, pce_dev->dummyreq_in_buf, len); sg_mark_end(&pce_dev->dummyreq.sg); pce_dev->dummyreq.sreq.alg = QCE_HASH_SHA1; pce_dev->dummyreq.sreq.qce_cb = qce_dummy_complete; pce_dev->dummyreq.sreq.src = &pce_dev->dummyreq.sg; pce_dev->dummyreq.sreq.auth_data[0] = 0; pce_dev->dummyreq.sreq.auth_data[1] = 0; pce_dev->dummyreq.sreq.auth_data[2] = 0; pce_dev->dummyreq.sreq.auth_data[3] = 0; pce_dev->dummyreq.sreq.first_blk = 1; pce_dev->dummyreq.sreq.last_blk = 1; pce_dev->dummyreq.sreq.size = len; pce_dev->dummyreq.sreq.areq = &pce_dev->dummyreq.areq; pce_dev->dummyreq.sreq.flags = 0; pce_dev->dummyreq.sreq.authkey = NULL; pce_dev->dummyreq.areq.src = pce_dev->dummyreq.sreq.src; pce_dev->dummyreq.areq.nbytes = pce_dev->dummyreq.sreq.size; return 0; } /* crypto engine open function. */ void *qce_open(struct platform_device *pdev, int *rc) { struct qce_device *pce_dev; int i; static int pcedev_no = 1; pce_dev = kzalloc(sizeof(struct qce_device), GFP_KERNEL); if (!pce_dev) { *rc = -ENOMEM; pr_err("Can not allocate memory: %d\n", *rc); return NULL; } pce_dev->pdev = &pdev->dev; mutex_lock(&qce_iomap_mutex); if (pdev->dev.of_node) { *rc = __qce_get_device_tree_data(pdev, pce_dev); if (*rc) goto err_pce_dev; } else { *rc = -EINVAL; pr_err("Device Node not found.\n"); goto err_pce_dev; } for (i = 0; i < MAX_QCE_ALLOC_BAM_REQ; i++) atomic_set(&pce_dev->ce_request_info[i].in_use, false); pce_dev->ce_request_index = 0; pce_dev->memsize = 10 * PAGE_SIZE * MAX_QCE_ALLOC_BAM_REQ; pce_dev->coh_vmem = dma_alloc_coherent(pce_dev->pdev, pce_dev->memsize, &pce_dev->coh_pmem, GFP_KERNEL); if (pce_dev->coh_vmem == NULL) { *rc = -ENOMEM; pr_err("Can not allocate coherent memory for sps data\n"); goto err_iobase; } pce_dev->iovec_memsize = TOTAL_IOVEC_SPACE_PER_PIPE * MAX_QCE_ALLOC_BAM_REQ * 2; pce_dev->iovec_vmem = kzalloc(pce_dev->iovec_memsize, GFP_KERNEL); if (pce_dev->iovec_vmem == NULL) goto err_mem; pce_dev->dummyreq_in_buf = kzalloc(DUMMY_REQ_DATA_LEN, GFP_KERNEL); if (pce_dev->dummyreq_in_buf == NULL) goto err_mem; *rc = __qce_init_clk(pce_dev); if (*rc) goto err_mem; *rc = qce_enable_clk(pce_dev); if (*rc) goto err_enable_clk; if (_probe_ce_engine(pce_dev)) { *rc = -ENXIO; goto err; } *rc = 0; qce_init_ce_cfg_val(pce_dev); *rc = qce_sps_init(pce_dev); if (*rc) goto err; qce_setup_ce_sps_data(pce_dev); qce_disable_clk(pce_dev); setup_dummy_req(pce_dev); atomic_set(&pce_dev->no_of_queued_req, 0); pce_dev->mode = IN_INTERRUPT_MODE; init_timer(&(pce_dev->timer)); pce_dev->timer.function = qce_multireq_timeout; pce_dev->timer.data = (unsigned long)pce_dev; pce_dev->timer.expires = jiffies + DELAY_IN_JIFFIES; pce_dev->intr_cadence = 0; pce_dev->dev_no = pcedev_no; pcedev_no++; pce_dev->owner = QCE_OWNER_NONE; mutex_unlock(&qce_iomap_mutex); return pce_dev; err: qce_disable_clk(pce_dev); err_enable_clk: __qce_deinit_clk(pce_dev); err_mem: kfree(pce_dev->dummyreq_in_buf); kfree(pce_dev->iovec_vmem); if (pce_dev->coh_vmem) dma_free_coherent(pce_dev->pdev, pce_dev->memsize, pce_dev->coh_vmem, pce_dev->coh_pmem); err_iobase: if (pce_dev->iobase) iounmap(pce_dev->iobase); err_pce_dev: mutex_unlock(&qce_iomap_mutex); kfree(pce_dev); return NULL; } EXPORT_SYMBOL(qce_open); /* crypto engine close function. */ int qce_close(void *handle) { struct qce_device *pce_dev = (struct qce_device *) handle; if (handle == NULL) return -ENODEV; mutex_lock(&qce_iomap_mutex); qce_enable_clk(pce_dev); qce_sps_exit(pce_dev); if (pce_dev->iobase) iounmap(pce_dev->iobase); if (pce_dev->coh_vmem) dma_free_coherent(pce_dev->pdev, pce_dev->memsize, pce_dev->coh_vmem, pce_dev->coh_pmem); kfree(pce_dev->dummyreq_in_buf); kfree(pce_dev->iovec_vmem); qce_disable_clk(pce_dev); __qce_deinit_clk(pce_dev); mutex_unlock(&qce_iomap_mutex); kfree(handle); return 0; } EXPORT_SYMBOL(qce_close); #define OTA_SUPPORT_MASK (1 << CRYPTO_ENCR_SNOW3G_SEL |\ 1 << CRYPTO_ENCR_KASUMI_SEL |\ 1 << CRYPTO_AUTH_SNOW3G_SEL |\ 1 << CRYPTO_AUTH_KASUMI_SEL) int qce_hw_support(void *handle, struct ce_hw_support *ce_support) { struct qce_device *pce_dev = (struct qce_device *)handle; if (ce_support == NULL) return -EINVAL; ce_support->sha1_hmac_20 = false; ce_support->sha1_hmac = false; ce_support->sha256_hmac = false; ce_support->sha_hmac = true; ce_support->cmac = true; ce_support->aes_key_192 = false; ce_support->aes192_fb_sha1 = false; ce_support->aes_xts = true; if ((pce_dev->engines_avail & OTA_SUPPORT_MASK) == OTA_SUPPORT_MASK) ce_support->ota = true; else ce_support->ota = false; ce_support->bam = true; ce_support->is_shared = (pce_dev->is_shared == 1) ? true : false; ce_support->hw_key = pce_dev->support_hw_key; ce_support->aes_ccm = true; ce_support->clk_mgmt_sus_res = pce_dev->support_clk_mgmt_sus_res; if (pce_dev->ce_bam_info.minor_version) ce_support->aligned_only = false; else ce_support->aligned_only = true; ce_support->use_sw_aes_cbc_ecb_ctr_algo = pce_dev->use_sw_aes_cbc_ecb_ctr_algo; ce_support->use_sw_aead_algo = pce_dev->use_sw_aead_algo; ce_support->use_sw_aes_xts_algo = pce_dev->use_sw_aes_xts_algo; ce_support->use_sw_ahash_algo = pce_dev->use_sw_ahash_algo; ce_support->use_sw_hmac_algo = pce_dev->use_sw_hmac_algo; ce_support->use_sw_aes_ccm_algo = pce_dev->use_sw_aes_ccm_algo; ce_support->ce_device = pce_dev->ce_bam_info.ce_device; ce_support->ce_hw_instance = pce_dev->ce_bam_info.ce_hw_instance; if (pce_dev->no_get_around) ce_support->max_request = MAX_QCE_BAM_REQ; else ce_support->max_request = 1; return 0; } EXPORT_SYMBOL(qce_hw_support); void qce_dump_req(void *handle) { int i; bool req_in_use; struct qce_device *pce_dev = (struct qce_device *)handle; for (i = 0; i < MAX_QCE_BAM_REQ; i++) { req_in_use = atomic_read(&pce_dev->ce_request_info[i].in_use); pr_info("qce_dump_req %d %d\n", i, req_in_use); if (req_in_use == true) _qce_dump_descr_fifos(pce_dev, i); } } EXPORT_SYMBOL(qce_dump_req); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Crypto Engine driver");