--- zzzz-none-000/linux-3.10.107/drivers/dma/at_hdmac.c 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/drivers/dma/at_hdmac.c 2021-02-04 17:41:59.000000000 +0000 @@ -14,6 +14,7 @@ * found on AT91SAM9263. */ +#include #include #include #include @@ -41,6 +42,13 @@ #define ATC_DEFAULT_CFG (ATC_FIFOCFG_HALFFIFO) #define ATC_DEFAULT_CTRLB (ATC_SIF(AT_DMA_MEM_IF) \ |ATC_DIF(AT_DMA_MEM_IF)) +#define ATC_DMA_BUSWIDTHS\ + (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) |\ + BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |\ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |\ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) + +#define ATC_MAX_DSCR_TRIALS 10 /* * Initial number of descriptors to allocate for each channel. This could @@ -54,10 +62,26 @@ /* prototypes */ static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx); +static void atc_issue_pending(struct dma_chan *chan); /*----------------------------------------------------------------------*/ +static inline unsigned int atc_get_xfer_width(dma_addr_t src, dma_addr_t dst, + size_t len) +{ + unsigned int width; + + if (!((src | dst | len) & 3)) + width = 2; + else if (!((src | dst | len) & 1)) + width = 1; + else + width = 0; + + return width; +} + static struct at_desc *atc_first_active(struct at_dma_chan *atchan) { return list_first_entry(&atchan->active_list, @@ -225,11 +249,195 @@ channel_writel(atchan, CTRLA, 0); channel_writel(atchan, CTRLB, 0); channel_writel(atchan, DSCR, first->txd.phys); + channel_writel(atchan, SPIP, ATC_SPIP_HOLE(first->src_hole) | + ATC_SPIP_BOUNDARY(first->boundary)); + channel_writel(atchan, DPIP, ATC_DPIP_HOLE(first->dst_hole) | + ATC_DPIP_BOUNDARY(first->boundary)); dma_writel(atdma, CHER, atchan->mask); vdbg_dump_regs(atchan); } +/* + * atc_get_desc_by_cookie - get the descriptor of a cookie + * @atchan: the DMA channel + * @cookie: the cookie to get the descriptor for + */ +static struct at_desc *atc_get_desc_by_cookie(struct at_dma_chan *atchan, + dma_cookie_t cookie) +{ + struct at_desc *desc, *_desc; + + list_for_each_entry_safe(desc, _desc, &atchan->queue, desc_node) { + if (desc->txd.cookie == cookie) + return desc; + } + + list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { + if (desc->txd.cookie == cookie) + return desc; + } + + return NULL; +} + +/** + * atc_calc_bytes_left - calculates the number of bytes left according to the + * value read from CTRLA. + * + * @current_len: the number of bytes left before reading CTRLA + * @ctrla: the value of CTRLA + */ +static inline int atc_calc_bytes_left(int current_len, u32 ctrla) +{ + u32 btsize = (ctrla & ATC_BTSIZE_MAX); + u32 src_width = ATC_REG_TO_SRC_WIDTH(ctrla); + + /* + * According to the datasheet, when reading the Control A Register + * (ctrla), the Buffer Transfer Size (btsize) bitfield refers to the + * number of transfers completed on the Source Interface. + * So btsize is always a number of source width transfers. + */ + return current_len - (btsize << src_width); +} + +/** + * atc_get_bytes_left - get the number of bytes residue for a cookie + * @chan: DMA channel + * @cookie: transaction identifier to check status of + */ +static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_desc *desc_first = atc_first_active(atchan); + struct at_desc *desc; + int ret; + u32 ctrla, dscr, trials; + + /* + * If the cookie doesn't match to the currently running transfer then + * we can return the total length of the associated DMA transfer, + * because it is still queued. + */ + desc = atc_get_desc_by_cookie(atchan, cookie); + if (desc == NULL) + return -EINVAL; + else if (desc != desc_first) + return desc->total_len; + + /* cookie matches to the currently running transfer */ + ret = desc_first->total_len; + + if (desc_first->lli.dscr) { + /* hardware linked list transfer */ + + /* + * Calculate the residue by removing the length of the child + * descriptors already transferred from the total length. + * To get the current child descriptor we can use the value of + * the channel's DSCR register and compare it against the value + * of the hardware linked list structure of each child + * descriptor. + * + * The CTRLA register provides us with the amount of data + * already read from the source for the current child + * descriptor. So we can compute a more accurate residue by also + * removing the number of bytes corresponding to this amount of + * data. + * + * However, the DSCR and CTRLA registers cannot be read both + * atomically. Hence a race condition may occur: the first read + * register may refer to one child descriptor whereas the second + * read may refer to a later child descriptor in the list + * because of the DMA transfer progression inbetween the two + * reads. + * + * One solution could have been to pause the DMA transfer, read + * the DSCR and CTRLA then resume the DMA transfer. Nonetheless, + * this approach presents some drawbacks: + * - If the DMA transfer is paused, RX overruns or TX underruns + * are more likey to occur depending on the system latency. + * Taking the USART driver as an example, it uses a cyclic DMA + * transfer to read data from the Receive Holding Register + * (RHR) to avoid RX overruns since the RHR is not protected + * by any FIFO on most Atmel SoCs. So pausing the DMA transfer + * to compute the residue would break the USART driver design. + * - The atc_pause() function masks interrupts but we'd rather + * avoid to do so for system latency purpose. + * + * Then we'd rather use another solution: the DSCR is read a + * first time, the CTRLA is read in turn, next the DSCR is read + * a second time. If the two consecutive read values of the DSCR + * are the same then we assume both refers to the very same + * child descriptor as well as the CTRLA value read inbetween + * does. For cyclic tranfers, the assumption is that a full loop + * is "not so fast". + * If the two DSCR values are different, we read again the CTRLA + * then the DSCR till two consecutive read values from DSCR are + * equal or till the maxium trials is reach. + * This algorithm is very unlikely not to find a stable value for + * DSCR. + */ + + dscr = channel_readl(atchan, DSCR); + rmb(); /* ensure DSCR is read before CTRLA */ + ctrla = channel_readl(atchan, CTRLA); + for (trials = 0; trials < ATC_MAX_DSCR_TRIALS; ++trials) { + u32 new_dscr; + + rmb(); /* ensure DSCR is read after CTRLA */ + new_dscr = channel_readl(atchan, DSCR); + + /* + * If the DSCR register value has not changed inside the + * DMA controller since the previous read, we assume + * that both the dscr and ctrla values refers to the + * very same descriptor. + */ + if (likely(new_dscr == dscr)) + break; + + /* + * DSCR has changed inside the DMA controller, so the + * previouly read value of CTRLA may refer to an already + * processed descriptor hence could be outdated. + * We need to update ctrla to match the current + * descriptor. + */ + dscr = new_dscr; + rmb(); /* ensure DSCR is read before CTRLA */ + ctrla = channel_readl(atchan, CTRLA); + } + if (unlikely(trials >= ATC_MAX_DSCR_TRIALS)) + return -ETIMEDOUT; + + /* for the first descriptor we can be more accurate */ + if (desc_first->lli.dscr == dscr) + return atc_calc_bytes_left(ret, ctrla); + + ret -= desc_first->len; + list_for_each_entry(desc, &desc_first->tx_list, desc_node) { + if (desc->lli.dscr == dscr) + break; + + ret -= desc->len; + } + + /* + * For the current descriptor in the chain we can calculate + * the remaining bytes using the channel's register. + */ + ret = atc_calc_bytes_left(ret, ctrla); + } else { + /* single transfer */ + ctrla = channel_readl(atchan, CTRLA); + ret = atc_calc_bytes_left(ret, ctrla); + } + + return ret; +} + /** * atc_chain_complete - finish work for one transaction chain * @atchan: channel we work on @@ -240,6 +448,7 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) { struct dma_async_tx_descriptor *txd = &desc->txd; + struct at_dma *atdma = to_at_dma(atchan->chan_common.device); dev_vdbg(chan2dev(&atchan->chan_common), "descriptor %u complete\n", txd->cookie); @@ -248,36 +457,19 @@ if (!atc_chan_is_cyclic(atchan)) dma_cookie_complete(txd); + /* If the transfer was a memset, free our temporary buffer */ + if (desc->memset_buffer) { + dma_pool_free(atdma->memset_pool, desc->memset_vaddr, + desc->memset_paddr); + desc->memset_buffer = false; + } + /* move children to free_list */ list_splice_init(&desc->tx_list, &atchan->free_list); /* move myself to free_list */ list_move(&desc->desc_node, &atchan->free_list); - /* unmap dma addresses (not on slave channels) */ - if (!atchan->chan_common.private) { - struct device *parent = chan2parent(&atchan->chan_common); - if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) - dma_unmap_single(parent, - desc->lli.daddr, - desc->len, DMA_FROM_DEVICE); - else - dma_unmap_page(parent, - desc->lli.daddr, - desc->len, DMA_FROM_DEVICE); - } - if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) - dma_unmap_single(parent, - desc->lli.saddr, - desc->len, DMA_TO_DEVICE); - else - dma_unmap_page(parent, - desc->lli.saddr, - desc->len, DMA_TO_DEVICE); - } - } - + dma_descriptor_unmap(txd); /* for cyclic transfers, * no need to replay callback function while stopping */ if (!atc_chan_is_cyclic(atchan)) { @@ -327,37 +519,6 @@ } /** - * atc_cleanup_descriptors - cleanup up finished descriptors in active_list - * @atchan: channel to be cleaned up - * - * Called with atchan->lock held and bh disabled - */ -static void atc_cleanup_descriptors(struct at_dma_chan *atchan) -{ - struct at_desc *desc, *_desc; - struct at_desc *child; - - dev_vdbg(chan2dev(&atchan->chan_common), "cleanup descriptors\n"); - - list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { - if (!(desc->lli.ctrla & ATC_DONE)) - /* This one is currently in progress */ - return; - - list_for_each_entry(child, &desc->tx_list, desc_node) - if (!(child->lli.ctrla & ATC_DONE)) - /* Currently in progress */ - return; - - /* - * No descriptors so far seem to be in progress, i.e. - * this chain must be done. - */ - atc_chain_complete(atchan, desc); - } -} - -/** * atc_advance_work - at the end of a transaction, move forward * @atchan: channel where the transaction ended * @@ -544,6 +705,103 @@ } /** + * atc_prep_dma_interleaved - prepare memory to memory interleaved operation + * @chan: the channel to prepare operation on + * @xt: Interleaved transfer template + * @flags: tx descriptor status flags + */ +static struct dma_async_tx_descriptor * +atc_prep_dma_interleaved(struct dma_chan *chan, + struct dma_interleaved_template *xt, + unsigned long flags) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct data_chunk *first = xt->sgl; + struct at_desc *desc = NULL; + size_t xfer_count; + unsigned int dwidth; + u32 ctrla; + u32 ctrlb; + size_t len = 0; + int i; + + if (unlikely(!xt || xt->numf != 1 || !xt->frame_size)) + return NULL; + + dev_info(chan2dev(chan), + "%s: src=%pad, dest=%pad, numf=%d, frame_size=%d, flags=0x%lx\n", + __func__, &xt->src_start, &xt->dst_start, xt->numf, + xt->frame_size, flags); + + /* + * The controller can only "skip" X bytes every Y bytes, so we + * need to make sure we are given a template that fit that + * description, ie a template with chunks that always have the + * same size, with the same ICGs. + */ + for (i = 0; i < xt->frame_size; i++) { + struct data_chunk *chunk = xt->sgl + i; + + if ((chunk->size != xt->sgl->size) || + (dmaengine_get_dst_icg(xt, chunk) != dmaengine_get_dst_icg(xt, first)) || + (dmaengine_get_src_icg(xt, chunk) != dmaengine_get_src_icg(xt, first))) { + dev_err(chan2dev(chan), + "%s: the controller can transfer only identical chunks\n", + __func__); + return NULL; + } + + len += chunk->size; + } + + dwidth = atc_get_xfer_width(xt->src_start, + xt->dst_start, len); + + xfer_count = len >> dwidth; + if (xfer_count > ATC_BTSIZE_MAX) { + dev_err(chan2dev(chan), "%s: buffer is too big\n", __func__); + return NULL; + } + + ctrla = ATC_SRC_WIDTH(dwidth) | + ATC_DST_WIDTH(dwidth); + + ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN + | ATC_SRC_ADDR_MODE_INCR + | ATC_DST_ADDR_MODE_INCR + | ATC_SRC_PIP + | ATC_DST_PIP + | ATC_FC_MEM2MEM; + + /* create the transfer */ + desc = atc_desc_get(atchan); + if (!desc) { + dev_err(chan2dev(chan), + "%s: couldn't allocate our descriptor\n", __func__); + return NULL; + } + + desc->lli.saddr = xt->src_start; + desc->lli.daddr = xt->dst_start; + desc->lli.ctrla = ctrla | xfer_count; + desc->lli.ctrlb = ctrlb; + + desc->boundary = first->size >> dwidth; + desc->dst_hole = (dmaengine_get_dst_icg(xt, first) >> dwidth) + 1; + desc->src_hole = (dmaengine_get_src_icg(xt, first) >> dwidth) + 1; + + desc->txd.cookie = -EBUSY; + desc->total_len = desc->len = len; + + /* set end-of-link to the last link descriptor of list*/ + set_desc_eol(desc); + + desc->txd.flags = flags; /* client is in control of this ack */ + + return &desc->txd; +} + +/** * atc_prep_dma_memcpy - prepare a memcpy operation * @chan: the channel to prepare operation on * @dest: operation virtual destination address @@ -566,8 +824,8 @@ u32 ctrla; u32 ctrlb; - dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d0x%x s0x%x l0x%zx f0x%lx\n", - dest, src, len, flags); + dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n", + &dest, &src, len, flags); if (unlikely(!len)) { dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n"); @@ -583,16 +841,10 @@ * We can be a lot more clever here, but this should take care * of the most common optimization. */ - if (!((src | dest | len) & 3)) { - ctrla = ATC_SRC_WIDTH_WORD | ATC_DST_WIDTH_WORD; - src_width = dst_width = 2; - } else if (!((src | dest | len) & 1)) { - ctrla = ATC_SRC_WIDTH_HALFWORD | ATC_DST_WIDTH_HALFWORD; - src_width = dst_width = 1; - } else { - ctrla = ATC_SRC_WIDTH_BYTE | ATC_DST_WIDTH_BYTE; - src_width = dst_width = 0; - } + src_width = dst_width = atc_get_xfer_width(src, dest, len); + + ctrla = ATC_SRC_WIDTH(src_width) | + ATC_DST_WIDTH(dst_width); for (offset = 0; offset < len; offset += xfer_count << src_width) { xfer_count = min_t(size_t, (len - offset) >> src_width, @@ -608,13 +860,14 @@ desc->lli.ctrlb = ctrlb; desc->txd.cookie = 0; + desc->len = xfer_count << src_width; atc_desc_chain(&first, &prev, desc); } /* First descriptor of the chain embedds additional information */ first->txd.cookie = -EBUSY; - first->len = len; + first->total_len = len; /* set end-of-link to the last link descriptor of list*/ set_desc_eol(desc); @@ -628,6 +881,187 @@ return NULL; } +static struct at_desc *atc_create_memset_desc(struct dma_chan *chan, + dma_addr_t psrc, + dma_addr_t pdst, + size_t len) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_desc *desc; + size_t xfer_count; + + u32 ctrla = ATC_SRC_WIDTH(2) | ATC_DST_WIDTH(2); + u32 ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN | + ATC_SRC_ADDR_MODE_FIXED | + ATC_DST_ADDR_MODE_INCR | + ATC_FC_MEM2MEM; + + xfer_count = len >> 2; + if (xfer_count > ATC_BTSIZE_MAX) { + dev_err(chan2dev(chan), "%s: buffer is too big\n", + __func__); + return NULL; + } + + desc = atc_desc_get(atchan); + if (!desc) { + dev_err(chan2dev(chan), "%s: can't get a descriptor\n", + __func__); + return NULL; + } + + desc->lli.saddr = psrc; + desc->lli.daddr = pdst; + desc->lli.ctrla = ctrla | xfer_count; + desc->lli.ctrlb = ctrlb; + + desc->txd.cookie = 0; + desc->len = len; + + return desc; +} + +/** + * atc_prep_dma_memset - prepare a memcpy operation + * @chan: the channel to prepare operation on + * @dest: operation virtual destination address + * @value: value to set memory buffer to + * @len: operation length + * @flags: tx descriptor status flags + */ +static struct dma_async_tx_descriptor * +atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, + size_t len, unsigned long flags) +{ + struct at_dma *atdma = to_at_dma(chan->device); + struct at_desc *desc; + void __iomem *vaddr; + dma_addr_t paddr; + + dev_vdbg(chan2dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__, + &dest, value, len, flags); + + if (unlikely(!len)) { + dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__); + return NULL; + } + + if (!is_dma_fill_aligned(chan->device, dest, 0, len)) { + dev_dbg(chan2dev(chan), "%s: buffer is not aligned\n", + __func__); + return NULL; + } + + vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr); + if (!vaddr) { + dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", + __func__); + return NULL; + } + *(u32*)vaddr = value; + + desc = atc_create_memset_desc(chan, paddr, dest, len); + if (!desc) { + dev_err(chan2dev(chan), "%s: couldn't get a descriptor\n", + __func__); + goto err_free_buffer; + } + + desc->memset_paddr = paddr; + desc->memset_vaddr = vaddr; + desc->memset_buffer = true; + + desc->txd.cookie = -EBUSY; + desc->total_len = len; + + /* set end-of-link on the descriptor */ + set_desc_eol(desc); + + desc->txd.flags = flags; + + return &desc->txd; + +err_free_buffer: + dma_pool_free(atdma->memset_pool, vaddr, paddr); + return NULL; +} + +static struct dma_async_tx_descriptor * +atc_prep_dma_memset_sg(struct dma_chan *chan, + struct scatterlist *sgl, + unsigned int sg_len, int value, + unsigned long flags) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_dma *atdma = to_at_dma(chan->device); + struct at_desc *desc = NULL, *first = NULL, *prev = NULL; + struct scatterlist *sg; + void __iomem *vaddr; + dma_addr_t paddr; + size_t total_len = 0; + int i; + + dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%zx f0x%lx\n", __func__, + value, sg_len, flags); + + if (unlikely(!sgl || !sg_len)) { + dev_dbg(chan2dev(chan), "%s: scatterlist is empty!\n", + __func__); + return NULL; + } + + vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr); + if (!vaddr) { + dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", + __func__); + return NULL; + } + *(u32*)vaddr = value; + + for_each_sg(sgl, sg, sg_len, i) { + dma_addr_t dest = sg_dma_address(sg); + size_t len = sg_dma_len(sg); + + dev_vdbg(chan2dev(chan), "%s: d%pad, l0x%zx\n", + __func__, &dest, len); + + if (!is_dma_fill_aligned(chan->device, dest, 0, len)) { + dev_err(chan2dev(chan), "%s: buffer is not aligned\n", + __func__); + goto err_put_desc; + } + + desc = atc_create_memset_desc(chan, paddr, dest, len); + if (!desc) + goto err_put_desc; + + atc_desc_chain(&first, &prev, desc); + + total_len += len; + } + + /* + * Only set the buffer pointers on the last descriptor to + * avoid free'ing while we have our transfer still going + */ + desc->memset_paddr = paddr; + desc->memset_vaddr = vaddr; + desc->memset_buffer = true; + + first->txd.cookie = -EBUSY; + first->total_len = total_len; + + /* set end-of-link on the descriptor */ + set_desc_eol(desc); + + first->txd.flags = flags; + + return &first->txd; + +err_put_desc: + atc_desc_put(atchan, first); + return NULL; +} /** * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction @@ -706,6 +1140,7 @@ | ATC_SRC_WIDTH(mem_width) | len >> mem_width; desc->lli.ctrlb = ctrlb; + desc->len = len; atc_desc_chain(&first, &prev, desc); total_len += len; @@ -746,6 +1181,7 @@ | ATC_DST_WIDTH(mem_width) | len >> reg_width; desc->lli.ctrlb = ctrlb; + desc->len = len; atc_desc_chain(&first, &prev, desc); total_len += len; @@ -760,7 +1196,7 @@ /* First descriptor of the chain embedds additional information */ first->txd.cookie = -EBUSY; - first->len = total_len; + first->total_len = total_len; /* first link descriptor of list is responsible of flags */ first->txd.flags = flags; /* client is in control of this ack */ @@ -775,6 +1211,138 @@ } /** + * atc_prep_dma_sg - prepare memory to memory scather-gather operation + * @chan: the channel to prepare operation on + * @dst_sg: destination scatterlist + * @dst_nents: number of destination scatterlist entries + * @src_sg: source scatterlist + * @src_nents: number of source scatterlist entries + * @flags: tx descriptor status flags + */ +static struct dma_async_tx_descriptor * +atc_prep_dma_sg(struct dma_chan *chan, + struct scatterlist *dst_sg, unsigned int dst_nents, + struct scatterlist *src_sg, unsigned int src_nents, + unsigned long flags) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_desc *desc = NULL; + struct at_desc *first = NULL; + struct at_desc *prev = NULL; + unsigned int src_width; + unsigned int dst_width; + size_t xfer_count; + u32 ctrla; + u32 ctrlb; + size_t dst_len = 0, src_len = 0; + dma_addr_t dst = 0, src = 0; + size_t len = 0, total_len = 0; + + if (unlikely(dst_nents == 0 || src_nents == 0)) + return NULL; + + if (unlikely(dst_sg == NULL || src_sg == NULL)) + return NULL; + + ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN + | ATC_SRC_ADDR_MODE_INCR + | ATC_DST_ADDR_MODE_INCR + | ATC_FC_MEM2MEM; + + /* + * loop until there is either no more source or no more destination + * scatterlist entry + */ + while (true) { + + /* prepare the next transfer */ + if (dst_len == 0) { + + /* no more destination scatterlist entries */ + if (!dst_sg || !dst_nents) + break; + + dst = sg_dma_address(dst_sg); + dst_len = sg_dma_len(dst_sg); + + dst_sg = sg_next(dst_sg); + dst_nents--; + } + + if (src_len == 0) { + + /* no more source scatterlist entries */ + if (!src_sg || !src_nents) + break; + + src = sg_dma_address(src_sg); + src_len = sg_dma_len(src_sg); + + src_sg = sg_next(src_sg); + src_nents--; + } + + len = min_t(size_t, src_len, dst_len); + if (len == 0) + continue; + + /* take care for the alignment */ + src_width = dst_width = atc_get_xfer_width(src, dst, len); + + ctrla = ATC_SRC_WIDTH(src_width) | + ATC_DST_WIDTH(dst_width); + + /* + * The number of transfers to set up refer to the source width + * that depends on the alignment. + */ + xfer_count = len >> src_width; + if (xfer_count > ATC_BTSIZE_MAX) { + xfer_count = ATC_BTSIZE_MAX; + len = ATC_BTSIZE_MAX << src_width; + } + + /* create the transfer */ + desc = atc_desc_get(atchan); + if (!desc) + goto err_desc_get; + + desc->lli.saddr = src; + desc->lli.daddr = dst; + desc->lli.ctrla = ctrla | xfer_count; + desc->lli.ctrlb = ctrlb; + + desc->txd.cookie = 0; + desc->len = len; + + atc_desc_chain(&first, &prev, desc); + + /* update the lengths and addresses for the next loop cycle */ + dst_len -= len; + src_len -= len; + dst += len; + src += len; + + total_len += len; + } + + /* First descriptor of the chain embedds additional information */ + first->txd.cookie = -EBUSY; + first->total_len = total_len; + + /* set end-of-link to the last link descriptor of list*/ + set_desc_eol(desc); + + first->txd.flags = flags; /* client is in control of this ack */ + + return &first->txd; + +err_desc_get: + atc_desc_put(atchan, first); + return NULL; +} + +/** * atc_dma_cyclic_check_values * Check for too big/unaligned periods and unaligned DMA buffer */ @@ -825,6 +1393,7 @@ | ATC_FC_MEM2PER | ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if); + desc->len = period_len; break; case DMA_DEV_TO_MEM: @@ -836,6 +1405,7 @@ | ATC_FC_PER2MEM | ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if); + desc->len = period_len; break; default: @@ -853,12 +1423,11 @@ * @period_len: number of bytes for each period * @direction: transfer direction, to or from device * @flags: tx descriptor status flags - * @context: transfer context (ignored) */ static struct dma_async_tx_descriptor * atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, - unsigned long flags, void *context) + unsigned long flags) { struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma_slave *atslave = chan->private; @@ -870,9 +1439,9 @@ unsigned int periods = buf_len / period_len; unsigned int i; - dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@0x%08x - %d (%d/%d)\n", + dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%d/%d)\n", direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE", - buf_addr, + &buf_addr, periods, buf_len, period_len); if (unlikely(!atslave || !buf_len || !period_len)) { @@ -918,7 +1487,7 @@ /* First descriptor of the chain embedds additional information */ first->txd.cookie = -EBUSY; - first->len = buf_len; + first->total_len = buf_len; return &first->txd; @@ -930,11 +1499,13 @@ return NULL; } -static int set_runtime_config(struct dma_chan *chan, - struct dma_slave_config *sconfig) +static int atc_config(struct dma_chan *chan, + struct dma_slave_config *sconfig) { struct at_dma_chan *atchan = to_at_dma_chan(chan); + dev_vdbg(chan2dev(chan), "%s\n", __func__); + /* Check if it is chan is configured for slave transfers */ if (!chan->private) return -EINVAL; @@ -947,9 +1518,28 @@ return 0; } +static int atc_pause(struct dma_chan *chan) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_dma *atdma = to_at_dma(chan->device); + int chan_id = atchan->chan_common.chan_id; + unsigned long flags; + + LIST_HEAD(list); -static int atc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, - unsigned long arg) + dev_vdbg(chan2dev(chan), "%s\n", __func__); + + spin_lock_irqsave(&atchan->lock, flags); + + dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id)); + set_bit(ATC_IS_PAUSED, &atchan->status); + + spin_unlock_irqrestore(&atchan->lock, flags); + + return 0; +} + +static int atc_resume(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); @@ -958,60 +1548,61 @@ LIST_HEAD(list); - dev_vdbg(chan2dev(chan), "atc_control (%d)\n", cmd); + dev_vdbg(chan2dev(chan), "%s\n", __func__); - if (cmd == DMA_PAUSE) { - spin_lock_irqsave(&atchan->lock, flags); + if (!atc_chan_is_paused(atchan)) + return 0; - dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id)); - set_bit(ATC_IS_PAUSED, &atchan->status); + spin_lock_irqsave(&atchan->lock, flags); - spin_unlock_irqrestore(&atchan->lock, flags); - } else if (cmd == DMA_RESUME) { - if (!atc_chan_is_paused(atchan)) - return 0; + dma_writel(atdma, CHDR, AT_DMA_RES(chan_id)); + clear_bit(ATC_IS_PAUSED, &atchan->status); - spin_lock_irqsave(&atchan->lock, flags); + spin_unlock_irqrestore(&atchan->lock, flags); - dma_writel(atdma, CHDR, AT_DMA_RES(chan_id)); - clear_bit(ATC_IS_PAUSED, &atchan->status); + return 0; +} - spin_unlock_irqrestore(&atchan->lock, flags); - } else if (cmd == DMA_TERMINATE_ALL) { - struct at_desc *desc, *_desc; - /* - * This is only called when something went wrong elsewhere, so - * we don't really care about the data. Just disable the - * channel. We still have to poll the channel enable bit due - * to AHB/HSB limitations. - */ - spin_lock_irqsave(&atchan->lock, flags); +static int atc_terminate_all(struct dma_chan *chan) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_dma *atdma = to_at_dma(chan->device); + int chan_id = atchan->chan_common.chan_id; + struct at_desc *desc, *_desc; + unsigned long flags; - /* disabling channel: must also remove suspend state */ - dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); + LIST_HEAD(list); - /* confirm that this channel is disabled */ - while (dma_readl(atdma, CHSR) & atchan->mask) - cpu_relax(); - - /* active_list entries will end up before queued entries */ - list_splice_init(&atchan->queue, &list); - list_splice_init(&atchan->active_list, &list); - - /* Flush all pending and queued descriptors */ - list_for_each_entry_safe(desc, _desc, &list, desc_node) - atc_chain_complete(atchan, desc); - - clear_bit(ATC_IS_PAUSED, &atchan->status); - /* if channel dedicated to cyclic operations, free it */ - clear_bit(ATC_IS_CYCLIC, &atchan->status); + dev_vdbg(chan2dev(chan), "%s\n", __func__); - spin_unlock_irqrestore(&atchan->lock, flags); - } else if (cmd == DMA_SLAVE_CONFIG) { - return set_runtime_config(chan, (struct dma_slave_config *)arg); - } else { - return -ENXIO; - } + /* + * This is only called when something went wrong elsewhere, so + * we don't really care about the data. Just disable the + * channel. We still have to poll the channel enable bit due + * to AHB/HSB limitations. + */ + spin_lock_irqsave(&atchan->lock, flags); + + /* disabling channel: must also remove suspend state */ + dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); + + /* confirm that this channel is disabled */ + while (dma_readl(atdma, CHSR) & atchan->mask) + cpu_relax(); + + /* active_list entries will end up before queued entries */ + list_splice_init(&atchan->queue, &list); + list_splice_init(&atchan->active_list, &list); + + /* Flush all pending and queued descriptors */ + list_for_each_entry_safe(desc, _desc, &list, desc_node) + atc_chain_complete(atchan, desc); + + clear_bit(ATC_IS_PAUSED, &atchan->status); + /* if channel dedicated to cyclic operations, free it */ + clear_bit(ATC_IS_CYCLIC, &atchan->status); + + spin_unlock_irqrestore(&atchan->lock, flags); return 0; } @@ -1032,34 +1623,36 @@ struct dma_tx_state *txstate) { struct at_dma_chan *atchan = to_at_dma_chan(chan); - dma_cookie_t last_used; - dma_cookie_t last_complete; unsigned long flags; enum dma_status ret; - - spin_lock_irqsave(&atchan->lock, flags); + int bytes = 0; ret = dma_cookie_status(chan, cookie, txstate); - if (ret != DMA_SUCCESS) { - atc_cleanup_descriptors(atchan); + if (ret == DMA_COMPLETE) + return ret; + /* + * There's no point calculating the residue if there's + * no txstate to store the value. + */ + if (!txstate) + return DMA_ERROR; - ret = dma_cookie_status(chan, cookie, txstate); - } + spin_lock_irqsave(&atchan->lock, flags); - last_complete = chan->completed_cookie; - last_used = chan->cookie; + /* Get number of bytes left in the active transactions */ + bytes = atc_get_bytes_left(chan, cookie); spin_unlock_irqrestore(&atchan->lock, flags); - if (ret != DMA_SUCCESS) - dma_set_residue(txstate, atc_first_active(atchan)->len); - - if (atc_chan_is_paused(atchan)) - ret = DMA_PAUSED; + if (unlikely(bytes < 0)) { + dev_vdbg(chan2dev(chan), "get residual bytes error\n"); + return DMA_ERROR; + } else { + dma_set_residue(txstate, bytes); + } - dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d (d%d, u%d)\n", - ret, cookie, last_complete ? last_complete : 0, - last_used ? last_used : 0); + dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %d\n", + ret, cookie, bytes); return ret; } @@ -1120,7 +1713,7 @@ */ BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev); - /* if cfg configuration specified take it instad of default */ + /* if cfg configuration specified take it instead of default */ if (atslave->cfg) cfg = atslave->cfg; } @@ -1223,14 +1816,31 @@ atslave = devm_kzalloc(&dmac_pdev->dev, sizeof(*atslave), GFP_KERNEL); if (!atslave) return NULL; + + atslave->cfg = ATC_DST_H2SEL_HW | ATC_SRC_H2SEL_HW; /* * We can fill both SRC_PER and DST_PER, one of these fields will be * ignored depending on DMA transfer direction. */ - per_id = dma_spec->args[1]; - atslave->cfg = ATC_FIFOCFG_HALFFIFO | ATC_DST_H2SEL_HW - | ATC_SRC_H2SEL_HW | ATC_DST_PER(per_id) - | ATC_SRC_PER(per_id); + per_id = dma_spec->args[1] & AT91_DMA_CFG_PER_ID_MASK; + atslave->cfg |= ATC_DST_PER_MSB(per_id) | ATC_DST_PER(per_id) + | ATC_SRC_PER_MSB(per_id) | ATC_SRC_PER(per_id); + /* + * We have to translate the value we get from the device tree since + * the half FIFO configuration value had to be 0 to keep backward + * compatibility. + */ + switch (dma_spec->args[1] & AT91_DMA_CFG_FIFOCFG_MASK) { + case AT91_DMA_CFG_FIFOCFG_ALAP: + atslave->cfg |= ATC_FIFOCFG_LARGESTBURST; + break; + case AT91_DMA_CFG_FIFOCFG_ASAP: + atslave->cfg |= ATC_FIFOCFG_ENOUGHSPACE; + break; + case AT91_DMA_CFG_FIFOCFG_HALF: + default: + atslave->cfg |= ATC_FIFOCFG_HALFFIFO; + } atslave->dma_dev = &dmac_pdev->dev; chan = dma_request_channel(mask, at_dma_filter, atslave); @@ -1331,8 +1941,14 @@ /* setup platform data for each SoC */ dma_cap_set(DMA_MEMCPY, at91sam9rl_config.cap_mask); + dma_cap_set(DMA_SG, at91sam9rl_config.cap_mask); + dma_cap_set(DMA_INTERLEAVE, at91sam9g45_config.cap_mask); dma_cap_set(DMA_MEMCPY, at91sam9g45_config.cap_mask); + dma_cap_set(DMA_MEMSET, at91sam9g45_config.cap_mask); + dma_cap_set(DMA_MEMSET_SG, at91sam9g45_config.cap_mask); + dma_cap_set(DMA_PRIVATE, at91sam9g45_config.cap_mask); dma_cap_set(DMA_SLAVE, at91sam9g45_config.cap_mask); + dma_cap_set(DMA_SG, at91sam9g45_config.cap_mask); /* get DMA parameters from controller type */ plat_dat = at_dma_get_driver_data(pdev); @@ -1374,7 +1990,9 @@ err = PTR_ERR(atdma->clk); goto err_clk; } - clk_enable(atdma->clk); + err = clk_prepare_enable(atdma->clk); + if (err) + goto err_clk_prepare; /* force dma off, just in case */ at_dma_off(atdma); @@ -1392,7 +2010,16 @@ if (!atdma->dma_desc_pool) { dev_err(&pdev->dev, "No memory for descriptors dma pool\n"); err = -ENOMEM; - goto err_pool_create; + goto err_desc_pool_create; + } + + /* create a pool of consistent memory blocks for memset blocks */ + atdma->memset_pool = dma_pool_create("at_hdmac_memset_pool", + &pdev->dev, sizeof(int), 4, 0); + if (!atdma->memset_pool) { + dev_err(&pdev->dev, "No memory for memset dma pool\n"); + err = -ENOMEM; + goto err_memset_pool_create; } /* clear any pending interrupt */ @@ -1432,22 +2059,43 @@ atdma->dma_common.dev = &pdev->dev; /* set prep routines based on capability */ + if (dma_has_cap(DMA_INTERLEAVE, atdma->dma_common.cap_mask)) + atdma->dma_common.device_prep_interleaved_dma = atc_prep_dma_interleaved; + if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask)) atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy; + if (dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask)) { + atdma->dma_common.device_prep_dma_memset = atc_prep_dma_memset; + atdma->dma_common.device_prep_dma_memset_sg = atc_prep_dma_memset_sg; + atdma->dma_common.fill_align = DMAENGINE_ALIGN_4_BYTES; + } + if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) { atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg; /* controller can do slave DMA: can trigger cyclic transfers */ dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask); atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic; - atdma->dma_common.device_control = atc_control; + atdma->dma_common.device_config = atc_config; + atdma->dma_common.device_pause = atc_pause; + atdma->dma_common.device_resume = atc_resume; + atdma->dma_common.device_terminate_all = atc_terminate_all; + atdma->dma_common.src_addr_widths = ATC_DMA_BUSWIDTHS; + atdma->dma_common.dst_addr_widths = ATC_DMA_BUSWIDTHS; + atdma->dma_common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; } + if (dma_has_cap(DMA_SG, atdma->dma_common.cap_mask)) + atdma->dma_common.device_prep_dma_sg = atc_prep_dma_sg; + dma_writel(atdma, EN, AT_DMA_ENABLE); - dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s), %d channels\n", + dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s%s), %d channels\n", dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "", + dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "", dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "", + dma_has_cap(DMA_SG, atdma->dma_common.cap_mask) ? "sg-cpy " : "", plat_dat->nr_channels); dma_async_device_register(&atdma->dma_common); @@ -1470,12 +2118,14 @@ err_of_dma_controller_register: dma_async_device_unregister(&atdma->dma_common); + dma_pool_destroy(atdma->memset_pool); +err_memset_pool_create: dma_pool_destroy(atdma->dma_desc_pool); -err_pool_create: - platform_set_drvdata(pdev, NULL); +err_desc_pool_create: free_irq(platform_get_irq(pdev, 0), atdma); err_irq: - clk_disable(atdma->clk); + clk_disable_unprepare(atdma->clk); +err_clk_prepare: clk_put(atdma->clk); err_clk: iounmap(atdma->regs); @@ -1496,8 +2146,8 @@ at_dma_off(atdma); dma_async_device_unregister(&atdma->dma_common); + dma_pool_destroy(atdma->memset_pool); dma_pool_destroy(atdma->dma_desc_pool); - platform_set_drvdata(pdev, NULL); free_irq(platform_get_irq(pdev, 0), atdma); list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, @@ -1506,13 +2156,12 @@ /* Disable interrupts */ atc_disable_chan_irq(atdma, chan->chan_id); - tasklet_disable(&atchan->tasklet); tasklet_kill(&atchan->tasklet); list_del(&chan->device_node); } - clk_disable(atdma->clk); + clk_disable_unprepare(atdma->clk); clk_put(atdma->clk); iounmap(atdma->regs); @@ -1531,7 +2180,7 @@ struct at_dma *atdma = platform_get_drvdata(pdev); at_dma_off(platform_get_drvdata(pdev)); - clk_disable(atdma->clk); + clk_disable_unprepare(atdma->clk); } static int at_dma_prepare(struct device *dev) @@ -1559,7 +2208,7 @@ if (!atc_chan_is_paused(atchan)) { dev_warn(chan2dev(chan), "cyclic channel not paused, should be done by channel user\n"); - atc_control(chan, DMA_PAUSE, 0); + atc_pause(chan); } /* now preserve additional data for cyclic operations */ @@ -1588,7 +2237,7 @@ /* disable DMA controller */ at_dma_off(atdma); - clk_disable(atdma->clk); + clk_disable_unprepare(atdma->clk); return 0; } @@ -1618,7 +2267,7 @@ struct dma_chan *chan, *_chan; /* bring back DMA controller */ - clk_enable(atdma->clk); + clk_prepare_enable(atdma->clk); dma_writel(atdma, EN, AT_DMA_ENABLE); /* clear any pending interrupt */