--- zzzz-none-000/linux-3.10.107/drivers/dma/imx-sdma.c 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/drivers/dma/imx-sdma.c 2021-02-04 17:41:59.000000000 +0000 @@ -35,11 +35,16 @@ #include #include #include +#include #include +#include #include #include #include +#include +#include +#include #include "dmaengine.h" @@ -123,6 +128,56 @@ #define CHANGE_ENDIANNESS 0x80 /* + * p_2_p watermark_level description + * Bits Name Description + * 0-7 Lower WML Lower watermark level + * 8 PS 1: Pad Swallowing + * 0: No Pad Swallowing + * 9 PA 1: Pad Adding + * 0: No Pad Adding + * 10 SPDIF If this bit is set both source + * and destination are on SPBA + * 11 Source Bit(SP) 1: Source on SPBA + * 0: Source on AIPS + * 12 Destination Bit(DP) 1: Destination on SPBA + * 0: Destination on AIPS + * 13-15 --------- MUST BE 0 + * 16-23 Higher WML HWML + * 24-27 N Total number of samples after + * which Pad adding/Swallowing + * must be done. It must be odd. + * 28 Lower WML Event(LWE) SDMA events reg to check for + * LWML event mask + * 0: LWE in EVENTS register + * 1: LWE in EVENTS2 register + * 29 Higher WML Event(HWE) SDMA events reg to check for + * HWML event mask + * 0: HWE in EVENTS register + * 1: HWE in EVENTS2 register + * 30 --------- MUST BE 0 + * 31 CONT 1: Amount of samples to be + * transferred is unknown and + * script will keep on + * transferring samples as long as + * both events are detected and + * script must be manually stopped + * by the application + * 0: The amount of samples to be + * transferred is equal to the + * count field of mode word + */ +#define SDMA_WATERMARK_LEVEL_LWML 0xFF +#define SDMA_WATERMARK_LEVEL_PS BIT(8) +#define SDMA_WATERMARK_LEVEL_PA BIT(9) +#define SDMA_WATERMARK_LEVEL_SPDIF BIT(10) +#define SDMA_WATERMARK_LEVEL_SP BIT(11) +#define SDMA_WATERMARK_LEVEL_DP BIT(12) +#define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16) +#define SDMA_WATERMARK_LEVEL_LWE BIT(28) +#define SDMA_WATERMARK_LEVEL_HWE BIT(29) +#define SDMA_WATERMARK_LEVEL_CONT BIT(31) + +/* * Mode/Count of data node descriptors - IPCv2 */ struct sdma_mode_count { @@ -242,7 +297,6 @@ * @event_id1 for channels that use 2 events * @word_size peripheral access size * @buf_tail ID of the buffer that was processed - * @done channel completion * @num_bd max NUM_BD. number of descriptors currently handling */ struct sdma_channel { @@ -254,13 +308,14 @@ unsigned int event_id1; enum dma_slave_buswidth word_size; unsigned int buf_tail; - struct completion done; unsigned int num_bd; + unsigned int period_len; struct sdma_buffer_descriptor *bd; dma_addr_t bd_phys; unsigned int pc_from_device, pc_to_device; + unsigned int device_to_device; unsigned long flags; - dma_addr_t per_address; + dma_addr_t per_address, per_address2; unsigned long event_mask[2]; unsigned long watermark_level; u32 shp_addr, per_addr; @@ -271,6 +326,7 @@ unsigned int chn_count; unsigned int chn_real_count; struct tasklet_struct tasklet; + struct imx_dma_data data; }; #define IMX_DMA_SG_LOOP BIT(0) @@ -306,9 +362,10 @@ u32 ram_code_size; }; -enum sdma_devtype { - IMX31_SDMA, /* runs on i.mx31 */ - IMX35_SDMA, /* runs on i.mx35 and later */ +struct sdma_driver_data { + int chnenbl0; + int num_events; + struct sdma_script_start_addrs *script_addrs; }; struct sdma_engine { @@ -317,24 +374,128 @@ struct sdma_channel channel[MAX_DMA_CHANNELS]; struct sdma_channel_control *channel_control; void __iomem *regs; - enum sdma_devtype devtype; - unsigned int num_events; struct sdma_context_data *context; dma_addr_t context_phys; struct dma_device dma_device; struct clk *clk_ipg; struct clk *clk_ahb; spinlock_t channel_0_lock; + u32 script_number; struct sdma_script_start_addrs *script_addrs; + const struct sdma_driver_data *drvdata; + u32 spba_start_addr; + u32 spba_end_addr; +}; + +static struct sdma_driver_data sdma_imx31 = { + .chnenbl0 = SDMA_CHNENBL0_IMX31, + .num_events = 32, +}; + +static struct sdma_script_start_addrs sdma_script_imx25 = { + .ap_2_ap_addr = 729, + .uart_2_mcu_addr = 904, + .per_2_app_addr = 1255, + .mcu_2_app_addr = 834, + .uartsh_2_mcu_addr = 1120, + .per_2_shp_addr = 1329, + .mcu_2_shp_addr = 1048, + .ata_2_mcu_addr = 1560, + .mcu_2_ata_addr = 1479, + .app_2_per_addr = 1189, + .app_2_mcu_addr = 770, + .shp_2_per_addr = 1407, + .shp_2_mcu_addr = 979, +}; + +static struct sdma_driver_data sdma_imx25 = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx25, +}; + +static struct sdma_driver_data sdma_imx35 = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, +}; + +static struct sdma_script_start_addrs sdma_script_imx51 = { + .ap_2_ap_addr = 642, + .uart_2_mcu_addr = 817, + .mcu_2_app_addr = 747, + .mcu_2_shp_addr = 961, + .ata_2_mcu_addr = 1473, + .mcu_2_ata_addr = 1392, + .app_2_per_addr = 1033, + .app_2_mcu_addr = 683, + .shp_2_per_addr = 1251, + .shp_2_mcu_addr = 892, +}; + +static struct sdma_driver_data sdma_imx51 = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx51, +}; + +static struct sdma_script_start_addrs sdma_script_imx53 = { + .ap_2_ap_addr = 642, + .app_2_mcu_addr = 683, + .mcu_2_app_addr = 747, + .uart_2_mcu_addr = 817, + .shp_2_mcu_addr = 891, + .mcu_2_shp_addr = 960, + .uartsh_2_mcu_addr = 1032, + .spdif_2_mcu_addr = 1100, + .mcu_2_spdif_addr = 1134, + .firi_2_mcu_addr = 1193, + .mcu_2_firi_addr = 1290, +}; + +static struct sdma_driver_data sdma_imx53 = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx53, }; -static struct platform_device_id sdma_devtypes[] = { +static struct sdma_script_start_addrs sdma_script_imx6q = { + .ap_2_ap_addr = 642, + .uart_2_mcu_addr = 817, + .mcu_2_app_addr = 747, + .per_2_per_addr = 6331, + .uartsh_2_mcu_addr = 1032, + .mcu_2_shp_addr = 960, + .app_2_mcu_addr = 683, + .shp_2_mcu_addr = 891, + .spdif_2_mcu_addr = 1100, + .mcu_2_spdif_addr = 1134, +}; + +static struct sdma_driver_data sdma_imx6q = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx6q, +}; + +static const struct platform_device_id sdma_devtypes[] = { { + .name = "imx25-sdma", + .driver_data = (unsigned long)&sdma_imx25, + }, { .name = "imx31-sdma", - .driver_data = IMX31_SDMA, + .driver_data = (unsigned long)&sdma_imx31, }, { .name = "imx35-sdma", - .driver_data = IMX35_SDMA, + .driver_data = (unsigned long)&sdma_imx35, + }, { + .name = "imx51-sdma", + .driver_data = (unsigned long)&sdma_imx51, + }, { + .name = "imx53-sdma", + .driver_data = (unsigned long)&sdma_imx53, + }, { + .name = "imx6q-sdma", + .driver_data = (unsigned long)&sdma_imx6q, }, { /* sentinel */ } @@ -342,8 +503,12 @@ MODULE_DEVICE_TABLE(platform, sdma_devtypes); static const struct of_device_id sdma_dt_ids[] = { - { .compatible = "fsl,imx31-sdma", .data = &sdma_devtypes[IMX31_SDMA], }, - { .compatible = "fsl,imx35-sdma", .data = &sdma_devtypes[IMX35_SDMA], }, + { .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, }, + { .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, }, + { .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, }, + { .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, }, + { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, }, + { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, sdma_dt_ids); @@ -355,8 +520,7 @@ static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event) { - u32 chnenbl0 = (sdma->devtype == IMX31_SDMA ? SDMA_CHNENBL0_IMX31 : - SDMA_CHNENBL0_IMX35); + u32 chnenbl0 = sdma->drvdata->chnenbl0; return chnenbl0 + event * 4; } @@ -424,6 +588,10 @@ dev_err(sdma->dev, "Timeout waiting for CH0 ready\n"); } + /* Set bits of CONFIG register with dynamic context switching */ + if (readl(sdma->regs + SDMA_H_CONFIG) == 0) + writel_relaxed(SDMA_H_CONFIG_CSM, sdma->regs + SDMA_H_CONFIG); + return ret ? 0 : -ETIMEDOUT; } @@ -488,6 +656,12 @@ static void sdma_handle_channel_loop(struct sdma_channel *sdmac) { + if (sdmac->desc.callback) + sdmac->desc.callback(sdmac->desc.callback_param); +} + +static void sdma_update_channel_loop(struct sdma_channel *sdmac) +{ struct sdma_buffer_descriptor *bd; /* @@ -502,15 +676,10 @@ if (bd->mode.status & BD_RROR) sdmac->status = DMA_ERROR; - else - sdmac->status = DMA_IN_PROGRESS; bd->mode.status |= BD_DONE; sdmac->buf_tail++; sdmac->buf_tail %= sdmac->num_bd; - - if (sdmac->desc.callback) - sdmac->desc.callback(sdmac->desc.callback_param); } } @@ -535,7 +704,7 @@ if (error) sdmac->status = DMA_ERROR; else - sdmac->status = DMA_SUCCESS; + sdmac->status = DMA_COMPLETE; dma_cookie_complete(&sdmac->desc); if (sdmac->desc.callback) @@ -546,8 +715,6 @@ { struct sdma_channel *sdmac = (struct sdma_channel *) data; - complete(&sdmac->done); - if (sdmac->flags & IMX_DMA_SG_LOOP) sdma_handle_channel_loop(sdmac); else @@ -568,6 +735,9 @@ int channel = fls(stat) - 1; struct sdma_channel *sdmac = &sdma->channel[channel]; + if (sdmac->flags & IMX_DMA_SG_LOOP) + sdma_update_channel_loop(sdmac); + tasklet_schedule(&sdmac->tasklet); __clear_bit(channel, &stat); @@ -592,6 +762,7 @@ sdmac->pc_from_device = 0; sdmac->pc_to_device = 0; + sdmac->device_to_device = 0; switch (peripheral_type) { case IMX_DMATYPE_MEMORY: @@ -620,9 +791,14 @@ case IMX_DMATYPE_CSPI: case IMX_DMATYPE_EXT: case IMX_DMATYPE_SSI: + case IMX_DMATYPE_SAI: per_2_emi = sdma->script_addrs->app_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_app_addr; break; + case IMX_DMATYPE_SSI_DUAL: + per_2_emi = sdma->script_addrs->ssish_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_ssish_addr; + break; case IMX_DMATYPE_SSI_SP: case IMX_DMATYPE_MMC: case IMX_DMATYPE_SDHC: @@ -637,6 +813,11 @@ emi_2_per = sdma->script_addrs->asrc_2_mcu_addr; per_2_per = sdma->script_addrs->per_2_per_addr; break; + case IMX_DMATYPE_ASRC_SP: + per_2_emi = sdma->script_addrs->shp_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_shp_addr; + per_2_per = sdma->script_addrs->per_2_per_addr; + break; case IMX_DMATYPE_MSHC: per_2_emi = sdma->script_addrs->mshc_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_mshc_addr; @@ -657,6 +838,7 @@ sdmac->pc_from_device = per_2_emi; sdmac->pc_to_device = emi_2_per; + sdmac->device_to_device = per_2_per; } static int sdma_load_context(struct sdma_channel *sdmac) @@ -669,11 +851,12 @@ int ret; unsigned long flags; - if (sdmac->direction == DMA_DEV_TO_MEM) { + if (sdmac->direction == DMA_DEV_TO_MEM) load_address = sdmac->pc_from_device; - } else { + else if (sdmac->direction == DMA_DEV_TO_DEV) + load_address = sdmac->device_to_device; + else load_address = sdmac->pc_to_device; - } if (load_address < 0) return load_address; @@ -711,20 +894,69 @@ return ret; } -static void sdma_disable_channel(struct sdma_channel *sdmac) +static struct sdma_channel *to_sdma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct sdma_channel, chan); +} + +static int sdma_disable_channel(struct dma_chan *chan) { + struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; int channel = sdmac->channel; writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP); sdmac->status = DMA_ERROR; + + return 0; +} + +static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) +{ + struct sdma_engine *sdma = sdmac->sdma; + + int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML; + int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16; + + set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]); + set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]); + + if (sdmac->event_id0 > 31) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE; + + if (sdmac->event_id1 > 31) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE; + + /* + * If LWML(src_maxburst) > HWML(dst_maxburst), we need + * swap LWML and HWML of INFO(A.3.2.5.1), also need swap + * r0(event_mask[1]) and r1(event_mask[0]). + */ + if (lwml > hwml) { + sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML | + SDMA_WATERMARK_LEVEL_HWML); + sdmac->watermark_level |= hwml; + sdmac->watermark_level |= lwml << 16; + swap(sdmac->event_mask[0], sdmac->event_mask[1]); + } + + if (sdmac->per_address2 >= sdma->spba_start_addr && + sdmac->per_address2 <= sdma->spba_end_addr) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP; + + if (sdmac->per_address >= sdma->spba_start_addr && + sdmac->per_address <= sdma->spba_end_addr) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP; + + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT; } -static int sdma_config_channel(struct sdma_channel *sdmac) +static int sdma_config_channel(struct dma_chan *chan) { + struct sdma_channel *sdmac = to_sdma_chan(chan); int ret; - sdma_disable_channel(sdmac); + sdma_disable_channel(chan); sdmac->event_mask[0] = 0; sdmac->event_mask[1] = 0; @@ -732,11 +964,17 @@ sdmac->per_addr = 0; if (sdmac->event_id0) { - if (sdmac->event_id0 >= sdmac->sdma->num_events) + if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) return -EINVAL; sdma_event_enable(sdmac, sdmac->event_id0); } + if (sdmac->event_id1) { + if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) + return -EINVAL; + sdma_event_enable(sdmac, sdmac->event_id1); + } + switch (sdmac->peripheral_type) { case IMX_DMATYPE_DSP: sdma_config_ownership(sdmac, false, true, true); @@ -755,19 +993,17 @@ (sdmac->peripheral_type != IMX_DMATYPE_DSP)) { /* Handle multiple event channels differently */ if (sdmac->event_id1) { - sdmac->event_mask[1] = BIT(sdmac->event_id1 % 32); - if (sdmac->event_id1 > 31) - __set_bit(31, &sdmac->watermark_level); - sdmac->event_mask[0] = BIT(sdmac->event_id0 % 32); - if (sdmac->event_id0 > 31) - __set_bit(30, &sdmac->watermark_level); - } else { + if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP || + sdmac->peripheral_type == IMX_DMATYPE_ASRC) + sdma_set_watermarklevel_for_p2p(sdmac); + } else __set_bit(sdmac->event_id0, sdmac->event_mask); - } + /* Watermark Level */ sdmac->watermark_level |= sdmac->watermark_level; /* Address */ sdmac->shp_addr = sdmac->per_address; + sdmac->per_addr = sdmac->per_address2; } else { sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */ } @@ -799,32 +1035,23 @@ int channel = sdmac->channel; int ret = -EBUSY; - sdmac->bd = dma_alloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys, GFP_KERNEL); + sdmac->bd = dma_zalloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys, + GFP_KERNEL); if (!sdmac->bd) { ret = -ENOMEM; goto out; } - memset(sdmac->bd, 0, PAGE_SIZE); - sdma->channel_control[channel].base_bd_ptr = sdmac->bd_phys; sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys; sdma_set_channel_priority(sdmac, MXC_SDMA_DEFAULT_PRIORITY); - - init_completion(&sdmac->done); - return 0; out: return ret; } -static struct sdma_channel *to_sdma_chan(struct dma_chan *chan) -{ - return container_of(chan, struct sdma_channel, chan); -} - static dma_cookie_t sdma_tx_submit(struct dma_async_tx_descriptor *tx) { unsigned long flags; @@ -864,17 +1091,22 @@ sdmac->peripheral_type = data->peripheral_type; sdmac->event_id0 = data->dma_request; + sdmac->event_id1 = data->dma_request2; - clk_enable(sdmac->sdma->clk_ipg); - clk_enable(sdmac->sdma->clk_ahb); + ret = clk_enable(sdmac->sdma->clk_ipg); + if (ret) + return ret; + ret = clk_enable(sdmac->sdma->clk_ahb); + if (ret) + goto disable_clk_ipg; ret = sdma_request_channel(sdmac); if (ret) - return ret; + goto disable_clk_ahb; ret = sdma_set_channel_priority(sdmac, prio); if (ret) - return ret; + goto disable_clk_ahb; dma_async_tx_descriptor_init(&sdmac->desc, chan); sdmac->desc.tx_submit = sdma_tx_submit; @@ -882,6 +1114,12 @@ sdmac->desc.flags = DMA_CTRL_ACK; return 0; + +disable_clk_ahb: + clk_disable(sdmac->sdma->clk_ahb); +disable_clk_ipg: + clk_disable(sdmac->sdma->clk_ipg); + return ret; } static void sdma_free_chan_resources(struct dma_chan *chan) @@ -889,7 +1127,7 @@ struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; - sdma_disable_channel(sdmac); + sdma_disable_channel(chan); if (sdmac->event_id0) sdma_event_disable(sdmac, sdmac->event_id0); @@ -991,8 +1229,8 @@ param &= ~BD_CONT; } - dev_dbg(sdma->dev, "entry %d: count: %d dma: 0x%08x %s%s\n", - i, count, sg->dma_address, + dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n", + i, count, (u64)sg->dma_address, param & BD_WRAP ? "wrap" : "", param & BD_INTR ? " intr" : ""); @@ -1011,7 +1249,7 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, - unsigned long flags, void *context) + unsigned long flags) { struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; @@ -1027,6 +1265,7 @@ sdmac->status = DMA_IN_PROGRESS; sdmac->buf_tail = 0; + sdmac->period_len = period_len; sdmac->flags |= IMX_DMA_SG_LOOP; sdmac->direction = direction; @@ -1065,8 +1304,8 @@ if (i + 1 == num_periods) param |= BD_WRAP; - dev_dbg(sdma->dev, "entry %d: count: %d dma: 0x%08x %s%s\n", - i, period_len, dma_addr, + dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n", + i, period_len, (u64)dma_addr, param & BD_WRAP ? "wrap" : "", param & BD_INTR ? " intr" : ""); @@ -1087,48 +1326,48 @@ return NULL; } -static int sdma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, - unsigned long arg) +static int sdma_config(struct dma_chan *chan, + struct dma_slave_config *dmaengine_cfg) { struct sdma_channel *sdmac = to_sdma_chan(chan); - struct dma_slave_config *dmaengine_cfg = (void *)arg; - switch (cmd) { - case DMA_TERMINATE_ALL: - sdma_disable_channel(sdmac); - return 0; - case DMA_SLAVE_CONFIG: - if (dmaengine_cfg->direction == DMA_DEV_TO_MEM) { - sdmac->per_address = dmaengine_cfg->src_addr; - sdmac->watermark_level = dmaengine_cfg->src_maxburst * - dmaengine_cfg->src_addr_width; - sdmac->word_size = dmaengine_cfg->src_addr_width; - } else { - sdmac->per_address = dmaengine_cfg->dst_addr; - sdmac->watermark_level = dmaengine_cfg->dst_maxburst * - dmaengine_cfg->dst_addr_width; - sdmac->word_size = dmaengine_cfg->dst_addr_width; - } - sdmac->direction = dmaengine_cfg->direction; - return sdma_config_channel(sdmac); - default: - return -ENOSYS; + if (dmaengine_cfg->direction == DMA_DEV_TO_MEM) { + sdmac->per_address = dmaengine_cfg->src_addr; + sdmac->watermark_level = dmaengine_cfg->src_maxburst * + dmaengine_cfg->src_addr_width; + sdmac->word_size = dmaengine_cfg->src_addr_width; + } else if (dmaengine_cfg->direction == DMA_DEV_TO_DEV) { + sdmac->per_address2 = dmaengine_cfg->src_addr; + sdmac->per_address = dmaengine_cfg->dst_addr; + sdmac->watermark_level = dmaengine_cfg->src_maxburst & + SDMA_WATERMARK_LEVEL_LWML; + sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) & + SDMA_WATERMARK_LEVEL_HWML; + sdmac->word_size = dmaengine_cfg->dst_addr_width; + } else { + sdmac->per_address = dmaengine_cfg->dst_addr; + sdmac->watermark_level = dmaengine_cfg->dst_maxburst * + dmaengine_cfg->dst_addr_width; + sdmac->word_size = dmaengine_cfg->dst_addr_width; } - - return -EINVAL; + sdmac->direction = dmaengine_cfg->direction; + return sdma_config_channel(chan); } static enum dma_status sdma_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, - struct dma_tx_state *txstate) + dma_cookie_t cookie, + struct dma_tx_state *txstate) { struct sdma_channel *sdmac = to_sdma_chan(chan); - dma_cookie_t last_used; + u32 residue; - last_used = chan->cookie; + if (sdmac->flags & IMX_DMA_SG_LOOP) + residue = (sdmac->num_bd - sdmac->buf_tail) * sdmac->period_len; + else + residue = sdmac->chn_count - sdmac->chn_real_count; - dma_set_tx_state(txstate, chan->completed_cookie, last_used, - sdmac->chn_count - sdmac->chn_real_count); + dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie, + residue); return sdmac->status; } @@ -1143,6 +1382,8 @@ } #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 41 static void sdma_add_scripts(struct sdma_engine *sdma, const struct sdma_script_start_addrs *addr) @@ -1151,7 +1392,11 @@ s32 *saddr_arr = (u32 *)sdma->script_addrs; int i; - for (i = 0; i < SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; i++) + /* use the default firmware in ROM if missing external firmware */ + if (!sdma->script_number) + sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; + + for (i = 0; i < sdma->script_number; i++) if (addr_arr[i] > 0) saddr_arr[i] = addr_arr[i]; } @@ -1164,7 +1409,8 @@ unsigned short *ram_code; if (!fw) { - dev_err(sdma->dev, "firmware not found\n"); + dev_info(sdma->dev, "external firmware not found, using ROM firmware\n"); + /* In this case we just use the ROM firmware. */ return; } @@ -1177,6 +1423,20 @@ goto err_firmware; if (header->ram_code_start + header->ram_code_size > fw->size) goto err_firmware; + switch (header->version_major) { + case 1: + sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; + break; + case 2: + sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2; + break; + case 3: + sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3; + break; + default: + dev_err(sdma->dev, "unknown firmware version\n"); + goto err_firmware; + } addr = (void *)header + header->script_addrs_start; ram_code = (void *)header + header->ram_code_start; @@ -1200,7 +1460,73 @@ release_firmware(fw); } -static int __init sdma_get_firmware(struct sdma_engine *sdma, +#define EVENT_REMAP_CELLS 3 + +static int sdma_event_remap(struct sdma_engine *sdma) +{ + struct device_node *np = sdma->dev->of_node; + struct device_node *gpr_np = of_parse_phandle(np, "gpr", 0); + struct property *event_remap; + struct regmap *gpr; + char propname[] = "fsl,sdma-event-remap"; + u32 reg, val, shift, num_map, i; + int ret = 0; + + if (IS_ERR(np) || IS_ERR(gpr_np)) + goto out; + + event_remap = of_find_property(np, propname, NULL); + num_map = event_remap ? (event_remap->length / sizeof(u32)) : 0; + if (!num_map) { + dev_dbg(sdma->dev, "no event needs to be remapped\n"); + goto out; + } else if (num_map % EVENT_REMAP_CELLS) { + dev_err(sdma->dev, "the property %s must modulo %d\n", + propname, EVENT_REMAP_CELLS); + ret = -EINVAL; + goto out; + } + + gpr = syscon_node_to_regmap(gpr_np); + if (IS_ERR(gpr)) { + dev_err(sdma->dev, "failed to get gpr regmap\n"); + ret = PTR_ERR(gpr); + goto out; + } + + for (i = 0; i < num_map; i += EVENT_REMAP_CELLS) { + ret = of_property_read_u32_index(np, propname, i, ®); + if (ret) { + dev_err(sdma->dev, "failed to read property %s index %d\n", + propname, i); + goto out; + } + + ret = of_property_read_u32_index(np, propname, i + 1, &shift); + if (ret) { + dev_err(sdma->dev, "failed to read property %s index %d\n", + propname, i + 1); + goto out; + } + + ret = of_property_read_u32_index(np, propname, i + 2, &val); + if (ret) { + dev_err(sdma->dev, "failed to read property %s index %d\n", + propname, i + 2); + goto out; + } + + regmap_update_bits(gpr, reg, BIT(shift), val << shift); + } + +out: + if (!IS_ERR(gpr_np)) + of_node_put(gpr_np); + + return ret; +} + +static int sdma_get_firmware(struct sdma_engine *sdma, const char *fw_name) { int ret; @@ -1212,26 +1538,17 @@ return ret; } -static int __init sdma_init(struct sdma_engine *sdma) +static int sdma_init(struct sdma_engine *sdma) { int i, ret; dma_addr_t ccb_phys; - switch (sdma->devtype) { - case IMX31_SDMA: - sdma->num_events = 32; - break; - case IMX35_SDMA: - sdma->num_events = 48; - break; - default: - dev_err(sdma->dev, "Unknown sdma type %d. aborting\n", - sdma->devtype); - return -ENODEV; - } - - clk_enable(sdma->clk_ipg); - clk_enable(sdma->clk_ahb); + ret = clk_enable(sdma->clk_ipg); + if (ret) + return ret; + ret = clk_enable(sdma->clk_ahb); + if (ret) + goto disable_clk_ipg; /* Be sure SDMA has not started yet */ writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); @@ -1256,7 +1573,7 @@ MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control)); /* disable all channels */ - for (i = 0; i < sdma->num_events; i++) + for (i = 0; i < sdma->drvdata->num_events; i++) writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i)); /* All channels have priority 0 */ @@ -1278,9 +1595,6 @@ writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR); - /* Set bits of CONFIG register with given context switching mode */ - writel_relaxed(SDMA_H_CONFIG_CSM, sdma->regs + SDMA_H_CONFIG); - /* Initializes channel's priorities */ sdma_set_channel_priority(&sdma->channel[0], 7); @@ -1290,86 +1604,126 @@ return 0; err_dma_alloc: - clk_disable(sdma->clk_ipg); clk_disable(sdma->clk_ahb); +disable_clk_ipg: + clk_disable(sdma->clk_ipg); dev_err(sdma->dev, "initialisation failed with %d\n", ret); return ret; } -static int __init sdma_probe(struct platform_device *pdev) +static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param) +{ + struct sdma_channel *sdmac = to_sdma_chan(chan); + struct imx_dma_data *data = fn_param; + + if (!imx_dma_is_general_purpose(chan)) + return false; + + sdmac->data = *data; + chan->private = &sdmac->data; + + return true; +} + +static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct sdma_engine *sdma = ofdma->of_dma_data; + dma_cap_mask_t mask = sdma->dma_device.cap_mask; + struct imx_dma_data data; + + if (dma_spec->args_count != 3) + return NULL; + + data.dma_request = dma_spec->args[0]; + data.peripheral_type = dma_spec->args[1]; + data.priority = dma_spec->args[2]; + /* + * init dma_request2 to zero, which is not used by the dts. + * For P2P, dma_request2 is init from dma_request_channel(), + * chan->private will point to the imx_dma_data, and in + * device_alloc_chan_resources(), imx_dma_data.dma_request2 will + * be set to sdmac->event_id1. + */ + data.dma_request2 = 0; + + return dma_request_channel(mask, sdma_filter_fn, &data); +} + +static int sdma_probe(struct platform_device *pdev) { const struct of_device_id *of_id = of_match_device(sdma_dt_ids, &pdev->dev); struct device_node *np = pdev->dev.of_node; + struct device_node *spba_bus; const char *fw_name; int ret; int irq; struct resource *iores; - struct sdma_platform_data *pdata = pdev->dev.platform_data; + struct resource spba_res; + struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev); int i; struct sdma_engine *sdma; s32 *saddr_arr; + const struct sdma_driver_data *drvdata = NULL; - sdma = kzalloc(sizeof(*sdma), GFP_KERNEL); + if (of_id) + drvdata = of_id->data; + else if (pdev->id_entry) + drvdata = (void *)pdev->id_entry->driver_data; + + if (!drvdata) { + dev_err(&pdev->dev, "unable to find driver data\n"); + return -EINVAL; + } + + ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + + sdma = devm_kzalloc(&pdev->dev, sizeof(*sdma), GFP_KERNEL); if (!sdma) return -ENOMEM; spin_lock_init(&sdma->channel_0_lock); sdma->dev = &pdev->dev; + sdma->drvdata = drvdata; - iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); irq = platform_get_irq(pdev, 0); - if (!iores || irq < 0) { - ret = -EINVAL; - goto err_irq; - } + if (irq < 0) + return irq; - if (!request_mem_region(iores->start, resource_size(iores), pdev->name)) { - ret = -EBUSY; - goto err_request_region; - } + iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); + sdma->regs = devm_ioremap_resource(&pdev->dev, iores); + if (IS_ERR(sdma->regs)) + return PTR_ERR(sdma->regs); sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); - if (IS_ERR(sdma->clk_ipg)) { - ret = PTR_ERR(sdma->clk_ipg); - goto err_clk; - } + if (IS_ERR(sdma->clk_ipg)) + return PTR_ERR(sdma->clk_ipg); sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb"); - if (IS_ERR(sdma->clk_ahb)) { - ret = PTR_ERR(sdma->clk_ahb); - goto err_clk; - } + if (IS_ERR(sdma->clk_ahb)) + return PTR_ERR(sdma->clk_ahb); clk_prepare(sdma->clk_ipg); clk_prepare(sdma->clk_ahb); - sdma->regs = ioremap(iores->start, resource_size(iores)); - if (!sdma->regs) { - ret = -ENOMEM; - goto err_ioremap; - } - - ret = request_irq(irq, sdma_int_handler, 0, "sdma", sdma); + ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, "sdma", + sdma); if (ret) - goto err_request_irq; + return ret; sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL); - if (!sdma->script_addrs) { - ret = -ENOMEM; - goto err_alloc; - } + if (!sdma->script_addrs) + return -ENOMEM; /* initially no scripts available */ saddr_arr = (s32 *)sdma->script_addrs; for (i = 0; i < SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; i++) saddr_arr[i] = -EINVAL; - if (of_id) - pdev->id_entry = of_id->data; - sdma->devtype = pdev->id_entry->driver_data; - dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask); dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask); @@ -1401,6 +1755,12 @@ if (ret) goto err_init; + ret = sdma_event_remap(sdma); + if (ret) + goto err_init; + + if (sdma->drvdata->script_addrs) + sdma_add_scripts(sdma, sdma->drvdata->script_addrs); if (pdata && pdata->script_addrs) sdma_add_scripts(sdma, pdata->script_addrs); @@ -1432,39 +1792,65 @@ sdma->dma_device.device_tx_status = sdma_tx_status; sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg; sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic; - sdma->dma_device.device_control = sdma_control; + sdma->dma_device.device_config = sdma_config; + sdma->dma_device.device_terminate_all = sdma_disable_channel; + sdma->dma_device.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + sdma->dma_device.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + sdma->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; sdma->dma_device.device_issue_pending = sdma_issue_pending; sdma->dma_device.dev->dma_parms = &sdma->dma_parms; dma_set_max_seg_size(sdma->dma_device.dev, 65535); + platform_set_drvdata(pdev, sdma); + ret = dma_async_device_register(&sdma->dma_device); if (ret) { dev_err(&pdev->dev, "unable to register\n"); goto err_init; } - dev_info(sdma->dev, "initialized\n"); + if (np) { + ret = of_dma_controller_register(np, sdma_xlate, sdma); + if (ret) { + dev_err(&pdev->dev, "failed to register controller\n"); + goto err_register; + } + + spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus"); + ret = of_address_to_resource(spba_bus, 0, &spba_res); + if (!ret) { + sdma->spba_start_addr = spba_res.start; + sdma->spba_end_addr = spba_res.end; + } + of_node_put(spba_bus); + } return 0; +err_register: + dma_async_device_unregister(&sdma->dma_device); err_init: kfree(sdma->script_addrs); -err_alloc: - free_irq(irq, sdma); -err_request_irq: - iounmap(sdma->regs); -err_ioremap: -err_clk: - release_mem_region(iores->start, resource_size(iores)); -err_request_region: -err_irq: - kfree(sdma); return ret; } static int sdma_remove(struct platform_device *pdev) { - return -EBUSY; + struct sdma_engine *sdma = platform_get_drvdata(pdev); + int i; + + dma_async_device_unregister(&sdma->dma_device); + kfree(sdma->script_addrs); + /* Kill the tasklet */ + for (i = 0; i < MAX_DMA_CHANNELS; i++) { + struct sdma_channel *sdmac = &sdma->channel[i]; + + tasklet_kill(&sdmac->tasklet); + } + + platform_set_drvdata(pdev, NULL); + return 0; } static struct platform_driver sdma_driver = { @@ -1474,13 +1860,10 @@ }, .id_table = sdma_devtypes, .remove = sdma_remove, + .probe = sdma_probe, }; -static int __init sdma_module_init(void) -{ - return platform_driver_probe(&sdma_driver, sdma_probe); -} -module_init(sdma_module_init); +module_platform_driver(sdma_driver); MODULE_AUTHOR("Sascha Hauer, Pengutronix "); MODULE_DESCRIPTION("i.MX SDMA driver");