--- zzzz-none-000/linux-5.4.213/drivers/net/ethernet/mediatek/mtk_eth_soc.c 2022-09-15 10:04:56.000000000 +0000 +++ miami-7690-761/linux-5.4.213/drivers/net/ethernet/mediatek/mtk_eth_soc.c 2024-05-29 11:19:54.000000000 +0000 @@ -19,6 +19,9 @@ #include #include #include +#include +#include +#include #include "mtk_eth_soc.h" @@ -85,7 +88,7 @@ return 0; if (time_after(jiffies, t_start + PHY_IAC_TIMEOUT)) break; - usleep_range(10, 20); + cond_resched(); } dev_err(eth->dev, "mdio: MDIO timeout\n"); @@ -793,13 +796,18 @@ return buf_size; } -static inline void mtk_rx_get_desc(struct mtk_rx_dma *rxd, +static inline bool mtk_rx_get_desc(struct mtk_rx_dma *rxd, struct mtk_rx_dma *dma_rxd) { - rxd->rxd1 = READ_ONCE(dma_rxd->rxd1); rxd->rxd2 = READ_ONCE(dma_rxd->rxd2); + if (!(rxd->rxd2 & RX_DMA_DONE)) + return false; + + rxd->rxd1 = READ_ONCE(dma_rxd->rxd1); rxd->rxd3 = READ_ONCE(dma_rxd->rxd3); rxd->rxd4 = READ_ONCE(dma_rxd->rxd4); + + return true; } static void *mtk_max_lro_buf_alloc(gfp_t gfp_mask) @@ -885,7 +893,8 @@ return ((void *)dma - (void *)ring->dma) / sizeof(*dma); } -static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf) +static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf, + bool napi) { if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) { if (tx_buf->flags & MTK_TX_FLAGS_SINGLE0) { @@ -917,8 +926,12 @@ tx_buf->flags = 0; if (tx_buf->skb && - (tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC)) - dev_kfree_skb_any(tx_buf->skb); + (tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC)) { + if (napi) + napi_consume_skb(tx_buf->skb, napi); + else + dev_kfree_skb_any(tx_buf->skb); + } tx_buf->skb = NULL; } @@ -1096,7 +1109,7 @@ tx_buf = mtk_desc_to_tx_buf(ring, itxd); /* unmap dma */ - mtk_tx_unmap(eth, tx_buf); + mtk_tx_unmap(eth, tx_buf, false); itxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU; if (!MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) @@ -1153,17 +1166,6 @@ } } -static void mtk_stop_queue(struct mtk_eth *eth) -{ - int i; - - for (i = 0; i < MTK_MAC_COUNT; i++) { - if (!eth->netdev[i]) - continue; - netif_stop_queue(eth->netdev[i]); - } -} - static int mtk_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct mtk_mac *mac = netdev_priv(dev); @@ -1184,7 +1186,7 @@ tx_num = mtk_cal_txd_req(skb); if (unlikely(atomic_read(&ring->free_count) <= tx_num)) { - mtk_stop_queue(eth); + netif_stop_queue(dev); netif_err(eth, tx_queued, dev, "Tx Ring full when queue awake!\n"); spin_unlock(ð->page_lock); @@ -1210,7 +1212,7 @@ goto drop; if (unlikely(atomic_read(&ring->free_count) <= ring->thresh)) - mtk_stop_queue(eth); + netif_stop_queue(dev); spin_unlock(ð->page_lock); @@ -1266,18 +1268,20 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, struct mtk_eth *eth) { + struct dim_sample dim_sample = {}; struct mtk_rx_ring *ring; int idx; struct sk_buff *skb; u8 *data, *new_data; struct mtk_rx_dma *rxd, trxd; - int done = 0; + int done = 0, bytes = 0; while (done < budget) { struct net_device *netdev; unsigned int pktlen; dma_addr_t dma_addr; int mac; + u16 hash; ring = mtk_get_rx_ring(eth); if (unlikely(!ring)) @@ -1287,18 +1291,16 @@ rxd = &ring->dma[idx]; data = ring->data[idx]; - mtk_rx_get_desc(&trxd, rxd); - if (!(trxd.rxd2 & RX_DMA_DONE)) + if (!mtk_rx_get_desc(&trxd, rxd)) break; /* find out which mac the packet come from. values start at 1 */ - if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) { + if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) || + (trxd.rxd4 & RX_DMA_SPECIAL_TAG)) mac = 0; - } else { - mac = (trxd.rxd4 >> RX_DMA_FPORT_SHIFT) & - RX_DMA_FPORT_MASK; - mac--; - } + else + mac = ((trxd.rxd4 >> RX_DMA_FPORT_SHIFT) & + RX_DMA_FPORT_MASK) - 1; if (unlikely(mac < 0 || mac >= MTK_MAC_COUNT || !eth->netdev[mac])) @@ -1329,17 +1331,18 @@ goto release_desc; } + dma_unmap_single(eth->dev, trxd.rxd1, + ring->buf_size, DMA_FROM_DEVICE); + /* receive data */ skb = build_skb(data, ring->frag_size); if (unlikely(!skb)) { - skb_free_frag(new_data); + skb_free_frag(data); netdev->stats.rx_dropped++; - goto release_desc; + goto skip_rx; } skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); - dma_unmap_single(eth->dev, trxd.rxd1, - ring->buf_size, DMA_FROM_DEVICE); pktlen = RX_DMA_GET_PLEN0(trxd.rxd2); skb->dev = netdev; skb_put(skb, pktlen); @@ -1348,14 +1351,24 @@ else skb_checksum_none_assert(skb); skb->protocol = eth_type_trans(skb, netdev); + bytes += pktlen; + + hash = trxd.rxd4 & GENMASK(13, 0); + if (hash != GENMASK(13, 0)) + skb_set_hash(skb, hash, PKT_HASH_TYPE_L4); if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX && (trxd.rxd2 & RX_DMA_VTAG)) __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), RX_DMA_VID(trxd.rxd3)); - skb_record_rx_queue(skb, 0); - napi_gro_receive(napi, skb); + if (mtk_offload_check_rx(eth, skb, trxd.rxd4) == 0) { + skb_record_rx_queue(skb, 0); + napi_gro_receive(napi, skb); + } else { + dev_kfree_skb(skb); + } +skip_rx: ring->data[idx] = new_data; rxd->rxd1 = (unsigned int)dma_addr; @@ -1379,6 +1392,12 @@ mtk_update_rx_cpu_idx(eth); } + eth->rx_packets += done; + eth->rx_bytes += bytes; + dim_update_sample(eth->rx_events, eth->rx_packets, eth->rx_bytes, + &dim_sample); + net_dim(ð->rx_dim, dim_sample); + return done; } @@ -1391,7 +1410,7 @@ struct mtk_tx_buf *tx_buf; u32 cpu, dma; - cpu = mtk_r32(eth, MTK_QTX_CRX_PTR); + cpu = ring->last_free_ptr; dma = mtk_r32(eth, MTK_QTX_DRX_PTR); desc = mtk_qdma_phys_to_virt(ring, cpu); @@ -1417,7 +1436,7 @@ done[mac]++; budget--; } - mtk_tx_unmap(eth, tx_buf); + mtk_tx_unmap(eth, tx_buf, true); ring->last_free = desc; atomic_inc(&ring->free_count); @@ -1425,6 +1444,7 @@ cpu = next_cpu; } + ring->last_free_ptr = cpu; mtk_w32(eth, cpu, MTK_QTX_CRX_PTR); return budget; @@ -1454,7 +1474,7 @@ budget--; } - mtk_tx_unmap(eth, tx_buf); + mtk_tx_unmap(eth, tx_buf, true); desc = &ring->dma[cpu]; ring->last_free = desc; @@ -1471,6 +1491,7 @@ static int mtk_poll_tx(struct mtk_eth *eth, int budget) { struct mtk_tx_ring *ring = ð->tx_ring; + struct dim_sample dim_sample = {}; unsigned int done[MTK_MAX_DEVS]; unsigned int bytes[MTK_MAX_DEVS]; int total = 0, i; @@ -1488,8 +1509,14 @@ continue; netdev_completed_queue(eth->netdev[i], done[i], bytes[i]); total += done[i]; + eth->tx_packets += done[i]; + eth->tx_bytes += bytes[i]; } + dim_update_sample(eth->tx_events, eth->tx_packets, eth->tx_bytes, + &dim_sample); + net_dim(ð->tx_dim, dim_sample); + if (mtk_queue_stopped(eth) && (atomic_read(&ring->free_count) > ring->thresh)) mtk_wake_queue(eth); @@ -1534,8 +1561,8 @@ if (status & MTK_TX_DONE_INT) return budget; - napi_complete(napi); - mtk_tx_irq_enable(eth, MTK_TX_DONE_INT); + if (napi_complete(napi)) + mtk_tx_irq_enable(eth, MTK_TX_DONE_INT); return tx_done; } @@ -1568,8 +1595,9 @@ remain_budget -= rx_done; goto poll_again; } - napi_complete(napi); - mtk_rx_irq_enable(eth, MTK_RX_DONE_INT); + + if (napi_complete(napi)) + mtk_rx_irq_enable(eth, MTK_RX_DONE_INT); return rx_done + budget - remain_budget; } @@ -1618,6 +1646,7 @@ atomic_set(&ring->free_count, MTK_DMA_SIZE - 2); ring->next_free = &ring->dma[0]; ring->last_free = &ring->dma[MTK_DMA_SIZE - 1]; + ring->last_free_ptr = (u32)(ring->phys + ((MTK_DMA_SIZE - 1) * sz)); ring->thresh = MAX_SKB_FRAGS; /* make sure that all changes to the dma ring are flushed before we @@ -1631,9 +1660,7 @@ mtk_w32(eth, ring->phys + ((MTK_DMA_SIZE - 1) * sz), MTK_QTX_CRX_PTR); - mtk_w32(eth, - ring->phys + ((MTK_DMA_SIZE - 1) * sz), - MTK_QTX_DRX_PTR); + mtk_w32(eth, ring->last_free_ptr, MTK_QTX_DRX_PTR); mtk_w32(eth, (QDMA_RES_THRES << 8) | QDMA_RES_THRES, MTK_QTX_CFG(0)); } else { @@ -1656,7 +1683,7 @@ if (ring->buf) { for (i = 0; i < MTK_DMA_SIZE; i++) - mtk_tx_unmap(eth, &ring->buf[i]); + mtk_tx_unmap(eth, &ring->buf[i], false); kfree(ring->buf); ring->buf = NULL; } @@ -2170,6 +2197,7 @@ { struct mtk_eth *eth = _eth; + eth->rx_events++; if (likely(napi_schedule_prep(ð->rx_napi))) { __napi_schedule(ð->rx_napi); mtk_rx_irq_disable(eth, MTK_RX_DONE_INT); @@ -2182,6 +2210,7 @@ { struct mtk_eth *eth = _eth; + eth->tx_events++; if (likely(napi_schedule_prep(ð->tx_napi))) { __napi_schedule(ð->tx_napi); mtk_tx_irq_disable(eth, MTK_TX_DONE_INT); @@ -2234,7 +2263,7 @@ if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) { mtk_w32(eth, MTK_TX_WB_DDONE | MTK_TX_DMA_EN | - MTK_DMA_SIZE_16DWORDS | MTK_NDP_CO_PRO | + MTK_TX_BT_32DWORDS | MTK_NDP_CO_PRO | MTK_RX_DMA_EN | MTK_RX_2B_OFFSET | MTK_RX_BT_32DWORDS, MTK_QDMA_GLO_CFG); @@ -2252,6 +2281,34 @@ return 0; } +static void mtk_gdm_config(struct mtk_eth *eth, u32 config) +{ + int i; + + if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) + return; + + for (i = 0; i < MTK_MAC_COUNT; i++) { + u32 val = mtk_r32(eth, MTK_GDMA_FWD_CFG(i)); + + /* default setup the forward port to send frame to PDMA */ + val &= ~0xffff; + + /* Enable RX checksum */ + val |= MTK_GDMA_ICS_EN | MTK_GDMA_TCS_EN | MTK_GDMA_UCS_EN; + + val |= config; + + if (!i && eth->netdev[0] && netdev_uses_dsa(eth->netdev[0])) + val |= MTK_GDMA_SPECIAL_TAG; + + mtk_w32(eth, val, MTK_GDMA_FWD_CFG(i)); + } + /* Reset and enable PSE */ + mtk_w32(eth, RST_GL_PSE, MTK_RST_GL); + mtk_w32(eth, 0, MTK_RST_GL); +} + static int mtk_open(struct net_device *dev) { struct mtk_mac *mac = netdev_priv(dev); @@ -2267,11 +2324,18 @@ /* we run 2 netdevs on the same dma ring so we only bring it up once */ if (!refcount_read(ð->dma_refcnt)) { - int err = mtk_start_dma(eth); + u32 gdm_config = MTK_GDMA_TO_PDMA; + int err; + err = mtk_start_dma(eth); if (err) return err; + if (eth->soc->offload_version && mtk_ppe_start(ð->ppe) == 0) + gdm_config = MTK_GDMA_TO_PPE; + + mtk_gdm_config(eth, gdm_config); + napi_enable(ð->tx_napi); napi_enable(ð->rx_napi); mtk_tx_irq_enable(eth, MTK_TX_DONE_INT); @@ -2324,17 +2388,25 @@ if (!refcount_dec_and_test(ð->dma_refcnt)) return 0; + mtk_gdm_config(eth, MTK_GDMA_DROP_ALL); + mtk_tx_irq_disable(eth, MTK_TX_DONE_INT); mtk_rx_irq_disable(eth, MTK_RX_DONE_INT); napi_disable(ð->tx_napi); napi_disable(ð->rx_napi); + cancel_work_sync(ð->rx_dim.work); + cancel_work_sync(ð->tx_dim.work); + if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) mtk_stop_dma(eth, MTK_QDMA_GLO_CFG); mtk_stop_dma(eth, MTK_PDMA_GLO_CFG); mtk_dma_free(eth); + if (eth->soc->offload_version) + mtk_ppe_stop(ð->ppe); + return 0; } @@ -2378,6 +2450,64 @@ return ret; } +static void mtk_dim_rx(struct work_struct *work) +{ + struct dim *dim = container_of(work, struct dim, work); + struct mtk_eth *eth = container_of(dim, struct mtk_eth, rx_dim); + struct dim_cq_moder cur_profile; + u32 val, cur; + + cur_profile = net_dim_get_rx_moderation(eth->rx_dim.mode, + dim->profile_ix); + spin_lock_bh(ð->dim_lock); + + val = mtk_r32(eth, MTK_PDMA_DELAY_INT); + val &= MTK_PDMA_DELAY_TX_MASK; + val |= MTK_PDMA_DELAY_RX_EN; + + cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK); + val |= cur << MTK_PDMA_DELAY_RX_PTIME_SHIFT; + + cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK); + val |= cur << MTK_PDMA_DELAY_RX_PINT_SHIFT; + + mtk_w32(eth, val, MTK_PDMA_DELAY_INT); + mtk_w32(eth, val, MTK_QDMA_DELAY_INT); + + spin_unlock_bh(ð->dim_lock); + + dim->state = DIM_START_MEASURE; +} + +static void mtk_dim_tx(struct work_struct *work) +{ + struct dim *dim = container_of(work, struct dim, work); + struct mtk_eth *eth = container_of(dim, struct mtk_eth, tx_dim); + struct dim_cq_moder cur_profile; + u32 val, cur; + + cur_profile = net_dim_get_tx_moderation(eth->tx_dim.mode, + dim->profile_ix); + spin_lock_bh(ð->dim_lock); + + val = mtk_r32(eth, MTK_PDMA_DELAY_INT); + val &= MTK_PDMA_DELAY_RX_MASK; + val |= MTK_PDMA_DELAY_TX_EN; + + cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK); + val |= cur << MTK_PDMA_DELAY_TX_PTIME_SHIFT; + + cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK); + val |= cur << MTK_PDMA_DELAY_TX_PINT_SHIFT; + + mtk_w32(eth, val, MTK_PDMA_DELAY_INT); + mtk_w32(eth, val, MTK_QDMA_DELAY_INT); + + spin_unlock_bh(ð->dim_lock); + + dim->state = DIM_START_MEASURE; +} + static int mtk_hw_init(struct mtk_eth *eth) { int i, val, ret; @@ -2399,9 +2529,6 @@ goto err_disable_pm; } - /* enable interrupt delay for RX */ - mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT); - /* disable delay and normal interrupt */ mtk_tx_irq_disable(eth, ~0); mtk_rx_irq_disable(eth, ~0); @@ -2440,15 +2567,12 @@ /* Enable RX VLan Offloading */ mtk_w32(eth, 1, MTK_CDMP_EG_CTRL); - /* enable interrupt delay for RX */ - mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT); + mtk_dim_rx(ð->rx_dim.work); + mtk_dim_tx(ð->tx_dim.work); /* disable delay and normal interrupt */ - mtk_w32(eth, 0, MTK_QDMA_DELAY_INT); mtk_tx_irq_disable(eth, ~0); mtk_rx_irq_disable(eth, ~0); - mtk_w32(eth, RST_GL_PSE, MTK_RST_GL); - mtk_w32(eth, 0, MTK_RST_GL); /* FE int grouping */ mtk_w32(eth, MTK_TX_DONE_INT, MTK_PDMA_INT_GRP1); @@ -2457,19 +2581,6 @@ mtk_w32(eth, MTK_RX_DONE_INT, MTK_QDMA_INT_GRP2); mtk_w32(eth, 0x21021000, MTK_FE_INT_GRP); - for (i = 0; i < MTK_MAC_COUNT; i++) { - u32 val = mtk_r32(eth, MTK_GDMA_FWD_CFG(i)); - - /* setup the forward port to send frame to PDMA */ - val &= ~0xffff; - - /* Enable RX checksum */ - val |= MTK_GDMA_ICS_EN | MTK_GDMA_TCS_EN | MTK_GDMA_UCS_EN; - - /* setup the mac dma */ - mtk_w32(eth, val, MTK_GDMA_FWD_CFG(i)); - } - return 0; err_disable_pm: @@ -2794,6 +2905,25 @@ return ret; } +static int +mtk_flow_offload(enum flow_offload_type type, struct flow_offload *flow, + struct flow_offload_hw_path *src, + struct flow_offload_hw_path *dest) +{ + struct mtk_mac *mac = netdev_priv(src->dev); + struct mtk_eth *eth = mac->hw; + + if (!eth->soc->offload_version) + return -EINVAL; + + if (src->dev->base_addr != dest->dev->base_addr) + return -EINVAL; + + mac = netdev_priv(src->dev); + + return mtk_flow_offload_add(eth, type, flow, src, dest); +} + static const struct ethtool_ops mtk_ethtool_ops = { .get_link_ksettings = mtk_get_link_ksettings, .set_link_ksettings = mtk_set_link_ksettings, @@ -2825,6 +2955,7 @@ #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = mtk_poll_controller, #endif + .ndo_flow_offload = mtk_flow_offload, }; static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) @@ -2963,6 +3094,13 @@ spin_lock_init(ð->page_lock); spin_lock_init(ð->tx_irq_lock); spin_lock_init(ð->rx_irq_lock); + spin_lock_init(ð->dim_lock); + + eth->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; + INIT_WORK(ð->rx_dim.work, mtk_dim_rx); + + eth->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; + INIT_WORK(ð->tx_dim.work, mtk_dim_tx); if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) { eth->ethsys = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, @@ -3078,6 +3216,17 @@ goto err_free_dev; } + if (eth->soc->offload_version) { + err = mtk_ppe_init(ð->ppe, eth->dev, + eth->base + MTK_ETH_PPE_BASE, 2); + if (err) + goto err_free_dev; + + err = mtk_flow_offload_init(eth); + if (err) + goto err_free_dev; + } + for (i = 0; i < MTK_MAX_DEVS; i++) { if (!eth->netdev[i]) continue; @@ -3152,6 +3301,7 @@ .hw_features = MTK_HW_FEATURES, .required_clks = MT7621_CLKS_BITMAP, .required_pctl = false, + .offload_version = 2, }; static const struct mtk_soc_data mt7622_data = { @@ -3160,6 +3310,7 @@ .hw_features = MTK_HW_FEATURES, .required_clks = MT7622_CLKS_BITMAP, .required_pctl = false, + .offload_version = 2, }; static const struct mtk_soc_data mt7623_data = {