--- zzzz-none-000/linux-3.10.107/drivers/net/ethernet/marvell/mvneta.c 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/drivers/net/ethernet/marvell/mvneta.c 2021-02-04 17:41:59.000000000 +0000 @@ -20,8 +20,11 @@ #include #include #include +#include #include #include +#include +#include #include #include #include @@ -29,10 +32,11 @@ #include #include #include +#include /* Registers */ #define MVNETA_RXQ_CONFIG_REG(q) (0x1400 + ((q) << 2)) -#define MVNETA_RXQ_HW_BUF_ALLOC BIT(1) +#define MVNETA_RXQ_HW_BUF_ALLOC BIT(0) #define MVNETA_RXQ_PKT_OFFSET_ALL_MASK (0xf << 8) #define MVNETA_RXQ_PKT_OFFSET_MASK(offs) ((offs) << 8) #define MVNETA_RXQ_THRESHOLD_REG(q) (0x14c0 + ((q) << 2)) @@ -58,6 +62,7 @@ #define MVNETA_WIN_SIZE(w) (0x2204 + ((w) << 3)) #define MVNETA_WIN_REMAP(w) (0x2280 + ((w) << 2)) #define MVNETA_BASE_ADDR_ENABLE 0x2290 +#define MVNETA_ACCESS_PROTECT_ENABLE 0x2294 #define MVNETA_PORT_CONFIG 0x2400 #define MVNETA_UNI_PROMISC_MODE BIT(0) #define MVNETA_DEF_RXQ(q) ((q) << 1) @@ -79,21 +84,28 @@ #define MVNETA_MAC_ADDR_HIGH 0x2418 #define MVNETA_SDMA_CONFIG 0x241c #define MVNETA_SDMA_BRST_SIZE_16 4 -#define MVNETA_NO_DESC_SWAP 0x0 #define MVNETA_RX_BRST_SZ_MASK(burst) ((burst) << 1) #define MVNETA_RX_NO_DATA_SWAP BIT(4) #define MVNETA_TX_NO_DATA_SWAP BIT(5) +#define MVNETA_DESC_SWAP BIT(6) #define MVNETA_TX_BRST_SZ_MASK(burst) ((burst) << 22) #define MVNETA_PORT_STATUS 0x2444 #define MVNETA_TX_IN_PRGRS BIT(1) #define MVNETA_TX_FIFO_EMPTY BIT(8) #define MVNETA_RX_MIN_FRAME_SIZE 0x247c +#define MVNETA_SERDES_CFG 0x24A0 +#define MVNETA_SGMII_SERDES_PROTO 0x0cc7 +#define MVNETA_QSGMII_SERDES_PROTO 0x0667 #define MVNETA_TYPE_PRIO 0x24bc #define MVNETA_FORCE_UNI BIT(21) #define MVNETA_TXQ_CMD_1 0x24e4 #define MVNETA_TXQ_CMD 0x2448 #define MVNETA_TXQ_DISABLE_SHIFT 8 #define MVNETA_TXQ_ENABLE_MASK 0x000000ff +#define MVNETA_RX_DISCARD_FRAME_COUNT 0x2484 +#define MVNETA_OVERRUN_FRAME_COUNT 0x2488 +#define MVNETA_GMAC_CLOCK_DIVIDER 0x24f4 +#define MVNETA_GMAC_1MS_CLOCK_ENABLE BIT(31) #define MVNETA_ACC_MODE 0x2500 #define MVNETA_CPU_MAP(cpu) (0x2540 + ((cpu) << 2)) #define MVNETA_CPU_RXQ_ACCESS_ALL_MASK 0x000000ff @@ -116,6 +128,7 @@ #define MVNETA_TX_INTR_MASK_ALL (0xff << 0) #define MVNETA_RX_INTR_MASK(nr_rxqs) (((1 << nr_rxqs) - 1) << 8) #define MVNETA_RX_INTR_MASK_ALL (0xff << 8) +#define MVNETA_MISCINTR_INTR_MASK BIT(31) #define MVNETA_INTR_OLD_CAUSE 0x25a8 #define MVNETA_INTR_OLD_MASK 0x25ac @@ -147,7 +160,7 @@ #define MVNETA_INTR_ENABLE 0x25b8 #define MVNETA_TXQ_INTR_ENABLE_ALL_MASK 0x0000ff00 -#define MVNETA_RXQ_INTR_ENABLE_ALL_MASK 0xff000000 // note: neta says it's 0x000000FF +#define MVNETA_RXQ_INTR_ENABLE_ALL_MASK 0x000000ff #define MVNETA_RXQ_CMD 0x2680 #define MVNETA_RXQ_DISABLE_SHIFT 8 @@ -159,6 +172,7 @@ #define MVNETA_GMAC_MAX_RX_SIZE_MASK 0x7ffc #define MVNETA_GMAC0_PORT_ENABLE BIT(0) #define MVNETA_GMAC_CTRL_2 0x2c08 +#define MVNETA_GMAC2_INBAND_AN_ENABLE BIT(0) #define MVNETA_GMAC2_PCS_ENABLE BIT(3) #define MVNETA_GMAC2_PORT_RGMII BIT(4) #define MVNETA_GMAC2_PORT_RESET BIT(6) @@ -174,12 +188,14 @@ #define MVNETA_GMAC_AUTONEG_CONFIG 0x2c0c #define MVNETA_GMAC_FORCE_LINK_DOWN BIT(0) #define MVNETA_GMAC_FORCE_LINK_PASS BIT(1) +#define MVNETA_GMAC_INBAND_AN_ENABLE BIT(2) #define MVNETA_GMAC_CONFIG_MII_SPEED BIT(5) #define MVNETA_GMAC_CONFIG_GMII_SPEED BIT(6) #define MVNETA_GMAC_AN_SPEED_EN BIT(7) +#define MVNETA_GMAC_AN_FLOW_CTRL_EN BIT(11) #define MVNETA_GMAC_CONFIG_FULL_DUPLEX BIT(12) #define MVNETA_GMAC_AN_DUPLEX_EN BIT(13) -#define MVNETA_MIB_COUNTERS_BASE 0x3080 +#define MVNETA_MIB_COUNTERS_BASE 0x3000 #define MVNETA_MIB_LATE_COLLISION 0x7c #define MVNETA_DA_FILT_SPEC_MCAST 0x3400 #define MVNETA_DA_FILT_OTH_MCAST 0x3500 @@ -214,9 +230,6 @@ #define MVNETA_RX_COAL_PKTS 32 #define MVNETA_RX_COAL_USEC 100 -/* Napi polling weight */ -#define MVNETA_RX_POLL_WEIGHT 64 - /* The two bytes Marvell header. Either contains a special value used * by Marvell switches when a specific hardware mode is enabled (not * supported by this driver) or is filled automatically by zeroes on @@ -230,6 +243,7 @@ #define MVNETA_VLAN_TAG_LEN 4 #define MVNETA_CPU_D_CACHE_LINE_SIZE 32 +#define MVNETA_TX_CSUM_DEF_SIZE 1600 #define MVNETA_TX_CSUM_MAX_SIZE 9800 #define MVNETA_ACC_MODE_EXT 1 @@ -240,12 +254,20 @@ #define MVNETA_TX_MTU_MAX 0x3ffff +/* TSO header size */ +#define TSO_HEADER_SIZE 128 + /* Max number of Rx descriptors */ #define MVNETA_MAX_RXD 128 /* Max number of Tx descriptors */ #define MVNETA_MAX_TXD 532 +/* Max number of allowed TCP segments for software TSO */ +#define MVNETA_MAX_TSO_SEGS 100 + +#define MVNETA_MAX_SKB_DESCS (MVNETA_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS) + /* descriptor aligned size */ #define MVNETA_DESC_ALIGNED_SIZE 32 @@ -254,8 +276,56 @@ ETH_HLEN + ETH_FCS_LEN, \ MVNETA_CPU_D_CACHE_LINE_SIZE) +#define IS_TSO_HEADER(txq, addr) \ + ((addr >= txq->tso_hdrs_phys) && \ + (addr < txq->tso_hdrs_phys + txq->size * TSO_HEADER_SIZE)) + #define MVNETA_RX_BUF_SIZE(pkt_size) ((pkt_size) + NET_SKB_PAD) +struct mvneta_statistic { + unsigned short offset; + unsigned short type; + const char name[ETH_GSTRING_LEN]; +}; + +#define T_REG_32 32 +#define T_REG_64 64 + +static const struct mvneta_statistic mvneta_statistics[] = { + { 0x3000, T_REG_64, "good_octets_received", }, + { 0x3010, T_REG_32, "good_frames_received", }, + { 0x3008, T_REG_32, "bad_octets_received", }, + { 0x3014, T_REG_32, "bad_frames_received", }, + { 0x3018, T_REG_32, "broadcast_frames_received", }, + { 0x301c, T_REG_32, "multicast_frames_received", }, + { 0x3050, T_REG_32, "unrec_mac_control_received", }, + { 0x3058, T_REG_32, "good_fc_received", }, + { 0x305c, T_REG_32, "bad_fc_received", }, + { 0x3060, T_REG_32, "undersize_received", }, + { 0x3064, T_REG_32, "fragments_received", }, + { 0x3068, T_REG_32, "oversize_received", }, + { 0x306c, T_REG_32, "jabber_received", }, + { 0x3070, T_REG_32, "mac_receive_error", }, + { 0x3074, T_REG_32, "bad_crc_event", }, + { 0x3078, T_REG_32, "collision", }, + { 0x307c, T_REG_32, "late_collision", }, + { 0x2484, T_REG_32, "rx_discard", }, + { 0x2488, T_REG_32, "rx_overrun", }, + { 0x3020, T_REG_32, "frames_64_octets", }, + { 0x3024, T_REG_32, "frames_65_to_127_octets", }, + { 0x3028, T_REG_32, "frames_128_to_255_octets", }, + { 0x302c, T_REG_32, "frames_256_to_511_octets", }, + { 0x3030, T_REG_32, "frames_512_to_1023_octets", }, + { 0x3034, T_REG_32, "frames_1024_to_max_octets", }, + { 0x3038, T_REG_64, "good_octets_sent", }, + { 0x3040, T_REG_32, "good_frames_sent", }, + { 0x3044, T_REG_32, "excessive_collision", }, + { 0x3048, T_REG_32, "multicast_frames_sent", }, + { 0x304c, T_REG_32, "broadcast_frames_sent", }, + { 0x3054, T_REG_32, "fc_sent", }, + { 0x300c, T_REG_32, "internal_mac_transmit_err", }, +}; + struct mvneta_pcpu_stats { struct u64_stats_sync syncp; u64 rx_packets; @@ -264,25 +334,34 @@ u64 tx_bytes; }; +struct mvneta_pcpu_port { + /* Pointer to the shared port */ + struct mvneta_port *pp; + + /* Pointer to the CPU-local NAPI struct */ + struct napi_struct napi; + + /* Cause of the previous interrupt */ + u32 cause_rx_tx; +}; + struct mvneta_port { + struct mvneta_pcpu_port __percpu *ports; + struct mvneta_pcpu_stats __percpu *stats; + int pkt_size; + unsigned int frag_size; void __iomem *base; struct mvneta_rx_queue *rxqs; struct mvneta_tx_queue *txqs; struct net_device *dev; - - u32 cause_rx_tx; - struct napi_struct napi; - - /* Napi weight */ - int weight; + struct notifier_block cpu_notifier; /* Core clock */ struct clk *clk; u8 mcast_count[256]; u16 tx_ring_size; u16 rx_ring_size; - struct mvneta_pcpu_stats *stats; struct mii_bus *mii_bus; struct phy_device *phy_dev; @@ -291,14 +370,17 @@ unsigned int link; unsigned int duplex; unsigned int speed; + unsigned int tx_csum_limit; + int use_inband_status:1; + + u64 ethtool_stats[ARRAY_SIZE(mvneta_statistics)]; }; /* The mvneta_tx_desc and mvneta_rx_desc structures describe the * layout of the transmit and reception DMA descriptors, and their * layout is therefore defined by the hardware design */ -struct mvneta_tx_desc { - u32 command; /* Options used by HW for packet transmitting.*/ + #define MVNETA_TX_L3_OFF_SHIFT 0 #define MVNETA_TX_IP_HLEN_SHIFT 8 #define MVNETA_TX_L4_UDP BIT(16) @@ -313,15 +395,6 @@ #define MVNETA_TX_L4_CSUM_FULL BIT(30) #define MVNETA_TX_L4_CSUM_NOT BIT(31) - u16 reserverd1; /* csum_l4 (for future use) */ - u16 data_size; /* Data size of transmitted packet in bytes */ - u32 buf_phys_addr; /* Physical addr of transmitted buffer */ - u32 reserved2; /* hw_cmd - (for future use, PMT) */ - u32 reserved3[4]; /* Reserved - (for future use) */ -}; - -struct mvneta_rx_desc { - u32 status; /* Info about received packet */ #define MVNETA_RXD_ERR_CRC 0x0 #define MVNETA_RXD_ERR_SUMMARY BIT(16) #define MVNETA_RXD_ERR_OVERRUN BIT(17) @@ -332,16 +405,57 @@ #define MVNETA_RXD_FIRST_LAST_DESC (BIT(26) | BIT(27)) #define MVNETA_RXD_L4_CSUM_OK BIT(30) +#if defined(__LITTLE_ENDIAN) +struct mvneta_tx_desc { + u32 command; /* Options used by HW for packet transmitting.*/ + u16 reserverd1; /* csum_l4 (for future use) */ + u16 data_size; /* Data size of transmitted packet in bytes */ + u32 buf_phys_addr; /* Physical addr of transmitted buffer */ + u32 reserved2; /* hw_cmd - (for future use, PMT) */ + u32 reserved3[4]; /* Reserved - (for future use) */ +}; + +struct mvneta_rx_desc { + u32 status; /* Info about received packet */ u16 reserved1; /* pnc_info - (for future use, PnC) */ u16 data_size; /* Size of received packet in bytes */ + u32 buf_phys_addr; /* Physical address of the buffer */ u32 reserved2; /* pnc_flow_id (for future use, PnC) */ + u32 buf_cookie; /* cookie for access to RX buffer in rx path */ u16 reserved3; /* prefetch_cmd, for future use */ u16 reserved4; /* csum_l4 - (for future use, PnC) */ + + u32 reserved5; /* pnc_extra PnC (for future use, PnC) */ + u32 reserved6; /* hw_cmd (for future use, PnC and HWF) */ +}; +#else +struct mvneta_tx_desc { + u16 data_size; /* Data size of transmitted packet in bytes */ + u16 reserverd1; /* csum_l4 (for future use) */ + u32 command; /* Options used by HW for packet transmitting.*/ + u32 reserved2; /* hw_cmd - (for future use, PMT) */ + u32 buf_phys_addr; /* Physical addr of transmitted buffer */ + u32 reserved3[4]; /* Reserved - (for future use) */ +}; + +struct mvneta_rx_desc { + u16 data_size; /* Size of received packet in bytes */ + u16 reserved1; /* pnc_info - (for future use, PnC) */ + u32 status; /* Info about received packet */ + + u32 reserved2; /* pnc_flow_id (for future use, PnC) */ + u32 buf_phys_addr; /* Physical address of the buffer */ + + u16 reserved4; /* csum_l4 - (for future use, PnC) */ + u16 reserved3; /* prefetch_cmd, for future use */ + u32 buf_cookie; /* cookie for access to RX buffer in rx path */ + u32 reserved5; /* pnc_extra PnC (for future use, PnC) */ u32 reserved6; /* hw_cmd (for future use, PnC and HWF) */ }; +#endif struct mvneta_tx_queue { /* Number of this TX queue, in the range 0-7 */ @@ -354,6 +468,8 @@ * descriptor ring */ int count; + int tx_stop_threshold; + int tx_wake_threshold; /* Array of transmitted skb */ struct sk_buff **tx_skb; @@ -377,6 +493,12 @@ /* Index of the next TX DMA descriptor to process */ int next_desc_to_proc; + + /* DMA buffers for TSO headers */ + char *tso_hdrs; + + /* DMA address of TSO headers */ + dma_addr_t tso_hdrs_phys; }; struct mvneta_rx_queue { @@ -405,11 +527,16 @@ int next_desc_to_proc; }; +/* The hardware supports eight (8) rx queues, but we are only allowing + * the first one to be used. Therefore, let's just allocate one queue. + */ static int rxq_number = 8; static int txq_number = 8; static int rxq_def; +static int rx_copybreak __read_mostly = 256; + #define MVNETA_DRIVER_NAME "mvneta" #define MVNETA_DRIVER_VERSION "1.0" @@ -453,6 +580,8 @@ /* Perform dummy reads from MIB counters */ for (i = 0; i < MVNETA_MIB_LATE_COLLISION; i += 4) dummy = mvreg_read(pp, (MVNETA_MIB_COUNTERS_BASE + i)); + dummy = mvreg_read(pp, MVNETA_RX_DISCARD_FRAME_COUNT); + dummy = mvreg_read(pp, MVNETA_OVERRUN_FRAME_COUNT); } /* Get System Network Statistics */ @@ -472,12 +601,12 @@ cpu_stats = per_cpu_ptr(pp->stats, cpu); do { - start = u64_stats_fetch_begin_bh(&cpu_stats->syncp); + start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); rx_packets = cpu_stats->rx_packets; rx_bytes = cpu_stats->rx_bytes; tx_packets = cpu_stats->tx_packets; tx_bytes = cpu_stats->tx_bytes; - } while (u64_stats_fetch_retry_bh(&cpu_stats->syncp, start)); + } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); stats->rx_packets += rx_packets; stats->rx_bytes += rx_bytes; @@ -495,14 +624,14 @@ /* Rx descriptors helper methods */ -/* Checks whether the given RX descriptor is both the first and the - * last descriptor for the RX packet. Each RX packet is currently +/* Checks whether the RX descriptor having this status is both the first + * and the last descriptor for the RX packet. Each RX packet is currently * received through a single RX descriptor, so not having each RX * descriptor with its first and last bits set is an error */ -static int mvneta_rxq_desc_is_first_last(struct mvneta_rx_desc *desc) +static int mvneta_rxq_desc_is_first_last(u32 status) { - return (desc->status & MVNETA_RXD_FIRST_LAST_DESC) == + return (status & MVNETA_RXD_FIRST_LAST_DESC) == MVNETA_RXD_FIRST_LAST_DESC; } @@ -578,6 +707,7 @@ int rx_desc = rxq->next_desc_to_proc; rxq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(rxq, rx_desc); + prefetch(rxq->descs + rxq->next_desc_to_proc); return rxq->descs + rx_desc; } @@ -673,33 +803,6 @@ mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); } - - -/* Sets the RGMII Enable bit (RGMIIEn) in port MAC control register */ -static void mvneta_gmac_rgmii_set(struct mvneta_port *pp, int enable) -{ - u32 val; - - val = mvreg_read(pp, MVNETA_GMAC_CTRL_2); - - if (enable) - val |= MVNETA_GMAC2_PORT_RGMII; - else - val &= ~MVNETA_GMAC2_PORT_RGMII; - - mvreg_write(pp, MVNETA_GMAC_CTRL_2, val); -} - -/* Config SGMII port */ -static void mvneta_port_sgmii_config(struct mvneta_port *pp) -{ - u32 val; - - val = mvreg_read(pp, MVNETA_GMAC_CTRL_2); - val |= MVNETA_GMAC2_PCS_ENABLE; - mvreg_write(pp, MVNETA_GMAC_CTRL_2, val); -} - /* Start the Ethernet port RX and TX activity */ static void mvneta_port_up(struct mvneta_port *pp) { @@ -707,7 +810,6 @@ u32 q_map; /* Enable all initialized TXs. */ - mvneta_mib_counters_clear(pp); q_map = 0; for (queue = 0; queue < txq_number; queue++) { struct mvneta_tx_queue *txq = &pp->txqs[queue]; @@ -717,14 +819,7 @@ mvreg_write(pp, MVNETA_TXQ_CMD, q_map); /* Enable all initialized RXQs. */ - q_map = 0; - for (queue = 0; queue < rxq_number; queue++) { - struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; - if (rxq->descs != NULL) - q_map |= (1 << queue); - } - - mvreg_write(pp, MVNETA_RXQ_CMD, q_map); + mvreg_write(pp, MVNETA_RXQ_CMD, BIT(rxq_def)); } /* Stop the Ethernet port activity */ @@ -947,9 +1042,11 @@ /* Default burst size */ val |= MVNETA_TX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16); val |= MVNETA_RX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16); + val |= MVNETA_RX_NO_DATA_SWAP | MVNETA_TX_NO_DATA_SWAP; - val |= (MVNETA_RX_NO_DATA_SWAP | MVNETA_TX_NO_DATA_SWAP | - MVNETA_NO_DESC_SWAP); +#if defined(__BIG_ENDIAN) + val |= MVNETA_DESC_SWAP; +#endif /* Assign port SDMA configuration */ mvreg_write(pp, MVNETA_SDMA_CONFIG, val); @@ -961,6 +1058,26 @@ val &= ~MVNETA_PHY_POLLING_ENABLE; mvreg_write(pp, MVNETA_UNIT_CONTROL, val); + if (pp->use_inband_status) { + val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); + val &= ~(MVNETA_GMAC_FORCE_LINK_PASS | + MVNETA_GMAC_FORCE_LINK_DOWN | + MVNETA_GMAC_AN_FLOW_CTRL_EN); + val |= MVNETA_GMAC_INBAND_AN_ENABLE | + MVNETA_GMAC_AN_SPEED_EN | + MVNETA_GMAC_AN_DUPLEX_EN; + mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); + val = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER); + val |= MVNETA_GMAC_1MS_CLOCK_ENABLE; + mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, val); + } else { + val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); + val &= ~(MVNETA_GMAC_INBAND_AN_ENABLE | + MVNETA_GMAC_AN_SPEED_EN | + MVNETA_GMAC_AN_DUPLEX_EN); + mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); + } + mvneta_set_ucast_table(pp, -1); mvneta_set_special_mcast_table(pp, -1); mvneta_set_other_mcast_table(pp, -1); @@ -969,6 +1086,8 @@ mvreg_write(pp, MVNETA_INTR_ENABLE, (MVNETA_RXQ_INTR_ENABLE_ALL_MASK | MVNETA_TXQ_INTR_ENABLE_ALL_MASK)); + + mvneta_mib_counters_clear(pp); } /* Set max sizes for tx queues */ @@ -1197,10 +1316,10 @@ { u32 status = rx_desc->status; - if (!mvneta_rxq_desc_is_first_last(rx_desc)) { + if (!mvneta_rxq_desc_is_first_last(status)) { netdev_err(pp->dev, "bad rx status %08x (buffer oversize), size=%d\n", - rx_desc->status, rx_desc->data_size); + status, rx_desc->data_size); return; } @@ -1224,13 +1343,12 @@ } } -/* Handle RX checksum offload */ -static void mvneta_rx_csum(struct mvneta_port *pp, - struct mvneta_rx_desc *rx_desc, +/* Handle RX checksum offload based on the descriptor's status */ +static void mvneta_rx_csum(struct mvneta_port *pp, u32 status, struct sk_buff *skb) { - if ((rx_desc->status & MVNETA_RXD_L3_IP4) && - (rx_desc->status & MVNETA_RXD_L4_CSUM_OK)) { + if ((status & MVNETA_RXD_L3_IP4) && + (status & MVNETA_RXD_L4_CSUM_OK)) { skb->csum = 0; skb->ip_summed = CHECKSUM_UNNECESSARY; return; @@ -1239,13 +1357,16 @@ skb->ip_summed = CHECKSUM_NONE; } -/* Return tx queue pointer (find last set bit) according to causeTxDone reg */ +/* Return tx queue pointer (find last set bit) according to returned + * form tx_done reg. must not be null. The return value is always a + * valid queue for matching the first one found in . + */ static struct mvneta_tx_queue *mvneta_tx_done_policy(struct mvneta_port *pp, u32 cause) { int queue = fls(cause) - 1; - return (queue < 0 || queue >= txq_number) ? NULL : &pp->txqs[queue]; + return &pp->txqs[queue]; } /* Free tx queue skbuffs */ @@ -1261,35 +1382,51 @@ mvneta_txq_inc_get(txq); + if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr)) + dma_unmap_single(pp->dev->dev.parent, + tx_desc->buf_phys_addr, + tx_desc->data_size, DMA_TO_DEVICE); if (!skb) continue; - - dma_unmap_single(pp->dev->dev.parent, tx_desc->buf_phys_addr, - tx_desc->data_size, DMA_TO_DEVICE); dev_kfree_skb_any(skb); } } /* Handle end of transmission */ -static int mvneta_txq_done(struct mvneta_port *pp, +static void mvneta_txq_done(struct mvneta_port *pp, struct mvneta_tx_queue *txq) { struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id); int tx_done; tx_done = mvneta_txq_sent_desc_proc(pp, txq); - if (tx_done == 0) - return tx_done; + if (!tx_done) + return; + mvneta_txq_bufs_free(pp, txq, tx_done); txq->count -= tx_done; if (netif_tx_queue_stopped(nq)) { - if (txq->size - txq->count >= MAX_SKB_FRAGS + 1) + if (txq->count <= txq->tx_wake_threshold) netif_tx_wake_queue(nq); } +} - return tx_done; +static void *mvneta_frag_alloc(const struct mvneta_port *pp) +{ + if (likely(pp->frag_size <= PAGE_SIZE)) + return netdev_alloc_frag(pp->frag_size); + else + return kmalloc(pp->frag_size, GFP_ATOMIC); +} + +static void mvneta_frag_free(const struct mvneta_port *pp, void *data) +{ + if (likely(pp->frag_size <= PAGE_SIZE)) + skb_free_frag(data); + else + kfree(data); } /* Refill processing */ @@ -1298,22 +1435,21 @@ { dma_addr_t phys_addr; - struct sk_buff *skb; + void *data; - skb = netdev_alloc_skb(pp->dev, pp->pkt_size); - if (!skb) + data = mvneta_frag_alloc(pp); + if (!data) return -ENOMEM; - phys_addr = dma_map_single(pp->dev->dev.parent, skb->head, + phys_addr = dma_map_single(pp->dev->dev.parent, data, MVNETA_RX_BUF_SIZE(pp->pkt_size), DMA_FROM_DEVICE); if (unlikely(dma_mapping_error(pp->dev->dev.parent, phys_addr))) { - dev_kfree_skb(skb); + mvneta_frag_free(pp, data); return -ENOMEM; } - mvneta_rx_desc_fill(rx_desc, phys_addr, (u32)skb); - + mvneta_rx_desc_fill(rx_desc, phys_addr, (u32)data); return 0; } @@ -1322,15 +1458,16 @@ { if (skb->ip_summed == CHECKSUM_PARTIAL) { int ip_hdr_len = 0; + __be16 l3_proto = vlan_get_protocol(skb); u8 l4_proto; - if (skb->protocol == htons(ETH_P_IP)) { + if (l3_proto == htons(ETH_P_IP)) { struct iphdr *ip4h = ip_hdr(skb); /* Calculate IPv4 checksum and L4 checksum */ ip_hdr_len = ip4h->ihl; l4_proto = ip4h->protocol; - } else if (skb->protocol == htons(ETH_P_IPV6)) { + } else if (l3_proto == htons(ETH_P_IPV6)) { struct ipv6hdr *ip6h = ipv6_hdr(skb); /* Read l4_protocol from one of IPv6 extra headers */ @@ -1341,23 +1478,12 @@ return MVNETA_TX_L4_CSUM_NOT; return mvneta_txq_desc_csum(skb_network_offset(skb), - skb->protocol, ip_hdr_len, l4_proto); + l3_proto, ip_hdr_len, l4_proto); } return MVNETA_TX_L4_CSUM_NOT; } -/* Returns rx queue pointer (find last set bit) according to causeRxTx - * value - */ -static struct mvneta_rx_queue *mvneta_rx_policy(struct mvneta_port *pp, - u32 cause) -{ - int queue = fls(cause >> 8) - 1; - - return (queue < 0 || queue >= rxq_number) ? NULL : &pp->rxqs[queue]; -} - /* Drop packets received by the RXQ and free buffers */ static void mvneta_rxq_drop_pkts(struct mvneta_port *pp, struct mvneta_rx_queue *rxq) @@ -1367,11 +1493,11 @@ rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq); for (i = 0; i < rxq->size; i++) { struct mvneta_rx_desc *rx_desc = rxq->descs + i; - struct sk_buff *skb = (struct sk_buff *)rx_desc->buf_cookie; + void *data = (void *)rx_desc->buf_cookie; - dev_kfree_skb_any(skb); dma_unmap_single(pp->dev->dev.parent, rx_desc->buf_phys_addr, - rx_desc->data_size, DMA_FROM_DEVICE); + MVNETA_RX_BUF_SIZE(pp->pkt_size), DMA_FROM_DEVICE); + mvneta_frag_free(pp, data); } if (rx_done) @@ -1382,8 +1508,9 @@ static int mvneta_rx(struct mvneta_port *pp, int rx_todo, struct mvneta_rx_queue *rxq) { + struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports); struct net_device *dev = pp->dev; - int rx_done, rx_filled; + int rx_done; u32 rcvd_pkts = 0; u32 rcvd_bytes = 0; @@ -1394,55 +1521,88 @@ rx_todo = rx_done; rx_done = 0; - rx_filled = 0; /* Fairness NAPI loop */ while (rx_done < rx_todo) { struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); struct sk_buff *skb; + unsigned char *data; + dma_addr_t phys_addr; u32 rx_status; int rx_bytes, err; - prefetch(rx_desc); rx_done++; - rx_filled++; rx_status = rx_desc->status; - skb = (struct sk_buff *)rx_desc->buf_cookie; + rx_bytes = rx_desc->data_size - (ETH_FCS_LEN + MVNETA_MH_SIZE); + data = (unsigned char *)rx_desc->buf_cookie; + phys_addr = rx_desc->buf_phys_addr; - if (!mvneta_rxq_desc_is_first_last(rx_desc) || + if (!mvneta_rxq_desc_is_first_last(rx_status) || (rx_status & MVNETA_RXD_ERR_SUMMARY)) { + err_drop_frame: dev->stats.rx_errors++; mvneta_rx_error(pp, rx_desc); - mvneta_rx_desc_fill(rx_desc, rx_desc->buf_phys_addr, - (u32)skb); + /* leave the descriptor untouched */ continue; } - dma_unmap_single(pp->dev->dev.parent, rx_desc->buf_phys_addr, - rx_desc->data_size, DMA_FROM_DEVICE); + if (rx_bytes <= rx_copybreak) { + /* better copy a small frame and not unmap the DMA region */ + skb = netdev_alloc_skb_ip_align(dev, rx_bytes); + if (unlikely(!skb)) + goto err_drop_frame; + + dma_sync_single_range_for_cpu(dev->dev.parent, + rx_desc->buf_phys_addr, + MVNETA_MH_SIZE + NET_SKB_PAD, + rx_bytes, + DMA_FROM_DEVICE); + memcpy(skb_put(skb, rx_bytes), + data + MVNETA_MH_SIZE + NET_SKB_PAD, + rx_bytes); + + skb->protocol = eth_type_trans(skb, dev); + mvneta_rx_csum(pp, rx_status, skb); + napi_gro_receive(&port->napi, skb); + + rcvd_pkts++; + rcvd_bytes += rx_bytes; + + /* leave the descriptor and buffer untouched */ + continue; + } + + /* Refill processing */ + err = mvneta_rx_refill(pp, rx_desc); + if (err) { + netdev_err(dev, "Linux processing - Can't refill\n"); + rxq->missed++; + goto err_drop_frame; + } + + skb = build_skb(data, pp->frag_size > PAGE_SIZE ? 0 : pp->frag_size); + + /* After refill old buffer has to be unmapped regardless + * the skb is successfully built or not. + */ + dma_unmap_single(dev->dev.parent, phys_addr, + MVNETA_RX_BUF_SIZE(pp->pkt_size), DMA_FROM_DEVICE); + + if (!skb) + goto err_drop_frame; - rx_bytes = rx_desc->data_size - - (ETH_FCS_LEN + MVNETA_MH_SIZE); rcvd_pkts++; rcvd_bytes += rx_bytes; /* Linux processing */ - skb_reserve(skb, MVNETA_MH_SIZE); + skb_reserve(skb, MVNETA_MH_SIZE + NET_SKB_PAD); skb_put(skb, rx_bytes); skb->protocol = eth_type_trans(skb, dev); - mvneta_rx_csum(pp, rx_desc, skb); + mvneta_rx_csum(pp, rx_status, skb); - napi_gro_receive(&pp->napi, skb); - - /* Refill processing */ - err = mvneta_rx_refill(pp, rx_desc); - if (err) { - netdev_err(pp->dev, "Linux processing - Can't refill\n"); - rxq->missed++; - rx_filled--; - } + napi_gro_receive(&port->napi, skb); } if (rcvd_pkts) { @@ -1455,19 +1615,139 @@ } /* Update rxq management counters */ - mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_filled); + mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done); return rx_done; } +static inline void +mvneta_tso_put_hdr(struct sk_buff *skb, + struct mvneta_port *pp, struct mvneta_tx_queue *txq) +{ + struct mvneta_tx_desc *tx_desc; + int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + + txq->tx_skb[txq->txq_put_index] = NULL; + tx_desc = mvneta_txq_next_desc_get(txq); + tx_desc->data_size = hdr_len; + tx_desc->command = mvneta_skb_tx_csum(pp, skb); + tx_desc->command |= MVNETA_TXD_F_DESC; + tx_desc->buf_phys_addr = txq->tso_hdrs_phys + + txq->txq_put_index * TSO_HEADER_SIZE; + mvneta_txq_inc_put(txq); +} + +static inline int +mvneta_tso_put_data(struct net_device *dev, struct mvneta_tx_queue *txq, + struct sk_buff *skb, char *data, int size, + bool last_tcp, bool is_last) +{ + struct mvneta_tx_desc *tx_desc; + + tx_desc = mvneta_txq_next_desc_get(txq); + tx_desc->data_size = size; + tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, data, + size, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev->dev.parent, + tx_desc->buf_phys_addr))) { + mvneta_txq_desc_put(txq); + return -ENOMEM; + } + + tx_desc->command = 0; + txq->tx_skb[txq->txq_put_index] = NULL; + + if (last_tcp) { + /* last descriptor in the TCP packet */ + tx_desc->command = MVNETA_TXD_L_DESC; + + /* last descriptor in SKB */ + if (is_last) + txq->tx_skb[txq->txq_put_index] = skb; + } + mvneta_txq_inc_put(txq); + return 0; +} + +static int mvneta_tx_tso(struct sk_buff *skb, struct net_device *dev, + struct mvneta_tx_queue *txq) +{ + int total_len, data_left; + int desc_count = 0; + struct mvneta_port *pp = netdev_priv(dev); + struct tso_t tso; + int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + int i; + + /* Count needed descriptors */ + if ((txq->count + tso_count_descs(skb)) >= txq->size) + return 0; + + if (skb_headlen(skb) < (skb_transport_offset(skb) + tcp_hdrlen(skb))) { + pr_info("*** Is this even possible???!?!?\n"); + return 0; + } + + /* Initialize the TSO handler, and prepare the first payload */ + tso_start(skb, &tso); + + total_len = skb->len - hdr_len; + while (total_len > 0) { + char *hdr; + + data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len); + total_len -= data_left; + desc_count++; + + /* prepare packet headers: MAC + IP + TCP */ + hdr = txq->tso_hdrs + txq->txq_put_index * TSO_HEADER_SIZE; + tso_build_hdr(skb, hdr, &tso, data_left, total_len == 0); + + mvneta_tso_put_hdr(skb, pp, txq); + + while (data_left > 0) { + int size; + desc_count++; + + size = min_t(int, tso.size, data_left); + + if (mvneta_tso_put_data(dev, txq, skb, + tso.data, size, + size == data_left, + total_len == 0)) + goto err_release; + data_left -= size; + + tso_build_data(skb, &tso, size); + } + } + + return desc_count; + +err_release: + /* Release all used data descriptors; header descriptors must not + * be DMA-unmapped. + */ + for (i = desc_count - 1; i >= 0; i--) { + struct mvneta_tx_desc *tx_desc = txq->descs + i; + if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr)) + dma_unmap_single(pp->dev->dev.parent, + tx_desc->buf_phys_addr, + tx_desc->data_size, + DMA_TO_DEVICE); + mvneta_txq_desc_put(txq); + } + return 0; +} + /* Handle tx fragmentation processing */ static int mvneta_tx_frag_process(struct mvneta_port *pp, struct sk_buff *skb, struct mvneta_tx_queue *txq) { struct mvneta_tx_desc *tx_desc; - int i; + int i, nr_frags = skb_shinfo(skb)->nr_frags; - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + for (i = 0; i < nr_frags; i++) { skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; void *addr = page_address(frag->page.p) + frag->page_offset; @@ -1484,20 +1764,16 @@ goto error; } - if (i == (skb_shinfo(skb)->nr_frags - 1)) { + if (i == nr_frags - 1) { /* Last descriptor */ tx_desc->command = MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD; - txq->tx_skb[txq->txq_put_index] = skb; - - mvneta_txq_inc_put(txq); } else { /* Descriptor in the middle: Not First, Not Last */ tx_desc->command = 0; - txq->tx_skb[txq->txq_put_index] = NULL; - mvneta_txq_inc_put(txq); } + mvneta_txq_inc_put(txq); } return 0; @@ -1525,15 +1801,19 @@ u16 txq_id = skb_get_queue_mapping(skb); struct mvneta_tx_queue *txq = &pp->txqs[txq_id]; struct mvneta_tx_desc *tx_desc; - struct netdev_queue *nq; + int len = skb->len; int frags = 0; u32 tx_cmd; if (!netif_running(dev)) goto out; + if (skb_is_gso(skb)) { + frags = mvneta_tx_tso(skb, dev, txq); + goto out; + } + frags = skb_shinfo(skb)->nr_frags + 1; - nq = netdev_get_tx_queue(dev, txq_id); /* Get a descriptor for the first part of the packet */ tx_desc = mvneta_txq_next_desc_get(txq); @@ -1576,19 +1856,20 @@ } } - txq->count += frags; - mvneta_txq_pend_desc_add(pp, txq, frags); - - if (txq->size - txq->count < MAX_SKB_FRAGS + 1) - netif_tx_stop_queue(nq); - out: if (frags > 0) { struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); + struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id); + + txq->count += frags; + mvneta_txq_pend_desc_add(pp, txq, frags); + + if (txq->count >= txq->tx_stop_threshold) + netif_tx_stop_queue(nq); u64_stats_update_begin(&stats->syncp); stats->tx_packets++; - stats->tx_bytes += skb->len; + stats->tx_bytes += len; u64_stats_update_end(&stats->syncp); } else { dev->stats.tx_dropped++; @@ -1614,33 +1895,26 @@ txq->txq_get_index = 0; } -/* handle tx done - called from tx done timer callback */ -static u32 mvneta_tx_done_gbe(struct mvneta_port *pp, u32 cause_tx_done, - int *tx_todo) +/* Handle tx done - called in softirq context. The argument + * must be a valid cause according to MVNETA_TXQ_INTR_MASK_ALL. + */ +static void mvneta_tx_done_gbe(struct mvneta_port *pp, u32 cause_tx_done) { struct mvneta_tx_queue *txq; - u32 tx_done = 0; struct netdev_queue *nq; - *tx_todo = 0; - while (cause_tx_done != 0) { + while (cause_tx_done) { txq = mvneta_tx_done_policy(pp, cause_tx_done); - if (!txq) - break; nq = netdev_get_tx_queue(pp->dev, txq->id); __netif_tx_lock(nq, smp_processor_id()); - if (txq->count) { - tx_done += mvneta_txq_done(pp, txq); - *tx_todo += txq->count; - } + if (txq->count) + mvneta_txq_done(pp, txq); __netif_tx_unlock(nq); cause_tx_done &= ~((1 << txq->id)); } - - return tx_done; } /* Compute crc8 of the specified address, using a unique algorithm , @@ -1839,16 +2113,36 @@ /* Interrupt handling - the callback for request_irq() */ static irqreturn_t mvneta_isr(int irq, void *dev_id) { - struct mvneta_port *pp = (struct mvneta_port *)dev_id; + struct mvneta_pcpu_port *port = (struct mvneta_pcpu_port *)dev_id; - /* Mask all interrupts */ - mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0); - - napi_schedule(&pp->napi); + disable_percpu_irq(port->pp->dev->irq); + napi_schedule(&port->napi); return IRQ_HANDLED; } +static int mvneta_fixed_link_update(struct mvneta_port *pp, + struct phy_device *phy) +{ + struct fixed_phy_status status; + struct fixed_phy_status changed = {}; + u32 gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS); + + status.link = !!(gmac_stat & MVNETA_GMAC_LINK_UP); + if (gmac_stat & MVNETA_GMAC_SPEED_1000) + status.speed = SPEED_1000; + else if (gmac_stat & MVNETA_GMAC_SPEED_100) + status.speed = SPEED_100; + else + status.speed = SPEED_10; + status.duplex = !!(gmac_stat & MVNETA_GMAC_FULL_DUPLEX); + changed.link = 1; + changed.speed = 1; + changed.duplex = 1; + fixed_phy_update_state(phy, &status, &changed); + return 0; +} + /* NAPI handler * Bits 0 - 7 of the causeRxTx register indicate that are transmitted * packets on the corresponding TXQ (Bit 0 is for TX queue 1). @@ -1860,68 +2154,48 @@ { int rx_done = 0; u32 cause_rx_tx; - unsigned long flags; struct mvneta_port *pp = netdev_priv(napi->dev); + struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports); if (!netif_running(pp->dev)) { - napi_complete(napi); + napi_complete(&port->napi); return rx_done; } /* Read cause register */ - cause_rx_tx = mvreg_read(pp, MVNETA_INTR_NEW_CAUSE) & - (MVNETA_RX_INTR_MASK(rxq_number) | MVNETA_TX_INTR_MASK(txq_number)); + cause_rx_tx = mvreg_read(pp, MVNETA_INTR_NEW_CAUSE); + if (cause_rx_tx & MVNETA_MISCINTR_INTR_MASK) { + u32 cause_misc = mvreg_read(pp, MVNETA_INTR_MISC_CAUSE); + + mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0); + if (pp->use_inband_status && (cause_misc & + (MVNETA_CAUSE_PHY_STATUS_CHANGE | + MVNETA_CAUSE_LINK_CHANGE | + MVNETA_CAUSE_PSC_SYNC_CHANGE))) { + mvneta_fixed_link_update(pp, pp->phy_dev); + } + } /* Release Tx descriptors */ if (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL) { - int tx_todo = 0; - - mvneta_tx_done_gbe(pp, (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL), &tx_todo); + mvneta_tx_done_gbe(pp, (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL)); cause_rx_tx &= ~MVNETA_TX_INTR_MASK_ALL; } /* For the case where the last mvneta_poll did not process all * RX packets */ - cause_rx_tx |= pp->cause_rx_tx; - if (rxq_number > 1) { - while ((cause_rx_tx & MVNETA_RX_INTR_MASK_ALL) && (budget > 0)) { - int count; - struct mvneta_rx_queue *rxq; - /* get rx queue number from cause_rx_tx */ - rxq = mvneta_rx_policy(pp, cause_rx_tx); - if (!rxq) - break; - - /* process the packet in that rx queue */ - count = mvneta_rx(pp, budget, rxq); - rx_done += count; - budget -= count; - if (budget > 0) { - /* set off the rx bit of the - * corresponding bit in the cause rx - * tx register, so that next iteration - * will find the next rx queue where - * packets are received on - */ - cause_rx_tx &= ~((1 << rxq->id) << 8); - } - } - } else { - rx_done = mvneta_rx(pp, budget, &pp->rxqs[rxq_def]); - budget -= rx_done; - } + cause_rx_tx |= port->cause_rx_tx; + rx_done = mvneta_rx(pp, budget, &pp->rxqs[rxq_def]); + budget -= rx_done; if (budget > 0) { cause_rx_tx = 0; - napi_complete(napi); - local_irq_save(flags); - mvreg_write(pp, MVNETA_INTR_NEW_MASK, - MVNETA_RX_INTR_MASK(rxq_number) | MVNETA_TX_INTR_MASK(txq_number)); - local_irq_restore(flags); + napi_complete(&port->napi); + enable_percpu_irq(pp->dev->irq, 0); } - pp->cause_rx_tx = cause_rx_tx; + port->cause_rx_tx = cause_rx_tx; return rx_done; } @@ -1929,32 +2203,15 @@ static int mvneta_rxq_fill(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, int num) { - struct net_device *dev = pp->dev; int i; for (i = 0; i < num; i++) { - struct sk_buff *skb; - struct mvneta_rx_desc *rx_desc; - unsigned long phys_addr; - - skb = dev_alloc_skb(pp->pkt_size); - if (!skb) { - netdev_err(dev, "%s:rxq %d, %d of %d buffs filled\n", + memset(rxq->descs + i, 0, sizeof(struct mvneta_rx_desc)); + if (mvneta_rx_refill(pp, rxq->descs + i) != 0) { + netdev_err(pp->dev, "%s:rxq %d, %d of %d buffs filled\n", __func__, rxq->id, i, num); break; } - - rx_desc = rxq->descs + i; - memset(rx_desc, 0, sizeof(struct mvneta_rx_desc)); - phys_addr = dma_map_single(dev->dev.parent, skb->head, - MVNETA_RX_BUF_SIZE(pp->pkt_size), - DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev->dev.parent, phys_addr))) { - dev_kfree_skb(skb); - break; - } - - mvneta_rx_desc_fill(rx_desc, phys_addr, (u32)skb); } /* Add this number of RX descriptors as non occupied (ready to @@ -1970,7 +2227,7 @@ { int queue; - /* free the skb's in the hal tx ring */ + /* free the skb's in the tx ring */ for (queue = 0; queue < txq_number; queue++) mvneta_txq_done_force(pp, &pp->txqs[queue]); @@ -2048,6 +2305,14 @@ { txq->size = pp->tx_ring_size; + /* A queue must always have room for at least one skb. + * Therefore, stop the queue when the free entries reaches + * the maximum number of descriptors per skb. + */ + txq->tx_stop_threshold = txq->size - MVNETA_MAX_SKB_DESCS; + txq->tx_wake_threshold = txq->tx_stop_threshold / 2; + + /* Allocate memory for TX descriptors */ txq->descs = dma_alloc_coherent(pp->dev->dev.parent, txq->size * MVNETA_DESC_ALIGNED_SIZE, @@ -2076,6 +2341,18 @@ txq->descs, txq->descs_phys); return -ENOMEM; } + + /* Allocate DMA buffers for TSO MAC/IP/TCP headers */ + txq->tso_hdrs = dma_alloc_coherent(pp->dev->dev.parent, + txq->size * TSO_HEADER_SIZE, + &txq->tso_hdrs_phys, GFP_KERNEL); + if (txq->tso_hdrs == NULL) { + kfree(txq->tx_skb); + dma_free_coherent(pp->dev->dev.parent, + txq->size * MVNETA_DESC_ALIGNED_SIZE, + txq->descs, txq->descs_phys); + return -ENOMEM; + } mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal); return 0; @@ -2087,6 +2364,10 @@ { kfree(txq->tx_skb); + if (txq->tso_hdrs) + dma_free_coherent(pp->dev->dev.parent, + txq->size * TSO_HEADER_SIZE, + txq->tso_hdrs, txq->tso_hdrs_phys); if (txq->descs) dma_free_coherent(pp->dev->dev.parent, txq->size * MVNETA_DESC_ALIGNED_SIZE, @@ -2118,26 +2399,19 @@ /* Cleanup all Rx queues */ static void mvneta_cleanup_rxqs(struct mvneta_port *pp) { - int queue; - - for (queue = 0; queue < rxq_number; queue++) - mvneta_rxq_deinit(pp, &pp->rxqs[queue]); + mvneta_rxq_deinit(pp, &pp->rxqs[rxq_def]); } /* Init all Rx queues */ static int mvneta_setup_rxqs(struct mvneta_port *pp) { - int queue; - - for (queue = 0; queue < rxq_number; queue++) { - int err = mvneta_rxq_init(pp, &pp->rxqs[queue]); - if (err) { - netdev_err(pp->dev, "%s: can't create rxq=%d\n", - __func__, queue); - mvneta_cleanup_rxqs(pp); - return err; - } + int err = mvneta_rxq_init(pp, &pp->rxqs[rxq_def]); + if (err) { + netdev_err(pp->dev, "%s: can't create rxq=%d\n", + __func__, rxq_def); + mvneta_cleanup_rxqs(pp); + return err; } return 0; @@ -2163,6 +2437,8 @@ static void mvneta_start_dev(struct mvneta_port *pp) { + unsigned int cpu; + mvneta_max_rx_size_set(pp, pp->pkt_size); mvneta_txq_max_tx_size_set(pp, pp->pkt_size); @@ -2170,11 +2446,21 @@ mvneta_port_enable(pp); /* Enable polling on the port */ - napi_enable(&pp->napi); + for_each_present_cpu(cpu) { + struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); + + napi_enable(&port->napi); + } /* Unmask interrupts */ mvreg_write(pp, MVNETA_INTR_NEW_MASK, - MVNETA_RX_INTR_MASK(rxq_number) | MVNETA_TX_INTR_MASK(txq_number)); + MVNETA_RX_INTR_MASK(rxq_number) | + MVNETA_TX_INTR_MASK(txq_number) | + MVNETA_MISCINTR_INTR_MASK); + mvreg_write(pp, MVNETA_INTR_MISC_MASK, + MVNETA_CAUSE_PHY_STATUS_CHANGE | + MVNETA_CAUSE_LINK_CHANGE | + MVNETA_CAUSE_PSC_SYNC_CHANGE); phy_start(pp->phy_dev); netif_tx_start_all_queues(pp->dev); @@ -2182,9 +2468,15 @@ static void mvneta_stop_dev(struct mvneta_port *pp) { + unsigned int cpu; + phy_stop(pp->phy_dev); - napi_disable(&pp->napi); + for_each_present_cpu(cpu) { + struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); + + napi_disable(&port->napi); + } netif_carrier_off(pp->dev); @@ -2242,53 +2534,90 @@ dev->mtu = mtu; - if (!netif_running(dev)) + if (!netif_running(dev)) { + netdev_update_features(dev); return 0; + } /* The interface is running, so we have to force a - * reallocation of the RXQs + * reallocation of the queues */ mvneta_stop_dev(pp); mvneta_cleanup_txqs(pp); mvneta_cleanup_rxqs(pp); - pp->pkt_size = MVNETA_RX_PKT_SIZE(pp->dev->mtu); + pp->pkt_size = MVNETA_RX_PKT_SIZE(dev->mtu); + pp->frag_size = SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(pp->pkt_size)) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); ret = mvneta_setup_rxqs(pp); if (ret) { - netdev_err(pp->dev, "unable to setup rxqs after MTU change\n"); + netdev_err(dev, "unable to setup rxqs after MTU change\n"); return ret; } - mvneta_setup_txqs(pp); + ret = mvneta_setup_txqs(pp); + if (ret) { + netdev_err(dev, "unable to setup txqs after MTU change\n"); + return ret; + } mvneta_start_dev(pp); mvneta_port_up(pp); + netdev_update_features(dev); + return 0; } +static netdev_features_t mvneta_fix_features(struct net_device *dev, + netdev_features_t features) +{ + struct mvneta_port *pp = netdev_priv(dev); + + if (pp->tx_csum_limit && dev->mtu > pp->tx_csum_limit) { + features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO); + netdev_info(dev, + "Disable IP checksum for MTU greater than %dB\n", + pp->tx_csum_limit); + } + + return features; +} + +/* Get mac address */ +static void mvneta_get_mac_addr(struct mvneta_port *pp, unsigned char *addr) +{ + u32 mac_addr_l, mac_addr_h; + + mac_addr_l = mvreg_read(pp, MVNETA_MAC_ADDR_LOW); + mac_addr_h = mvreg_read(pp, MVNETA_MAC_ADDR_HIGH); + addr[0] = (mac_addr_h >> 24) & 0xFF; + addr[1] = (mac_addr_h >> 16) & 0xFF; + addr[2] = (mac_addr_h >> 8) & 0xFF; + addr[3] = mac_addr_h & 0xFF; + addr[4] = (mac_addr_l >> 8) & 0xFF; + addr[5] = mac_addr_l & 0xFF; +} + /* Handle setting mac address */ static int mvneta_set_mac_addr(struct net_device *dev, void *addr) { struct mvneta_port *pp = netdev_priv(dev); - u8 *mac = addr + 2; - int i; - - if (netif_running(dev)) - return -EBUSY; + struct sockaddr *sockaddr = addr; + int ret; + ret = eth_prepare_mac_addr_change(dev, addr); + if (ret < 0) + return ret; /* Remove previous address table entry */ mvneta_mac_addr_set(pp, dev->dev_addr, -1); /* Set new addr in hw */ - mvneta_mac_addr_set(pp, mac, rxq_def); - - /* Set addr in the device */ - for (i = 0; i < ETH_ALEN; i++) - dev->dev_addr[i] = mac[i]; + mvneta_mac_addr_set(pp, sockaddr->sa_data, rxq_def); + eth_commit_mac_addr_change(dev, addr); return 0; } @@ -2306,9 +2635,7 @@ val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); val &= ~(MVNETA_GMAC_CONFIG_MII_SPEED | MVNETA_GMAC_CONFIG_GMII_SPEED | - MVNETA_GMAC_CONFIG_FULL_DUPLEX | - MVNETA_GMAC_AN_SPEED_EN | - MVNETA_GMAC_AN_DUPLEX_EN); + MVNETA_GMAC_CONFIG_FULL_DUPLEX); if (phydev->duplex) val |= MVNETA_GMAC_CONFIG_FULL_DUPLEX; @@ -2337,16 +2664,27 @@ if (status_change) { if (phydev->link) { - u32 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); - val |= (MVNETA_GMAC_FORCE_LINK_PASS | - MVNETA_GMAC_FORCE_LINK_DOWN); - mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); + if (!pp->use_inband_status) { + u32 val = mvreg_read(pp, + MVNETA_GMAC_AUTONEG_CONFIG); + val &= ~MVNETA_GMAC_FORCE_LINK_DOWN; + val |= MVNETA_GMAC_FORCE_LINK_PASS; + mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, + val); + } mvneta_port_up(pp); - netdev_info(pp->dev, "link up\n"); } else { + if (!pp->use_inband_status) { + u32 val = mvreg_read(pp, + MVNETA_GMAC_AUTONEG_CONFIG); + val &= ~MVNETA_GMAC_FORCE_LINK_PASS; + val |= MVNETA_GMAC_FORCE_LINK_DOWN; + mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, + val); + } mvneta_port_down(pp); - netdev_info(pp->dev, "link down\n"); } + phy_print_status(phydev); } } @@ -2378,14 +2716,133 @@ pp->phy_dev = NULL; } +static void mvneta_percpu_enable(void *arg) +{ + struct mvneta_port *pp = arg; + + enable_percpu_irq(pp->dev->irq, IRQ_TYPE_NONE); +} + +static void mvneta_percpu_disable(void *arg) +{ + struct mvneta_port *pp = arg; + + disable_percpu_irq(pp->dev->irq); +} + +static void mvneta_percpu_elect(struct mvneta_port *pp) +{ + int online_cpu_idx, cpu, i = 0; + + online_cpu_idx = rxq_def % num_online_cpus(); + + for_each_online_cpu(cpu) { + if (i == online_cpu_idx) + /* Enable per-CPU interrupt on the one CPU we + * just elected + */ + smp_call_function_single(cpu, mvneta_percpu_enable, + pp, true); + else + /* Disable per-CPU interrupt on all the other CPU */ + smp_call_function_single(cpu, mvneta_percpu_disable, + pp, true); + i++; + } +}; + +static int mvneta_percpu_notifier(struct notifier_block *nfb, + unsigned long action, void *hcpu) +{ + struct mvneta_port *pp = container_of(nfb, struct mvneta_port, + cpu_notifier); + int cpu = (unsigned long)hcpu, other_cpu; + struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); + + switch (action) { + case CPU_ONLINE: + case CPU_ONLINE_FROZEN: + netif_tx_stop_all_queues(pp->dev); + + /* We have to synchronise on tha napi of each CPU + * except the one just being waked up + */ + for_each_online_cpu(other_cpu) { + if (other_cpu != cpu) { + struct mvneta_pcpu_port *other_port = + per_cpu_ptr(pp->ports, other_cpu); + + napi_synchronize(&other_port->napi); + } + } + + /* Mask all ethernet port interrupts */ + mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0); + mvreg_write(pp, MVNETA_INTR_OLD_MASK, 0); + mvreg_write(pp, MVNETA_INTR_MISC_MASK, 0); + napi_enable(&port->napi); + + /* Enable per-CPU interrupt on the one CPU we care + * about. + */ + mvneta_percpu_elect(pp); + + /* Unmask all ethernet port interrupts */ + mvreg_write(pp, MVNETA_INTR_NEW_MASK, + MVNETA_RX_INTR_MASK(rxq_number) | + MVNETA_TX_INTR_MASK(txq_number) | + MVNETA_MISCINTR_INTR_MASK); + mvreg_write(pp, MVNETA_INTR_MISC_MASK, + MVNETA_CAUSE_PHY_STATUS_CHANGE | + MVNETA_CAUSE_LINK_CHANGE | + MVNETA_CAUSE_PSC_SYNC_CHANGE); + netif_tx_start_all_queues(pp->dev); + break; + case CPU_DOWN_PREPARE: + case CPU_DOWN_PREPARE_FROZEN: + netif_tx_stop_all_queues(pp->dev); + /* Mask all ethernet port interrupts */ + mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0); + mvreg_write(pp, MVNETA_INTR_OLD_MASK, 0); + mvreg_write(pp, MVNETA_INTR_MISC_MASK, 0); + + napi_synchronize(&port->napi); + napi_disable(&port->napi); + /* Disable per-CPU interrupts on the CPU that is + * brought down. + */ + smp_call_function_single(cpu, mvneta_percpu_disable, + pp, true); + + break; + case CPU_DEAD: + case CPU_DEAD_FROZEN: + /* Check if a new CPU must be elected now this on is down */ + mvneta_percpu_elect(pp); + /* Unmask all ethernet port interrupts */ + mvreg_write(pp, MVNETA_INTR_NEW_MASK, + MVNETA_RX_INTR_MASK(rxq_number) | + MVNETA_TX_INTR_MASK(txq_number) | + MVNETA_MISCINTR_INTR_MASK); + mvreg_write(pp, MVNETA_INTR_MISC_MASK, + MVNETA_CAUSE_PHY_STATUS_CHANGE | + MVNETA_CAUSE_LINK_CHANGE | + MVNETA_CAUSE_PSC_SYNC_CHANGE); + netif_tx_start_all_queues(pp->dev); + break; + } + + return NOTIFY_OK; +} + static int mvneta_open(struct net_device *dev) { struct mvneta_port *pp = netdev_priv(dev); int ret; - mvneta_mac_addr_set(pp, dev->dev_addr, rxq_def); - pp->pkt_size = MVNETA_RX_PKT_SIZE(pp->dev->mtu); + pp->frag_size = SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(pp->pkt_size)) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); ret = mvneta_setup_rxqs(pp); if (ret) @@ -2396,13 +2853,29 @@ goto err_cleanup_rxqs; /* Connect to port interrupt line */ - ret = request_irq(pp->dev->irq, mvneta_isr, 0, - MVNETA_DRIVER_NAME, pp); + ret = request_percpu_irq(pp->dev->irq, mvneta_isr, + MVNETA_DRIVER_NAME, pp->ports); if (ret) { netdev_err(pp->dev, "cannot request irq %d\n", pp->dev->irq); goto err_cleanup_txqs; } + /* Even though the documentation says that request_percpu_irq + * doesn't enable the interrupts automatically, it actually + * does so on the local CPU. + * + * Make sure it's disabled. + */ + mvneta_percpu_disable(pp); + + /* Elect a CPU to handle our RX queue interrupt */ + mvneta_percpu_elect(pp); + + /* Register a CPU notifier to handle the case where our CPU + * might be taken offline. + */ + register_cpu_notifier(&pp->cpu_notifier); + /* In default link is down */ netif_carrier_off(pp->dev); @@ -2417,7 +2890,7 @@ return 0; err_free_irq: - free_irq(pp->dev->irq, pp); + free_percpu_irq(pp->dev->irq, pp->ports); err_cleanup_txqs: mvneta_cleanup_txqs(pp); err_cleanup_rxqs: @@ -2429,16 +2902,30 @@ static int mvneta_stop(struct net_device *dev) { struct mvneta_port *pp = netdev_priv(dev); + int cpu; mvneta_stop_dev(pp); mvneta_mdio_remove(pp); - free_irq(dev->irq, pp); + unregister_cpu_notifier(&pp->cpu_notifier); + for_each_present_cpu(cpu) + smp_call_function_single(cpu, mvneta_percpu_disable, pp, true); + free_percpu_irq(dev->irq, pp->ports); mvneta_cleanup_rxqs(pp); mvneta_cleanup_txqs(pp); return 0; } +static int mvneta_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) +{ + struct mvneta_port *pp = netdev_priv(dev); + + if (!pp->phy_dev) + return -ENOTSUPP; + + return phy_mii_ioctl(pp->phy_dev, ifr, cmd); +} + /* Ethtool methods */ /* Get settings (phy address, speed) for ethtools */ @@ -2533,8 +3020,12 @@ return -EINVAL; pp->rx_ring_size = ring->rx_pending < MVNETA_MAX_RXD ? ring->rx_pending : MVNETA_MAX_RXD; - pp->tx_ring_size = ring->tx_pending < MVNETA_MAX_TXD ? - ring->tx_pending : MVNETA_MAX_TXD; + + pp->tx_ring_size = clamp_t(u16, ring->tx_pending, + MVNETA_MAX_SKB_DESCS * 2, MVNETA_MAX_TXD); + if (pp->tx_ring_size != ring->tx_pending) + netdev_warn(dev, "TX queue size set to %u (requested %u)\n", + pp->tx_ring_size, ring->tx_pending); if (netif_running(dev)) { mvneta_stop(dev); @@ -2548,6 +3039,65 @@ return 0; } +static void mvneta_ethtool_get_strings(struct net_device *netdev, u32 sset, + u8 *data) +{ + if (sset == ETH_SS_STATS) { + int i; + + for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++) + memcpy(data + i * ETH_GSTRING_LEN, + mvneta_statistics[i].name, ETH_GSTRING_LEN); + } +} + +static void mvneta_ethtool_update_stats(struct mvneta_port *pp) +{ + const struct mvneta_statistic *s; + void __iomem *base = pp->base; + u32 high, low, val; + int i; + + for (i = 0, s = mvneta_statistics; + s < mvneta_statistics + ARRAY_SIZE(mvneta_statistics); + s++, i++) { + val = 0; + + switch (s->type) { + case T_REG_32: + val = readl_relaxed(base + s->offset); + break; + case T_REG_64: + /* Docs say to read low 32-bit then high */ + low = readl_relaxed(base + s->offset); + high = readl_relaxed(base + s->offset + 4); + val = (u64)high << 32 | low; + break; + } + + pp->ethtool_stats[i] += val; + } +} + +static void mvneta_ethtool_get_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct mvneta_port *pp = netdev_priv(dev); + int i; + + mvneta_ethtool_update_stats(pp); + + for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++) + *data++ = pp->ethtool_stats[i]; +} + +static int mvneta_ethtool_get_sset_count(struct net_device *dev, int sset) +{ + if (sset == ETH_SS_STATS) + return ARRAY_SIZE(mvneta_statistics); + return -EOPNOTSUPP; +} + static const struct net_device_ops mvneta_netdev_ops = { .ndo_open = mvneta_open, .ndo_stop = mvneta_stop, @@ -2555,7 +3105,9 @@ .ndo_set_rx_mode = mvneta_set_rx_mode, .ndo_set_mac_address = mvneta_set_mac_addr, .ndo_change_mtu = mvneta_change_mtu, + .ndo_fix_features = mvneta_fix_features, .ndo_get_stats64 = mvneta_get_stats64, + .ndo_do_ioctl = mvneta_ioctl, }; const struct ethtool_ops mvneta_eth_tool_ops = { @@ -2567,10 +3119,13 @@ .get_drvinfo = mvneta_ethtool_get_drvinfo, .get_ringparam = mvneta_ethtool_get_ringparam, .set_ringparam = mvneta_ethtool_set_ringparam, + .get_strings = mvneta_ethtool_get_strings, + .get_ethtool_stats = mvneta_ethtool_get_stats, + .get_sset_count = mvneta_ethtool_get_sset_count, }; /* Initialize hw */ -static int mvneta_init(struct mvneta_port *pp, int phy_addr) +static int mvneta_init(struct device *dev, struct mvneta_port *pp) { int queue; @@ -2580,8 +3135,8 @@ /* Set port default values */ mvneta_defaults_set(pp); - pp->txqs = kzalloc(txq_number * sizeof(struct mvneta_tx_queue), - GFP_KERNEL); + pp->txqs = devm_kcalloc(dev, txq_number, sizeof(struct mvneta_tx_queue), + GFP_KERNEL); if (!pp->txqs) return -ENOMEM; @@ -2593,12 +3148,10 @@ txq->done_pkts_coal = MVNETA_TXDONE_COAL_PKTS; } - pp->rxqs = kzalloc(rxq_number * sizeof(struct mvneta_rx_queue), - GFP_KERNEL); - if (!pp->rxqs) { - kfree(pp->txqs); + pp->rxqs = devm_kcalloc(dev, rxq_number, sizeof(struct mvneta_rx_queue), + GFP_KERNEL); + if (!pp->rxqs) return -ENOMEM; - } /* Create Rx descriptor rings */ for (queue = 0; queue < rxq_number; queue++) { @@ -2612,12 +3165,6 @@ return 0; } -static void mvneta_deinit(struct mvneta_port *pp) -{ - kfree(pp->txqs); - kfree(pp->rxqs); -} - /* platform glue : initialize decoding windows */ static void mvneta_conf_mbus_windows(struct mvneta_port *pp, const struct mbus_dram_target_info *dram) @@ -2650,51 +3197,70 @@ } mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable); + mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect); } /* Power up the port */ -static void mvneta_port_power_up(struct mvneta_port *pp, int phy_mode) +static int mvneta_port_power_up(struct mvneta_port *pp, int phy_mode) { - u32 val; + u32 ctrl; /* MAC Cause register should be cleared */ mvreg_write(pp, MVNETA_UNIT_INTR_CAUSE, 0); - if (phy_mode == PHY_INTERFACE_MODE_SGMII) - mvneta_port_sgmii_config(pp); + ctrl = mvreg_read(pp, MVNETA_GMAC_CTRL_2); - mvneta_gmac_rgmii_set(pp, 1); + /* Even though it might look weird, when we're configured in + * SGMII or QSGMII mode, the RGMII bit needs to be set. + */ + switch(phy_mode) { + case PHY_INTERFACE_MODE_QSGMII: + mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_QSGMII_SERDES_PROTO); + ctrl |= MVNETA_GMAC2_PCS_ENABLE | MVNETA_GMAC2_PORT_RGMII; + break; + case PHY_INTERFACE_MODE_SGMII: + mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_SGMII_SERDES_PROTO); + ctrl |= MVNETA_GMAC2_PCS_ENABLE | MVNETA_GMAC2_PORT_RGMII; + break; + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + ctrl |= MVNETA_GMAC2_PORT_RGMII; + break; + default: + return -EINVAL; + } + + if (pp->use_inband_status) + ctrl |= MVNETA_GMAC2_INBAND_AN_ENABLE; /* Cancel Port Reset */ - val = mvreg_read(pp, MVNETA_GMAC_CTRL_2); - val &= ~MVNETA_GMAC2_PORT_RESET; - mvreg_write(pp, MVNETA_GMAC_CTRL_2, val); + ctrl &= ~MVNETA_GMAC2_PORT_RESET; + mvreg_write(pp, MVNETA_GMAC_CTRL_2, ctrl); while ((mvreg_read(pp, MVNETA_GMAC_CTRL_2) & MVNETA_GMAC2_PORT_RESET) != 0) continue; + + return 0; } /* Device initialization routine */ static int mvneta_probe(struct platform_device *pdev) { const struct mbus_dram_target_info *dram_target_info; + struct resource *res; struct device_node *dn = pdev->dev.of_node; struct device_node *phy_node; - u32 phy_addr; struct mvneta_port *pp; struct net_device *dev; - const char *mac_addr; + const char *dt_mac_addr; + char hw_mac_addr[ETH_ALEN]; + const char *mac_from; + const char *managed; + int tx_csum_limit; int phy_mode; int err; - - /* Our multiqueue support is not complete, so for now, only - * allow the usage of the first RX queue - */ - if (rxq_def != 0) { - dev_err(&pdev->dev, "Invalid rxq_def argument: %d\n", rxq_def); - return -EINVAL; - } + int cpu; dev = alloc_etherdev_mqs(sizeof(struct mvneta_port), txq_number, rxq_number); if (!dev) @@ -2708,102 +3274,168 @@ phy_node = of_parse_phandle(dn, "phy", 0); if (!phy_node) { - dev_err(&pdev->dev, "no associated PHY\n"); - err = -ENODEV; - goto err_free_irq; + if (!of_phy_is_fixed_link(dn)) { + dev_err(&pdev->dev, "no PHY specified\n"); + err = -ENODEV; + goto err_free_irq; + } + + err = of_phy_register_fixed_link(dn); + if (err < 0) { + dev_err(&pdev->dev, "cannot register fixed PHY\n"); + goto err_free_irq; + } + + /* In the case of a fixed PHY, the DT node associated + * to the PHY is the Ethernet MAC DT node. + */ + phy_node = of_node_get(dn); } phy_mode = of_get_phy_mode(dn); if (phy_mode < 0) { dev_err(&pdev->dev, "incorrect phy-mode\n"); err = -EINVAL; - goto err_free_irq; + goto err_put_phy_node; } - mac_addr = of_get_mac_address(dn); - - if (!mac_addr || !is_valid_ether_addr(mac_addr)) - eth_hw_addr_random(dev); - else - memcpy(dev->dev_addr, mac_addr, ETH_ALEN); - dev->tx_queue_len = MVNETA_MAX_TXD; dev->watchdog_timeo = 5 * HZ; dev->netdev_ops = &mvneta_netdev_ops; - SET_ETHTOOL_OPS(dev, &mvneta_eth_tool_ops); + dev->ethtool_ops = &mvneta_eth_tool_ops; pp = netdev_priv(dev); - - pp->weight = MVNETA_RX_POLL_WEIGHT; pp->phy_node = phy_node; pp->phy_interface = phy_mode; - pp->base = of_iomap(dn, 0); - if (pp->base == NULL) { - err = -ENOMEM; - goto err_free_irq; - } + err = of_property_read_string(dn, "managed", &managed); + pp->use_inband_status = (err == 0 && + strcmp(managed, "in-band-status") == 0); + pp->cpu_notifier.notifier_call = mvneta_percpu_notifier; pp->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(pp->clk)) { err = PTR_ERR(pp->clk); - goto err_unmap; + goto err_put_phy_node; } clk_prepare_enable(pp->clk); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + pp->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(pp->base)) { + err = PTR_ERR(pp->base); + goto err_clk; + } + + /* Alloc per-cpu port structure */ + pp->ports = alloc_percpu(struct mvneta_pcpu_port); + if (!pp->ports) { + err = -ENOMEM; + goto err_clk; + } + /* Alloc per-cpu stats */ - pp->stats = alloc_percpu(struct mvneta_pcpu_stats); + pp->stats = netdev_alloc_pcpu_stats(struct mvneta_pcpu_stats); if (!pp->stats) { err = -ENOMEM; - goto err_clk; + goto err_free_ports; } + dt_mac_addr = of_get_mac_address(dn); + if (dt_mac_addr) { + mac_from = "device tree"; + memcpy(dev->dev_addr, dt_mac_addr, ETH_ALEN); + } else { + mvneta_get_mac_addr(pp, hw_mac_addr); + if (is_valid_ether_addr(hw_mac_addr)) { + mac_from = "hardware"; + memcpy(dev->dev_addr, hw_mac_addr, ETH_ALEN); + } else { + mac_from = "random"; + eth_hw_addr_random(dev); + } + } + + if (!of_property_read_u32(dn, "tx-csum-limit", &tx_csum_limit)) { + if (tx_csum_limit < 0 || + tx_csum_limit > MVNETA_TX_CSUM_MAX_SIZE) { + tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE; + dev_info(&pdev->dev, + "Wrong TX csum limit in DT, set to %dB\n", + MVNETA_TX_CSUM_DEF_SIZE); + } + } else if (of_device_is_compatible(dn, "marvell,armada-370-neta")) { + tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE; + } else { + tx_csum_limit = MVNETA_TX_CSUM_MAX_SIZE; + } + + pp->tx_csum_limit = tx_csum_limit; + pp->tx_ring_size = MVNETA_MAX_TXD; pp->rx_ring_size = MVNETA_MAX_RXD; pp->dev = dev; SET_NETDEV_DEV(dev, &pdev->dev); - err = mvneta_init(pp, phy_addr); + err = mvneta_init(&pdev->dev, pp); + if (err < 0) + goto err_free_stats; + + err = mvneta_port_power_up(pp, phy_mode); if (err < 0) { - dev_err(&pdev->dev, "can't init eth hal\n"); + dev_err(&pdev->dev, "can't power up port\n"); goto err_free_stats; } - mvneta_port_power_up(pp, phy_mode); dram_target_info = mv_mbus_dram_info(); if (dram_target_info) mvneta_conf_mbus_windows(pp, dram_target_info); - netif_napi_add(dev, &pp->napi, mvneta_poll, pp->weight); + for_each_present_cpu(cpu) { + struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); + + netif_napi_add(dev, &port->napi, mvneta_poll, NAPI_POLL_WEIGHT); + port->pp = pp; + } - dev->features = NETIF_F_SG | NETIF_F_IP_CSUM; - dev->hw_features |= NETIF_F_SG | NETIF_F_IP_CSUM; - dev->vlan_features |= NETIF_F_SG | NETIF_F_IP_CSUM; - dev->priv_flags |= IFF_UNICAST_FLT; + dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO; + dev->hw_features |= dev->features; + dev->vlan_features |= dev->features; + dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE; + dev->gso_max_segs = MVNETA_MAX_TSO_SEGS; err = register_netdev(dev); if (err < 0) { dev_err(&pdev->dev, "failed to register\n"); - goto err_deinit; + goto err_free_stats; } - netdev_info(dev, "mac: %pM\n", dev->dev_addr); + netdev_info(dev, "Using %s mac address %pM\n", mac_from, + dev->dev_addr); platform_set_drvdata(pdev, pp->dev); + if (pp->use_inband_status) { + struct phy_device *phy = of_phy_find_device(dn); + + mvneta_fixed_link_update(pp, phy); + + put_device(&phy->dev); + } + return 0; -err_deinit: - mvneta_deinit(pp); err_free_stats: free_percpu(pp->stats); +err_free_ports: + free_percpu(pp->ports); err_clk: clk_disable_unprepare(pp->clk); -err_unmap: - iounmap(pp->base); +err_put_phy_node: + of_node_put(phy_node); err_free_irq: irq_dispose_mapping(dev->irq); err_free_netdev: @@ -2818,20 +3450,19 @@ struct mvneta_port *pp = netdev_priv(dev); unregister_netdev(dev); - mvneta_deinit(pp); clk_disable_unprepare(pp->clk); + free_percpu(pp->ports); free_percpu(pp->stats); - iounmap(pp->base); irq_dispose_mapping(dev->irq); + of_node_put(pp->phy_node); free_netdev(dev); - platform_set_drvdata(pdev, NULL); - return 0; } static const struct of_device_id mvneta_match[] = { { .compatible = "marvell,armada-370-neta" }, + { .compatible = "marvell,armada-xp-neta" }, { } }; MODULE_DEVICE_TABLE(of, mvneta_match); @@ -2855,3 +3486,4 @@ module_param(txq_number, int, S_IRUGO); module_param(rxq_def, int, S_IRUGO); +module_param(rx_copybreak, int, S_IRUGO | S_IWUSR);