--- zzzz-none-000/linux-5.4.213/include/linux/netdevice.h 2022-09-15 10:04:56.000000000 +0000 +++ miami-7690-761/linux-5.4.213/include/linux/netdevice.h 2024-05-29 11:20:02.000000000 +0000 @@ -49,6 +49,16 @@ #include #include + +/* Do not include avm_pa.h to avoid include cycle. Since we can't determine + * the size here we reserve some space, including some extra space for future + * extensions without having to rebuild the world. + * + * avm_pa contains a BUILD_BUG_ON() in case the reserve is too small. + */ +#define AVM_PA_DEV_INFO_DEV_RESERVE 32 +struct avm_pa_dev_info; + struct netpoll_info; struct device; struct phy_device; @@ -138,8 +148,8 @@ #if defined(CONFIG_HYPERV_NET) # define LL_MAX_HEADER 128 -#elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25) -# if defined(CONFIG_MAC80211_MESH) +#elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25) || 1 +# if defined(CONFIG_MAC80211_MESH) || 1 # define LL_MAX_HEADER 128 # else # define LL_MAX_HEADER 96 @@ -186,7 +196,6 @@ unsigned long tx_compressed; }; - #include #include @@ -288,7 +297,6 @@ __LINK_STATE_DORMANT, }; - /* * This structure holds boot-time configured netdevice settings. They * are then used in the device probing. @@ -340,6 +348,7 @@ struct list_head dev_list; struct hlist_node napi_hash_node; unsigned int napi_id; + struct work_struct work; }; enum { @@ -350,6 +359,7 @@ NAPI_STATE_HASHED, /* In NAPI hash (busy polling possible) */ NAPI_STATE_NO_BUSY_POLL,/* Do not add in napi_hash, no busy polling */ NAPI_STATE_IN_BUSY_POLL,/* sk_busy_loop() owns this NAPI */ + NAPI_STATE_THREADED, /* Use threaded NAPI */ }; enum { @@ -360,6 +370,7 @@ NAPIF_STATE_HASHED = BIT(NAPI_STATE_HASHED), NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL), NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL), + NAPIF_STATE_THREADED = BIT(NAPI_STATE_THREADED), }; enum gro_result { @@ -927,6 +938,13 @@ struct devlink; struct tlsdev_ops; +struct flow_offload; +struct flow_offload_hw_path; + +enum flow_offload_type { + FLOW_OFFLOAD_ADD = 0, + FLOW_OFFLOAD_DEL, +}; /* * This structure defines the management hooks for network devices. @@ -1160,6 +1178,17 @@ * int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh, * u16 flags); * + * int (*ndo_flow_offload_check)(struct flow_offload_hw_path *path); + * For virtual devices like bridges, vlan, and pppoe, fill in the + * underlying network device that can be used for offloading connections. + * Return an error if offloading is not supported. + * + * int (*ndo_flow_offload)(enum flow_offload_type type, + * struct flow_offload *flow, + * struct flow_offload_hw_path *src, + * struct flow_offload_hw_path *dest); + * Adds/deletes flow entry to/from net device flowtable. + * * int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier); * Called to change device carrier. Soft-devices (like dummy, team, etc) * which do not represent real hardware may define this to allow their @@ -1407,6 +1436,11 @@ int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh, u16 flags); + int (*ndo_flow_offload_check)(struct flow_offload_hw_path *path); + int (*ndo_flow_offload)(enum flow_offload_type type, + struct flow_offload *flow, + struct flow_offload_hw_path *src, + struct flow_offload_hw_path *dest); int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier); int (*ndo_get_phys_port_id)(struct net_device *dev, @@ -1522,6 +1556,36 @@ IFF_FAILOVER_SLAVE = 1<<28, IFF_L3MDEV_RX_HANDLER = 1<<29, IFF_LIVE_RENAME_OK = 1<<30, + IFF_NO_IP_ALIGN = 1<<31, +}; + +/** + * enum netdev_priv_flags_ext - &struct net_device priv_flags_ext + * + * These flags are used to check for device type and can be + * set and used by the drivers + * + * @IFF_EXT_TUN_TAP: device is a TUN/TAP device + * @IFF_EXT_PPP_L2TPV2: device is a L2TPV2 device + * @IFF_EXT_PPP_L2TPV3: device is a L2TPV3 PPP device + * @IFF_EXT_PPP_PPTP: device is a PPTP device + * @IFF_EXT_GRE_V4_TAP: device is a GRE IPv4 TAP device + * @IFF_EXT_GRE_V6_TAP: device is a GRE IPv6 TAP device + * @IFF_EXT_IFB: device is an IFB device + * @IFF_EXT_MAPT: device is an MAPT device + * @IFF_EXT_L2TPV3: device is a L2TPV3 Ethernet device + */ +enum netdev_priv_flags_ext { + IFF_EXT_TUN_TAP = 1<<0, + IFF_EXT_PPP_L2TPV2 = 1<<1, + IFF_EXT_PPP_L2TPV3 = 1<<2, + IFF_EXT_PPP_PPTP = 1<<3, + IFF_EXT_GRE_V4_TAP = 1<<4, + IFF_EXT_GRE_V6_TAP = 1<<5, + IFF_EXT_IFB = 1<<6, + IFF_EXT_MAPT = 1<<7, + IFF_EXT_HW_NO_OFFLOAD = 1<<8, + IFF_EXT_ETH_L2TPV3 = 1<<9, }; #define IFF_802_1Q_VLAN IFF_802_1Q_VLAN @@ -1554,6 +1618,7 @@ #define IFF_FAILOVER_SLAVE IFF_FAILOVER_SLAVE #define IFF_L3MDEV_RX_HANDLER IFF_L3MDEV_RX_HANDLER #define IFF_LIVE_RENAME_OK IFF_LIVE_RENAME_OK +#define IFF_NO_IP_ALIGN IFF_NO_IP_ALIGN /* Specifies the type of the struct net_device::ml_priv pointer */ enum netdev_ml_priv_type { @@ -1633,6 +1698,8 @@ * @flags: Interface flags (a la BSD) * @priv_flags: Like 'flags' but invisible to userspace, * see if.h for the definitions + * @priv_flags_ext: Extension for 'priv_flags' + * * @gflags: Global flags ( kept as legacy ) * @padded: How much padding added by alloc_netdev() * @operstate: RFC2863 operstate @@ -1744,6 +1811,9 @@ * @dstats: Dummy statistics * @vstats: Virtual ethernet statistics * + * @sawf_flags: Service-aware Wi-Fi flags + * @sawf_stats: Service-aware Wi-Fi latency statistics. + * * @garp_port: GARP * @mrp_port: MRP * @@ -1862,10 +1932,16 @@ const struct tlsdev_ops *tlsdev_ops; #endif +#ifdef CONFIG_ETHERNET_PACKET_MANGLE + void (*eth_mangle_rx)(struct net_device *dev, struct sk_buff *skb); + struct sk_buff *(*eth_mangle_tx)(struct net_device *dev, struct sk_buff *skb); +#endif + const struct header_ops *header_ops; unsigned int flags; unsigned int priv_flags; + unsigned int priv_flags_ext; unsigned short gflags; unsigned short padded; @@ -1907,13 +1983,14 @@ struct netdev_hw_addr_list mc; struct netdev_hw_addr_list dev_addrs; + unsigned char local_addr_mask[MAX_ADDR_LEN]; + #ifdef CONFIG_SYSFS struct kset *queues_kset; #endif unsigned int promiscuity; unsigned int allmulti; - /* Protocol-specific pointers */ #if IS_ENABLED(CONFIG_VLAN_8021Q) @@ -1942,6 +2019,10 @@ struct mpls_dev __rcu *mpls_ptr; #endif +#ifdef CONFIG_ETHERNET_PACKET_MANGLE + void *phy_ptr; /* PHY device specific data */ +#endif + /* * Cache lines mostly used on receive path (including eth_type_trans()) */ @@ -2035,6 +2116,16 @@ struct pcpu_dstats __percpu *dstats; }; +#ifdef CONFIG_AVM_PA + u8 avm_pa[AVM_PA_DEV_INFO_DEV_RESERVE]; +#define AVM_PA_DEVINFO(dev) ((struct avm_pa_dev_info *) (dev)->avm_pa) +#else +#define AVM_PA_DEVINFO(dev) ((struct avm_pa_dev_info *) NULL) +#endif + + uint16_t sawf_flags; + struct pcpu_sawf_stats __percpu *sawf_stats; + #if IS_ENABLED(CONFIG_GARP) struct garp_port __rcu *garp_port; #endif @@ -2255,6 +2346,26 @@ int (*poll)(struct napi_struct *, int), int weight); /** + * netif_threaded_napi_add - initialize a NAPI context + * @dev: network device + * @napi: NAPI context + * @poll: polling function + * @weight: default weight + * + * This variant of netif_napi_add() should be used from drivers using NAPI + * with CPU intensive poll functions. + * This will schedule polling from a high priority workqueue + */ +static inline void netif_threaded_napi_add(struct net_device *dev, + struct napi_struct *napi, + int (*poll)(struct napi_struct *, int), + int weight) +{ + set_bit(NAPI_STATE_THREADED, &napi->state); + netif_napi_add(dev, napi, poll, weight); +} + +/** * netif_tx_napi_add - initialize a NAPI context * @dev: network device * @napi: NAPI context @@ -2432,6 +2543,28 @@ struct u64_stats_sync syncp; } __aligned(2 * sizeof(u64)); +#define NETDEV_SAWF_HIST_BASE_US 1 /* Number of microseconds represented by bucket 0. */ +#define NETDEV_SAWF_DELAY_BUCKETS 8 /* Number of buckets in latency histogram. */ +#define NETDEV_SAWF_FLAG_ENABLED 0x1 /* Flag to enable SAWF features for netdevice. */ +#define NETDEV_SAWF_FLAG_RX_LAT 0x2 /* Flag to enable Rx hardware latency. */ +#define NETDEV_SAWF_FLAG_TX_LAT 0X4 /* Flag to enable Tx hardware latency. */ +#define NETDEV_SAWF_FLAG_DEBUG 0X8 /* Flag to enable debug service class latency. */ +#define NETDEV_SAWF_FLAG_DEBUG_SHIFT 8 /* Offset of debug service class ID. */ +#define NETDEV_SAWF_FLAG_DEBUG_MASK 0XFF00 /* Mask of debug service class ID. */ +#define NETDEV_SAWF_SID_MAX 256 /* Number of valid service class IDs. */ + +struct pcpu_sawf_stats { + u64 total_delay[NETDEV_SAWF_SID_MAX]; /* Total delay in milliseconds */ + u64 delay[NETDEV_SAWF_SID_MAX][NETDEV_SAWF_DELAY_BUCKETS]; /* Delay histogram; 8 bins over 256 potential service classes. */ + u64 tx_packets[NETDEV_SAWF_SID_MAX]; /* Packets sent per service class. */ + u64 tx_bytes[NETDEV_SAWF_SID_MAX]; /* Bytes sent per service class. */ + u32 debug_lat_max; /* Maximum latency for specified debug service class. */ + u32 debug_lat_min; /* Minimum measured latency for specified debug service class. */ + u32 debug_lat_ewma; /* Exponential weighted moving average latency for specified debug service class. */ + u32 debug_lat_last; /* Most recent latency for specified debug service class. */ + struct u64_stats_sync syncp; +}; + #define __netdev_alloc_pcpu_stats(type, gfp) \ ({ \ typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\ @@ -2522,6 +2655,8 @@ NETDEV_CVLAN_FILTER_DROP_INFO, NETDEV_SVLAN_FILTER_PUSH_INFO, NETDEV_SVLAN_FILTER_DROP_INFO, + NETDEV_BR_JOIN, + NETDEV_BR_LEAVE, }; const char *netdev_cmd_to_name(enum netdev_cmd cmd); @@ -2584,7 +2719,6 @@ int call_netdevice_notifiers(unsigned long val, struct net_device *dev); - extern rwlock_t dev_base_lock; /* Device list lock */ #define for_each_netdev(net, d) \ @@ -2667,6 +2801,10 @@ u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb, struct net_device *sb_dev); int dev_queue_xmit(struct sk_buff *skb); +bool dev_fast_xmit_vp(struct sk_buff *skb, struct net_device *dev); +bool dev_fast_xmit(struct sk_buff *skb, struct net_device *dev, + netdev_features_t features); +bool dev_fast_xmit_qdisc(struct sk_buff *skb, struct net_device *top_qdisc_dev, struct net_device *bottom_dev); int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev); int dev_direct_xmit(struct sk_buff *skb, u16 queue_id); int register_netdevice(struct net_device *dev); @@ -2691,6 +2829,16 @@ int dev_restart(struct net_device *dev); int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); +bool netdev_sawf_deinit(struct net_device *dev); +bool netdev_sawf_init(struct net_device *dev, uint16_t mode); +bool netdev_sawf_flags_update(struct net_device *dev, uint16_t mode); +bool netdev_sawf_enable(struct net_device *dev); +bool netdev_sawf_disable(struct net_device *dev); +bool netdev_sawf_debug_set(struct net_device *dev, uint8_t sid); +bool netdev_sawf_debug_unset(struct net_device *dev); +bool netdev_sawf_debug_get(struct net_device *dev, uint8_t *sid, uint32_t *max, uint32_t *min, uint32_t *avg, uint32_t *last); +bool netdev_sawf_lat_get(struct net_device *dev, uint8_t sid, uint64_t *hist, uint64_t *avg); + static inline unsigned int skb_gro_offset(const struct sk_buff *skb) { return NAPI_GRO_CB(skb)->data_offset; @@ -3683,7 +3831,8 @@ rx_handler_func_t *rx_handler, void *rx_handler_data); void netdev_rx_handler_unregister(struct net_device *dev); - +void set_avm_recvhook(int (*recvhook)(struct sk_buff *skb)); +void set_avm_early_recvhook(int (*recvhook)(struct sk_buff *skb)); bool dev_valid_name(const char *name); static inline bool is_socket_ioctl_cmd(unsigned int cmd) { @@ -3866,7 +4015,6 @@ return test_bit(__LINK_STATE_DORMANT, &dev->state); } - /** * netif_oper_up - test if device is operational * @dev: network device @@ -4199,6 +4347,15 @@ void dev_uc_init(struct net_device *dev); /** + * ifb_update_offload_stats - Update the IFB interface stats + * @dev: IFB device to update the stats + * @offload_stats: per CPU stats structure + * + * Allows update of IFB stats when flows are offloaded to an accelerator. + **/ +void ifb_update_offload_stats(struct net_device *dev, struct pcpu_sw_netstats *offload_stats); + +/** * __dev_uc_sync - Synchonize device's unicast list * @dev: device to sync * @sync: function to call if address should be added @@ -4475,12 +4632,82 @@ return __this_cpu_read(softnet_data.xmit.more); } +static inline bool netdev_check_sawf_debug_match(struct net_device *dev, uint8_t sid) +{ + return (dev->sawf_flags & NETDEV_SAWF_FLAG_DEBUG) && ((dev->sawf_flags >> NETDEV_SAWF_FLAG_DEBUG_SHIFT) == sid); +} + +static inline void netdev_sawf_latency_record(struct sk_buff *skb, struct net_device *dev) +{ + struct pcpu_sawf_stats *sawf_stats; + int64_t lat; + uint8_t sid; + int bucket; + + /* + * Return if latency does not need to be recorded. + */ + if ((dev->sawf_flags & (NETDEV_SAWF_FLAG_ENABLED | NETDEV_SAWF_FLAG_TX_LAT)) != NETDEV_SAWF_FLAG_ENABLED) { + return; + } + + sawf_stats = this_cpu_ptr(dev->sawf_stats); + if (!sawf_stats) { + return; + } + + + if (SKB_GET_SAWF_TAG(skb->mark) != SKB_SAWF_VALID_TAG) { + return; + } + + if (!skb->tstamp) { + return; + } + + lat = ktime_to_ns(net_timedelta(skb->tstamp)); + + sid = SKB_GET_SAWF_SERVICE_CLASS(skb->mark); + + /* + * Latency is divided by 1000 to convert from nanoseconds to microseconds. + */ + bucket = fls(div64_s64(lat, (1000 * NETDEV_SAWF_HIST_BASE_US))) - 1; + if (bucket < 0) { + bucket = 0; + } else if (bucket >= NETDEV_SAWF_DELAY_BUCKETS) { + bucket = NETDEV_SAWF_DELAY_BUCKETS - 1; + } + + u64_stats_update_begin(&sawf_stats->syncp); + sawf_stats->delay[sid][bucket]++; + sawf_stats->total_delay[sid] += lat; + sawf_stats->tx_packets[sid]++; + sawf_stats->tx_bytes[sid] += skb->len; + u64_stats_update_end(&sawf_stats->syncp); + + if (!netdev_check_sawf_debug_match(dev, sid)) { + return; + } + + sawf_stats->debug_lat_last = lat; + sawf_stats->debug_lat_ewma = sawf_stats->debug_lat_ewma - (sawf_stats->debug_lat_ewma >> 8) + (lat >> 8); + + if (lat > sawf_stats->debug_lat_max) { + sawf_stats->debug_lat_max = lat; + } + + if (lat < sawf_stats->debug_lat_min || sawf_stats->debug_lat_min == 0) { + sawf_stats->debug_lat_min = lat; + } +} + static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev, struct netdev_queue *txq, bool more) { const struct net_device_ops *ops = dev->netdev_ops; netdev_tx_t rc; - + netdev_sawf_latency_record(skb, dev); rc = __netdev_start_xmit(ops, skb, dev, more); if (rc == NETDEV_TX_OK) txq_trans_update(txq); @@ -4712,6 +4939,11 @@ return dev->priv_flags & IFF_FAILOVER_SLAVE; } +static inline bool netif_is_ifb_dev(const struct net_device *dev) +{ + return dev->priv_flags_ext & IFF_EXT_IFB; +} + /* This device needs to keep skb dst for qdisc enqueue or ndo_start_xmit() */ static inline void netif_keep_dst(struct net_device *dev) {