--- zzzz-none-000/linux-4.4.60/net/core/skbuff.c 2017-04-08 07:53:53.000000000 +0000 +++ scorpion-7490-727/linux-4.4.60/net/core/skbuff.c 2021-02-04 17:41:59.000000000 +0000 @@ -63,6 +63,7 @@ #include #include #include +#include #include #include @@ -77,6 +78,9 @@ #include #include +#include "skbuff_recycle.h" +#include "skbuff_debug.h" + struct kmem_cache *skbuff_head_cache __read_mostly; static struct kmem_cache *skbuff_fclone_cache __read_mostly; int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; @@ -166,6 +170,7 @@ gfp_mask & ~__GFP_DMA, node); if (!skb) goto out; + skbuff_debugobj_init_and_activate(skb); /* * Only clear those fields we need to clear, not those that we will @@ -218,6 +223,7 @@ skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); if (!skb) goto out; + skbuff_debugobj_init_and_activate(skb); prefetchw(skb); /* We do our best to align skb_shared_info on a separate cache @@ -275,6 +281,7 @@ out: return skb; nodata: + skbuff_debugobj_deactivate(skb); kmem_cache_free(cache, skb); skb = NULL; goto out; @@ -309,6 +316,7 @@ skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); if (!skb) return NULL; + skbuff_debugobj_init_and_activate(skb); size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); @@ -394,7 +402,7 @@ /** * __netdev_alloc_skb - allocate an skbuff for rx on a specific device * @dev: network device to receive on - * @len: length to allocate + * @length: length to allocate * @gfp_mask: get_free_pages mask, passed to alloc_skb * * Allocate a new &sk_buff and assign it a usage count of one. The @@ -404,19 +412,55 @@ * * %NULL is returned if there is no free memory. */ -struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, - gfp_t gfp_mask) +struct sk_buff *__netdev_alloc_skb(struct net_device *dev, + unsigned int length, gfp_t gfp_mask) { + struct sk_buff *skb; + unsigned int len = length; +#ifndef CONFIG_SKB_RECYCLER struct page_frag_cache *nc; unsigned long flags; - struct sk_buff *skb; bool pfmemalloc; + bool page_frag_alloc_enable = true; void *data; +#endif + +#ifdef CONFIG_SKB_RECYCLER + skb = skb_recycler_alloc(dev, length); + if (likely(skb)) { + /* SKBs in the recycler are from various unknown sources. + * Their truesize is unknown. We should set truesize + * as the needed buffer size before using it. + */ + skb->truesize = SKB_TRUESIZE(SKB_DATA_ALIGN(len + NET_SKB_PAD)); + return skb; + } + + len = SKB_RECYCLE_SIZE; + if (unlikely(length > SKB_RECYCLE_SIZE)) + len = length; + + skb = __alloc_skb(len + NET_SKB_PAD, gfp_mask, + SKB_ALLOC_RX, NUMA_NO_NODE); + if (!skb) + goto skb_fail; + /* Set truesize as the needed buffer size + * rather than the allocated size by __alloc_skb(). + */ + if (length + NET_SKB_PAD < SKB_WITH_OVERHEAD(PAGE_SIZE)) + skb->truesize = SKB_TRUESIZE(SKB_DATA_ALIGN(length + NET_SKB_PAD)); + + goto skb_success; +#else len += NET_SKB_PAD; +#ifdef CONFIG_ALLOC_SKB_PAGE_FRAG_DISABLE + page_frag_alloc_enable = false; +#endif if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) || - (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { + (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA)) || + !page_frag_alloc_enable) { skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); if (!skb) goto skb_fail; @@ -450,6 +494,7 @@ if (pfmemalloc) skb->pfmemalloc = 1; skb->head_frag = 1; +#endif skb_success: skb_reserve(skb, NET_SKB_PAD); @@ -520,6 +565,22 @@ } EXPORT_SYMBOL(__napi_alloc_skb); +struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev, + unsigned int length, gfp_t gfp) +{ + struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp); + +#ifdef CONFIG_ETHERNET_PACKET_MANGLE + if (dev && (dev->priv_flags & IFF_NO_IP_ALIGN)) + return skb; +#endif + + if (NET_IP_ALIGN && skb) + skb_reserve(skb, NET_IP_ALIGN); + return skb; +} +EXPORT_SYMBOL(__netdev_alloc_skb_ip_align); + void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, int size, unsigned int truesize) { @@ -571,7 +632,7 @@ kfree(head); } -static void skb_release_data(struct sk_buff *skb) +void skb_release_data(struct sk_buff *skb) { struct skb_shared_info *shinfo = skb_shinfo(skb); int i; @@ -605,12 +666,13 @@ /* * Free an skbuff by memory without cleaning the state. */ -static void kfree_skbmem(struct sk_buff *skb) +void kfree_skbmem(struct sk_buff *skb) { struct sk_buff_fclones *fclones; switch (skb->fclone) { case SKB_FCLONE_UNAVAILABLE: + skbuff_debugobj_deactivate(skb); kmem_cache_free(skbuff_head_cache, skb); return; @@ -631,7 +693,9 @@ } if (!atomic_dec_and_test(&fclones->fclone_ref)) return; + fastpath: + skbuff_debugobj_deactivate(&fclones->skb1); kmem_cache_free(skbuff_fclone_cache, fclones); } @@ -648,6 +712,9 @@ #if IS_ENABLED(CONFIG_NF_CONNTRACK) nf_conntrack_put(skb->nfct); #endif +#if IS_ENABLED(CONFIG_AVM_GENERIC_CONNTRACK) + generic_ct_put(skb->generic_ct); +#endif #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) nf_bridge_put(skb->nf_bridge); #endif @@ -740,12 +807,38 @@ { if (unlikely(!skb)) return; + + prefetch(&skb->destructor); + if (likely(atomic_read(&skb->users) == 1)) smp_rmb(); else if (likely(!atomic_dec_and_test(&skb->users))) return; + + /* If possible we'd like to recycle any skb rather than just free it, + * but in order to do that we need to release any head state too. + * We don't want to do this later because we'll be in a pre-emption + * disabled state. + */ + skb_release_head_state(skb); + + /* Can we recycle this skb? If we can then it will be much faster + * for us to recycle this one later than to allocate a new one + * from scratch. + */ + if (likely(skb->head) && likely(skb_recycler_consume(skb))) + return; + trace_consume_skb(skb); - __kfree_skb(skb); + + /* We're not recycling so now we need to do the rest of what we would + * have done in __kfree_skb (above and beyond the skb_release_head_state + * that we already did). + */ + if (likely(skb->head)) + skb_release_data(skb); + + kfree_skbmem(skb); } EXPORT_SYMBOL(consume_skb); @@ -761,6 +854,9 @@ new->tstamp = old->tstamp; /* We do not copy old->sk */ new->dev = old->dev; +#if IS_ENABLED(CONFIG_AVM_NET_SKB_INPUT_DEV) + new->input_dev = old->input_dev; +#endif memcpy(new->cb, old->cb, sizeof(old->cb)); skb_dst_copy(new, old); #ifdef CONFIG_XFRM @@ -806,6 +902,9 @@ CHECK_SKB_FIELD(tc_verd); #endif #endif +#ifdef CONFIG_AVM_PA + CHECK_SKB_FIELD(avm_pa); +#endif } @@ -956,6 +1055,7 @@ n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); if (!n) return NULL; + skbuff_debugobj_init_and_activate(n); kmemcheck_annotate_bitfield(n, flags1); n->fclone = SKB_FCLONE_UNAVAILABLE; @@ -3327,6 +3427,10 @@ 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); +#if IS_ENABLED(CONFIG_AVM_GENERIC_CONNTRACK) + generic_ct_init(); +#endif + skb_recycler_init(); } /** @@ -4115,6 +4219,7 @@ { if (head_stolen) { skb_release_head_state(skb); + skbuff_debugobj_deactivate(skb); kmem_cache_free(skbuff_head_cache, skb); } else { __kfree_skb(skb); @@ -4223,7 +4328,9 @@ skb_dst_drop(skb); skb_sender_cpu_clear(skb); secpath_reset(skb); - nf_reset(skb); + /* TMA/MQU 20170411: Is this the right thing for namespace + * changes? We think so. See JZ-30001. */ + nf_reset_no_generic_ct(skb); nf_reset_trace(skb); if (!xnet)