--- zzzz-none-000/linux-4.1.52/net/core/skbuff.c 2018-05-28 02:26:45.000000000 +0000 +++ bcm63-7530ax-731/linux-4.1.52/net/core/skbuff.c 2022-03-02 11:37:13.000000000 +0000 @@ -76,12 +76,187 @@ #include #include #include +#if defined(CONFIG_AVM_SIMPLE_PROFILING) +#include +#endif + struct kmem_cache *skbuff_head_cache __read_mostly; static struct kmem_cache *skbuff_fclone_cache __read_mostly; int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; EXPORT_SYMBOL(sysctl_max_skb_frags); +#if defined(CONFIG_BCM_KF_NBUFF) +#include +#include + +/* Returns size of struct sk_buff */ +size_t skb_size(void) +{ + return sizeof(struct sk_buff); +} +EXPORT_SYMBOL(skb_size); + +size_t skb_aligned_size(void) +{ + return ((sizeof(struct sk_buff) + 0x0f) & ~0x0f); +} +EXPORT_SYMBOL(skb_aligned_size); + +int skb_layout_test(int head_offset, int tail_offset, int end_offset) +{ +#define SKBOFFSETOF(member) ((size_t)&((struct sk_buff*)0)->member) + if ((SKBOFFSETOF(head) == head_offset) && + (SKBOFFSETOF(tail) == tail_offset) && + (SKBOFFSETOF(end) == end_offset)) + return 1; + return 0; +} +EXPORT_SYMBOL(skb_layout_test); + +int skb_avail_headroom(const struct sk_buff *skb) +{ +#if defined(CONFIG_BCM_USBNET_ACCELERATION) + if (skb->clone_fc_head) { + /* In this case it's unlikely but possible for + * the value of skb->data - skb->clone_fc_head to be negative + * the caller should check for negative value + */ + return skb->data - skb->clone_fc_head; + } else +#endif + return skb->data - skb->head; +} +EXPORT_SYMBOL(skb_avail_headroom); + +void skb_bpm_tainted(struct sk_buff *skb) +{ + /* recycle_flags &= ~SKB_BPM_PRISTINE, dirty_p = NULL */ + SKB_BPM_TAINTED(skb); +} +EXPORT_SYMBOL(skb_bpm_tainted); + +void skb_data_pristine(struct sk_buff *skb) +{ + SKB_DATA_PRISTINE(skb); /* skb->dirty = skb->head */ +} +EXPORT_SYMBOL(skb_data_pristine); + +static inline void _skb_headerreset(struct sk_buff *skb) +{ + memset(skb, 0, offsetof(struct sk_buff, truesize)); +} +void skb_headerreset(struct sk_buff *skb) +{ + _skb_headerreset(skb); + atomic_set(&skb->users, 1); +} +EXPORT_SYMBOL(skb_headerreset); + +static inline void _skb_shinforeset(struct skb_shared_info *skb_shinfo) +{ + memset(skb_shinfo, 0, offsetof(struct skb_shared_info, dataref)); +} +void skb_shinforeset(struct skb_shared_info *skb_shinfo) +{ + _skb_shinforeset(skb_shinfo); + atomic_set(&(skb_shinfo->dataref), 1); +} +EXPORT_SYMBOL(skb_shinforeset); + +/** + * + * skb_headerinit - initialize a socket buffer header + * @headroom: reserved headroom size + * @datalen: data buffer size, data buffer is allocated by caller + * @skb: skb allocated by caller + * @data: data buffer allocated by caller + * @recycle_hook: callback function to free data buffer and skb + * @recycle_context: context value passed to recycle_hook, param1 + * @blog_p: pass a blog to a skb for logging + * + * Initializes the socket buffer and assigns the data buffer to it. + * Both the sk_buff and the pointed data buffer are pre-allocated. + * + */ +void skb_headerinit(unsigned int headroom, unsigned int datalen, + struct sk_buff *skb, unsigned char *data, + RecycleFuncP recycle_hook, unsigned long recycle_context, + struct blog_t *blog_p) /* defined(CONFIG_BLOG) */ +{ + _skb_headerreset(skb); /* memset to truesize */ + + skb->truesize = datalen + sizeof(struct sk_buff); + atomic_set(&skb->users, 1); + skb->head = data - headroom; + skb->data = data; + skb_set_tail_pointer(skb, datalen); + /* FIXME!! check if this alignment is to ensure cache line aligned? + * make sure skb buf ends at 16 bytes boudary */ + skb->end = skb->tail + (0x10 - (((uintptr_t)skb_tail_pointer(skb)) & 0xf)); + skb->len = datalen; + +#if defined (CONFIG_BCM_KF_BPM_BUF_TRACKING) + GBPM_INC_REF(data); + KERN_GBPM_TRACK_SKB(skb, GBPM_VAL_INIT, 0); +#endif +#if defined(CONFIG_BCM_KF_BLOG) && defined(CONFIG_BLOG) + skb->blog_p = blog_p; + if (blog_p) + blog_p->skb_p = skb; + // skb->tunl = NULL; memset in _skb_headerreset +#endif +#if defined(CONFIG_BCM_KF_VLAN) && (defined(CONFIG_BCM_VLAN) || defined(CONFIG_BCM_VLAN_MODULE)) + // skb->vlan_count = 0; memset in _skb_headerreset +#endif +#if defined(CONFIG_BCM_KF_MAP) && (defined(CONFIG_BCM_MAP) || defined(CONFIG_BCM_MAP_MODULE)) + // skb->map_id = 0; memset in _skb_headerreset + // skb->map_offset = 0; memset in skb_headerreset +#endif + skb->fc_ctxt = 0; + skb->recycle_hook = recycle_hook; + skb->recycle_context = recycle_context; + skb->recycle_flags = SKB_RECYCLE | SKB_DATA_RECYCLE; + + _skb_shinforeset(skb_shinfo(skb)); + atomic_set(&(skb_shinfo(skb)->dataref), 1); +} +EXPORT_SYMBOL(skb_headerinit); + +struct sk_buff *skb_header_alloc(void) +{ + return kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); +} +EXPORT_SYMBOL(skb_header_alloc); + +/* + * ----------------------------------------------------------------------------- + * Function: Free or recycle an sk_buff memory + * + * CAUTION : API ignores any databuf attached to it. + * Caller of the function should handle databuf. + * ----------------------------------------------------------------------------- + */ +void skb_header_free(struct sk_buff *skb) +{ + /* If the skb came from a preallocated pool, pass it to recycler hook */ + if (skb->recycle_hook && (skb->recycle_flags & SKB_RECYCLE)) + { + /* Detach data buffer */ + /* Caller should take care of databuf */ + skb->head = NULL; + skb->data = NULL; + skb->recycle_flags &= ~ SKB_DATA_RECYCLE; + (*skb->recycle_hook)(skb, skb->recycle_context, SKB_RECYCLE); + } + else + { + kmem_cache_free(skbuff_head_cache, skb); + } +} +EXPORT_SYMBOL(skb_header_free); +#endif /* CONFIG_BCM_KF_NBUFF */ + /** * skb_panic - private function for out-of-line support * @skb: buffer @@ -208,6 +383,20 @@ u8 *data; bool pfmemalloc; +#if defined(CONFIG_BCM_KF_ARM64_BCM963XX) && defined(CONFIG_BCM94908) && defined(CONFIG_BCM_HND_EAP) + /* + * runner on 94908 supports only 30 bit addressing, which limits + * its operation to the lower 1 Gig memory space. Otherwise an + * extra packet copy is performed which slows its operation. + * Therefore, for this platform, force the buffer to always be + * dma'able, hence ensuring it always comes from the low 1 Gig + * address space. + */ + if (likely(!(gfp_mask & GFP_HIGHUSER))) + gfp_mask |= GFP_DMA; + +#endif /* defined(CONFIG_BCM_KF_ARM64_BCM963XX) && defined(CONFIG_BCM94908) && defined(CONFIG_BCM_HND_EAP) */ + cache = (flags & SKB_ALLOC_FCLONE) ? skbuff_fclone_cache : skbuff_head_cache; @@ -220,6 +409,29 @@ goto out; prefetchw(skb); +#if defined(CONFIG_BCM_KF_NBUFF) + if (size < ENET_MIN_MTU_SIZE_EXT_SWITCH) { + /* Add enough tailroom so that small packets can be padded + * with 0s to meet the Ethernet minimum pkt size of 60 bytes + + * EXT_SW TAG Total: 64 bytes) + * This will help improve performance with NAS test scenarios + * where the TCP ACK is usually less than 60 bytes + * WARNING: Note that the macro SKB_WITH_OVERHEAD() does not + * take into account this additional tailroom overhead. If + * the original size passed to this function uses the + * SKB_WITH_OVERHEAD macro to calculate the alloc length then + * this function will allocate more than what is expected and + * this can cause length (calculated using SKB_WITH_OVERHEAD) + * based operations to fail. We found few instances of skb + * allocations using the macro SKB_WITH_OVERHEAD (for ex: + * allocations using NLMSG_DEFAULT_SIZE in netlink.h). + * However, all those allocations allocate large skbs + * (page size 4k/8k) and will not enter this additional tailroom + * logic */ + size = ENET_MIN_MTU_SIZE_EXT_SWITCH; + } +#endif + /* We do our best to align skb_shared_info on a separate cache * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives * aligned memory blocks, unless SLUB/SLAB debug is enabled. @@ -237,12 +449,21 @@ size = SKB_WITH_OVERHEAD(ksize(data)); prefetchw(data + size); +#if defined(CONFIG_BCM_KF_NBUFF) + /* + * Clearing all fields -- fields that were not cleared before + * were moved to earlier locations in the structure, so just + * zeroing them out (OK, since we overwrite them shortly: + */ + memset(skb, 0, offsetof(struct sk_buff, truesize)); +#else /* * Only clear those fields we need to clear, not those that we will * actually initialise below. Hence, don't put any more fields after * the tail pointer in struct sk_buff! */ memset(skb, 0, offsetof(struct sk_buff, tail)); +#endif /* Account for allocated memory : skb + skb->head */ skb->truesize = SKB_TRUESIZE(size); skb->pfmemalloc = pfmemalloc; @@ -257,6 +478,10 @@ /* make sure we initialize shinfo sequentially */ shinfo = skb_shinfo(skb); memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); +#if defined(CONFIG_BCM_KF_NBUFF) + shinfo->dirty_p = NULL; +#endif + atomic_set(&shinfo->dataref, 1); kmemcheck_annotate_variable(shinfo->destructor_arg); @@ -273,6 +498,9 @@ fclones->skb2.pfmemalloc = pfmemalloc; } out: +#if defined(CONFIG_AVM_SIMPLE_PROFILING) + avm_simple_profiling_skb((unsigned int)skb->data, skb); +#endif return skb; nodata: kmem_cache_free(cache, skb); @@ -603,7 +831,11 @@ skb_drop_list(&skb_shinfo(skb)->frag_list); } +#if !defined(CONFIG_BCM_MPTCP) || !defined(CONFIG_BCM_KF_MPTCP) static void skb_clone_fraglist(struct sk_buff *skb) +#else +void skb_clone_fraglist(struct sk_buff *skb) +#endif { struct sk_buff *list; @@ -613,6 +845,15 @@ static void skb_free_head(struct sk_buff *skb) { +#if defined(CONFIG_BCM_KF_NBUFF) + /* If the data buffer came from a pre-allocated pool, recycle it. + * Recycling may only be performed when no references exist to it. */ + if (skb->recycle_hook && (skb->recycle_flags & SKB_DATA_RECYCLE)) { + (*skb->recycle_hook)(skb, skb->recycle_context, SKB_DATA_RECYCLE); + skb->recycle_flags &= SKB_DATA_NO_RECYCLE; /* mask out */ + return; + } +#endif if (skb->head_frag) put_page(virt_to_head_page(skb->head)); else @@ -657,6 +898,17 @@ { struct sk_buff_fclones *fclones; +#if defined(CONFIG_BCM_KF_NBUFF) +#if defined(CONFIG_BCM_KF_BLOG) && defined(CONFIG_BLOG) + blog_free(skb, blog_free_reason_kfree); +#endif + + /* If the skb came from a preallocated pool, pass it to recycler hook */ + if (skb->recycle_hook && (skb->recycle_flags & SKB_RECYCLE)) + (*skb->recycle_hook)(skb, skb->recycle_context, SKB_RECYCLE); + else { +#endif /* CONFIG_BCM_KF_NBUFF */ + switch (skb->fclone) { case SKB_FCLONE_UNAVAILABLE: kmem_cache_free(skbuff_head_cache, skb); @@ -681,6 +933,9 @@ return; fastpath: kmem_cache_free(skbuff_fclone_cache, fclones); +#if defined(CONFIG_BCM_KF_NBUFF) + } +#endif /* CONFIG_BCM_KF_NBUFF */ } static void skb_release_head_state(struct sk_buff *skb) @@ -699,6 +954,9 @@ #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) nf_bridge_put(skb->nf_bridge); #endif +#if defined(CONFIG_BCM_KF_NBUFF) + skb->tc_word = 0; +#endif /* CONFIG_BCM_KF_NBUFF */ } /* Free everything but the sk_buff shell. */ @@ -720,6 +978,19 @@ void __kfree_skb(struct sk_buff *skb) { +#if defined (CONFIG_BCM_KF_BPM_BUF_TRACKING) + GBPM_DEC_REF(skb->data); + KERN_GBPM_TRACK_SKB(skb, GBPM_VAL_FREE, 0); +#endif +#if defined(CONFIG_BCM_KF_NBUFF) + if (skb->recycle_hook && (skb->recycle_flags & SKB_RECYCLE_NOFREE)) { + (*skb->recycle_hook)(skb, skb->recycle_context, SKB_RECYCLE_NOFREE); + return; + } +#endif +#if defined(CONFIG_AVM_SIMPLE_PROFILING) + avm_simple_profiling_skb((unsigned int)skb->data, skb); +#endif skb_release_all(skb); kfree_skbmem(skb); } @@ -756,6 +1027,486 @@ } EXPORT_SYMBOL(kfree_skb_list); +#if defined(CONFIG_BCM_KF_NBUFF) +/* + * Translate a fkb to a skb, by allocating a skb from the skbuff_head_cache. + * PS. skb->dev is not set during initialization. + * + * Caller verifies whether the fkb is unshared: + * if fkb_p==NULL||IS_FKB_CLONE(fkb_p)||fkb_p->users>1 and return NULL skb. + * + * skb_xlate is deprecated. New code should call skb_xlate_dp directly. + */ +struct sk_buff * skb_xlate(struct fkbuff * fkb_p) +{ + return (skb_xlate_dp(fkb_p, NULL)); +} +EXPORT_SYMBOL(skb_xlate); + +struct sk_buff *skb_xlate_dp(struct fkbuff * fkb_p, uint8_t *dirty_p) +{ + struct sk_buff * skb_p; + unsigned int datalen; + + /* Optimization: use preallocated pool of skb with SKB_POOL_RECYCLE flag */ + skb_p = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); + if (!skb_p) + return skb_p; + skb_p->fclone = SKB_FCLONE_UNAVAILABLE; + + memset(skb_p, 0, offsetof(struct sk_buff, truesize)); + + datalen = SKB_DATA_ALIGN(fkb_p->len + BCM_SKB_TAILROOM); + + skb_p->data = fkb_p->data; + skb_p->head = (unsigned char *)(fkb_p + 1 ); + skb_set_tail_pointer(skb_p, fkb_p->len); + /* FIXME!! check whether this has to do with the cache line size + * make sure skb buf ends at 16 bytes boudary */ + skb_p->end = skb_p->tail + (0x10 - (((uintptr_t)skb_tail_pointer(skb_p)) & 0xf)); + +#if defined (CONFIG_BCM_KF_BPM_BUF_TRACKING) + GBPM_INC_REF(skb_p->data); + KERN_GBPM_TRACK_SKB(skb_p, GBPM_VAL_XLATE, 0); +#endif + +#define F2S(x) skb_p->x = fkb_p->x + F2S(len); + F2S(queue); + F2S(priority); + +#if defined(CONFIG_BCM_KF_BLOG) && defined(CONFIG_BLOG) + if (_IS_BPTR_(fkb_p->blog_p)) { /* should not happen */ + F2S(blog_p); + fkb_p->blog_p->skb_p = skb_p; + } +#endif + F2S(recycle_hook); + F2S(recycle_context); + skb_p->recycle_flags = SKB_DATA_RECYCLE; + + /* redundant: fkb_p must not be used henceforth */ + fkb_dec_ref(fkb_p); + + atomic_set(&skb_p->users, 1); + skb_p->truesize = datalen + sizeof(struct sk_buff); + + /* any change to skb_shinfo initialization in __alloc_skb must be ported + * to this block. */ + atomic_set(&(skb_shinfo(skb_p)->dataref), 1); + skb_shinfo(skb_p)->nr_frags = 0; + skb_shinfo(skb_p)->gso_size = 0; + skb_shinfo(skb_p)->gso_segs = 0; + skb_shinfo(skb_p)->gso_type = 0; + skb_shinfo(skb_p)->ip6_frag_id = 0; + skb_shinfo(skb_p)->tx_flags = 0; + skb_shinfo(skb_p)->frag_list = NULL; + memset(&(skb_shinfo(skb_p)->hwtstamps), 0, + sizeof(skb_shinfo(skb_p)->hwtstamps)); +#if defined(CONFIG_BCM_CSO) + if (fkb_p->rx_csum_verified) + skb_p->ip_summed = CHECKSUM_UNNECESSARY; +#endif + /* + * When fkb is xlated to skb, preserve the dirty_p info. + * This allows receiving driver to shorten its cache flush and also + * can shorten the cache flush when the buffer is recycled. Improves + * wlan perf by 10%. + */ + skb_shinfo(skb_p)->dirty_p = dirty_p; + + return skb_p; +#undef F2S +} +EXPORT_SYMBOL(skb_xlate_dp); + +#define NETDEV_XMIT(_dev, _buff) \ + _dev->netdev_ops->ndo_start_xmit(_buff, _dev) + +/* + * This fucntion fragments the skb into multiple skbs and xmits them + * this fucntion is a substitue for ip_fragment when Ip stack is skipped + * for packet acceleartion(fcache,CMF) + * + * Currently only IPv4 is supported + */ +void skb_frag_xmit4(struct sk_buff *origskb, struct net_device *txdev, + uint32_t is_pppoe, uint32_t minMtu, void *ipp) +{ + +#define DEBUG_SKBFRAG(args) + +#define IP_DF 0x4000 /* Flag: "Don't Fragment" */ +#define IP_MF 0x2000 /* Flag: "More Fragments" */ +#define IP_OFFSET 0x1FFF /* "Fragment Offset" part */ + + struct iphdr *iph; + int datapos, offset; + unsigned int max_dlen, hlen, hdrslen, left, len; + uint16_t not_last_frag; + struct sk_buff *fraglisthead; + struct sk_buff *fraglisttail; + struct sk_buff *skb2; + + DEBUG_SKBFRAG(("skb_frag_xmit4:enter origskb=%p,netdev=%p,is_pppoe=%d,\ + minMtu=%d ipp=%p\n",origskb, txdev, is_pppoe, + minMtu, ipp)); + + if (likely(origskb->len <= minMtu)) { + /* xmit packet */ + NETDEV_XMIT(txdev, (void *)CAST_REAL_TO_VIRT_PNBUFF(origskb, + SKBUFF_PTR)); + return; + } + + fraglisthead = NULL; + fraglisttail = NULL; + skb2 = NULL; + + DEBUG_SKBFRAG(("skb_frag_xmit4: checking for DF\n")); + iph = (struct iphdr *)ipp; + /* DROP the packet if DF flag is set */ + if (unlikely((iph->frag_off & htons(IP_DF)) && !(origskb->ignore_df))) { + /*----TODO: update error stats, send icmp error message ?--- */ + kfree_skb(origskb); + return; + } + + hlen = iph->ihl * 4; + + DEBUG_SKBFRAG(("skb_frag_xmit4: calculating hdrs len\n")); + /* calculate space for data,(ip payload) */ + hdrslen = ((uintptr_t)ipp - (uintptr_t)(origskb->data)) + hlen; + + left = origskb->len - hdrslen; /* Size of ip payload */ + datapos = hdrslen;/* Where to start from */ + max_dlen = minMtu - hdrslen; /* ip payload per frame */ + + DEBUG_SKBFRAG(("skb_frag_xmit4: computed hdrslen=%d, left=%d\n", + hdrslen, left)); + + /* frag_offset is represented in 8 byte blocks */ + offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3; + not_last_frag = iph->frag_off & htons(IP_MF); + + /* copy the excess data (>MTU size) from orig fkb to new fkb's */ + fraglisthead = origskb; + + while (left > 0) { + DEBUG_SKBFRAG(("skb_frag_xmit4: making fragments\n")); + len = left; + /* IF: it doesn't fit, use 'max_dlen' - the data space left */ + if (len > max_dlen) + len = max_dlen; + /* IF: we are not sending upto and including the packet end + then align the next start on an eight byte boundary */ + if (len < left) + len &= ~7; + + if (datapos == hdrslen) { + /*reuse the orig skb for 1st fragment */ + skb2 = origskb; + DEBUG_SKBFRAG(("skb_frag_xmit4: reusing skb\n")); + skb2->next = NULL; + fraglisttail = skb2; + skb2->len = hdrslen+len; + skb_set_tail_pointer(skb2, hdrslen+len); + } else { + + DEBUG_SKBFRAG(("skb_frag_xmit4: genrating new skb\n")); + /* Allocate a new skb */ + if ((skb2 = alloc_skb(len+hdrslen, GFP_ATOMIC)) == NULL) { + printk(KERN_INFO "no memory for new fragment!\n"); + goto fail; + } + + /* copy skb metadata */ + skb2->queue = origskb->queue; + skb2->priority = origskb->priority; + skb2->dev = origskb->dev; + + dst_release(skb_dst(skb2)); + skb_dst_set(skb2, dst_clone(skb_dst(origskb))); +#ifdef CONFIG_NET_SCHED + skb2->tc_index = origskb->tc_index; +#endif + + skb_put(skb2, len + hdrslen); + + DEBUG_SKBFRAG(("skb_frag_xmit4: copying headerto new skb\n")); + + /* copy the l2 header &l3 header to new fkb from orig fkb */ + memcpy(skb2->data, origskb->data, hdrslen); + + DEBUG_SKBFRAG(("skb_frag_xmit4: copying data to new skb\n")); + /* + * Copy a block of the IP datagram. + */ + memcpy(skb2->data + hdrslen, origskb->data + datapos, + len); + + skb2->next = NULL; + fraglisttail->next = skb2; + fraglisttail = skb2; + } + + /* Fill in the new header fields. */ + DEBUG_SKBFRAG(("skb_frag_xmit4: adjusting ipheader\n")); + iph = (struct iphdr *)(skb2->data + (hdrslen- hlen)); + iph->frag_off = htons((offset >> 3)); + iph->tot_len = htons(len + hlen); + + /* fix pppoelen */ + if (is_pppoe) + *((uint16_t*)iph - 2) = htons(len + hlen + + sizeof(uint16_t)); + + left -= len; + datapos += len; + offset += len; + + /* If we are fragmenting a fragment that's not the + * last fragment then keep MF on each fragment */ + if (left > 0 || not_last_frag) + iph->frag_off |= htons(IP_MF); + //else + //iph->frag_off &= ~htons(IP_MF);/*make sure MF is cleared */ + + DEBUG_SKBFRAG(("skb_frag_xmit4: computing ipcsum\n")); + /* fix ip checksum */ + iph->check = 0; + /* TODO replace with our own csum_calc */ + iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); + + DEBUG_SKBFRAG(("skb_frag_xmit4: loop done\n")); + } + + /* xmit skb's */ + while (fraglisthead) { + DEBUG_SKBFRAG(("skb_frag_xmit4: sending skb fragment \n")); + skb2 = fraglisthead; + fraglisthead = fraglisthead->next; + skb2->next = NULL; + NETDEV_XMIT(txdev, (void *)CAST_REAL_TO_VIRT_PNBUFF(skb2, + SKBUFF_PTR)); + } + return; + +fail: + DEBUG_SKBFRAG(("skb_frag_xmit4: ENTERED FAIL CASE\n")); + while (fraglisthead) { + skb2 = fraglisthead; + fraglisthead = fraglisthead->next; + kfree_skb(skb2); + } + return; + +} +EXPORT_SYMBOL(skb_frag_xmit4); + +#ifdef CONFIG_IPV6 +static void ipv6_generate_ident(struct frag_hdr *fhdr) +{ + static atomic_t ipv6_fragmentation_id; + int old, new; + + do { + old = atomic_read(&ipv6_fragmentation_id); + new = old + 1; + if (!new) + new = 1; + } while (atomic_cmpxchg(&ipv6_fragmentation_id, old, new) != old); + fhdr->identification = htonl(new); +} +#endif + +/* + * This fucntion fragments the skb into multiple skbs and xmits them + * this fucntion is a substitue for ip6_fragment when IPv6 stack is skipped + * for packet acceleartion + * + * Assumption: there should be no extension header in IPv6 header while + * learning the tunnel traffic + * + * Currently only IPv6 is supported + */ +void skb_frag_xmit6(struct sk_buff *origskb, struct net_device *txdev, + uint32_t is_pppoe, uint32_t minMtu, void *ipp) +{ +#ifdef CONFIG_IPV6 + struct ipv6hdr *iph; + int datapos, offset; + struct frag_hdr *fh; + __be32 frag_id = 0; + u8 nexthdr; + unsigned int max_dlen, hlen, hdrslen, left, len, frag_hdrs_len; + struct sk_buff *fraglisthead; + struct sk_buff *fraglisttail; + struct sk_buff *skb2; + + DEBUG_SKBFRAG(("skb_frag_xmit6:enter origskb=%p,netdev=%p,is_pppoe=%d,\ + minMtu=%d ipp=%p\n",origskb, txdev, is_pppoe, minMtu, ipp)); + + if (likely(origskb->len <= minMtu)) { + /* xmit packet */ + NETDEV_XMIT(txdev, (void *)CAST_REAL_TO_VIRT_PNBUFF(origskb, + SKBUFF_PTR)); + return; + } + + fraglisthead = NULL; + fraglisttail = NULL; + skb2 = NULL; + + iph = (struct ipv6hdr *)ipp; + hlen = sizeof(struct ipv6hdr); + + DEBUG_SKBFRAG(("skb_frag_xmit6: calculating hdrs len\n")); + /* calculate space for data,(ip payload) */ + hdrslen = ((uintptr_t)ipp - (uintptr_t)(origskb->data)) + hlen; + + left = origskb->len - hdrslen; /* Size of remaining ip payload */ + datapos = hdrslen;/* Where to start from */ + /* hdrlens including frag_hdr of packets after fragmented */ + frag_hdrs_len = hdrslen + sizeof(struct frag_hdr); + /* calculate max ip payload len per frame */ + max_dlen = minMtu - frag_hdrs_len; + nexthdr = iph->nexthdr; + + DEBUG_SKBFRAG(("skb_frag_xmit6: computed hdrslen=%d, left=%d, max=%d\n", + hdrslen, left, max_dlen)); + + offset = 0; + /* copy the excess data (>MTU size) from orig fkb to new fkb's */ + fraglisthead = origskb; + + /* len represents length of payload! */ + while (left > 0) { + DEBUG_SKBFRAG(("skb_frag_xmit6: making fragments\n")); + len = left; + /* IF: it doesn't fit, use 'max_dlen' - the data space left */ + if (len > max_dlen) + len = max_dlen; + /* IF: we are not sending upto and including the packet end + then align the next start on an eight byte boundary */ + if (len < left) + len &= ~7; + + /* + * Create new skbs to fragment the packet. Instead of reusing the + * orignal skb, a new skb is allocated to insert frag header + */ + DEBUG_SKBFRAG(("skb_frag_xmit6: genrating new skb\n")); + /* Allocate a new skb */ + if ((skb2 = alloc_skb(len+frag_hdrs_len, GFP_ATOMIC)) == NULL) { + printk(KERN_INFO "no memory for new fragment!\n"); + goto fail; + } + + /* copy skb metadata */ + skb2->queue = origskb->queue; + skb2->priority = origskb->priority; + skb2->dev = origskb->dev; + + dst_release(skb_dst(skb2)); + skb_dst_set(skb2, dst_clone(skb_dst(origskb))); +#ifdef CONFIG_NET_SCHED + skb2->tc_index = origskb->tc_index; +#endif + skb_put(skb2, len + frag_hdrs_len); + + DEBUG_SKBFRAG(("skb_frag_xmit6: copying headerto new skb\n")); + + /* copy the l2 header & l3 header to new fkb from orig fkb */ + memcpy(skb2->data, origskb->data, hdrslen); + + DEBUG_SKBFRAG(("skb_frag_xmit6: copying data to new skb\n")); + /* Copy a block of the IP datagram. */ + memcpy(skb2->data+frag_hdrs_len, origskb->data+datapos, len); + + skb2->next = NULL; + + /* first fragment, setup fraglist */ + if (datapos == hdrslen) { + fraglisthead = skb2; + fraglisttail = skb2; + } else { + fraglisttail->next = skb2; + fraglisttail = skb2; + } + + /* Fill in the new header fields. */ + DEBUG_SKBFRAG(("skb_frag_xmit6: adjusting IPv6 header\n")); + iph = (struct ipv6hdr *)(skb2->data + (hdrslen - hlen)); + iph->payload_len = htons(len + sizeof(struct frag_hdr)); + iph->nexthdr = NEXTHDR_FRAGMENT; + + /* insert fragmentation header */ + fh = (struct frag_hdr *)(iph + 1); + fh->nexthdr = nexthdr; + fh->reserved = 0; + if (!frag_id) { + ipv6_generate_ident(fh); + frag_id = fh->identification; + } else + fh->identification = frag_id; + fh->frag_off = htons(offset); + + /* fix pppoelen */ + if (is_pppoe) + *((uint16_t*)iph - 2) = htons(len + + sizeof(struct frag_hdr) + + sizeof(struct ipv6hdr) + + sizeof(uint16_t)); + left -= len; + datapos += len; + offset += len; + + /* If we are fragmenting a fragment that's not the + * last fragment then keep MF on each fragment */ + if (left > 0) + fh->frag_off |= htons(IP6_MF); + + DEBUG_SKBFRAG(("skb_frag_xmit6: loop done\n")); + } + + /* xmit skb's */ + while (fraglisthead) { + DEBUG_SKBFRAG(("skb_frag_xmit6: sending skb fragment \n")); + skb2 = fraglisthead; + fraglisthead = fraglisthead->next; + skb2->next = NULL; + NETDEV_XMIT(txdev, (void *)CAST_REAL_TO_VIRT_PNBUFF(skb2, + SKBUFF_PTR)); + } + + /* free the orignal skb */ + kfree_skb(origskb); + + return; + +fail: + DEBUG_SKBFRAG(("skb_frag_xmit6: ENTERED FAIL CASE\n")); + while (fraglisthead) { + skb2 = fraglisthead; + fraglisthead = fraglisthead->next; + kfree_skb(skb2); + } + + /* free the orignal skb */ + kfree_skb(origskb); + + return; + +#else /* !CONFIG_IPV6 */ + DEBUG_SKBFRAG(("skb_frag_xmit6: called while IPv6 is disabled in kernel?\n")); + kfree_skb(origskb); + return; +#endif +} +EXPORT_SYMBOL(skb_frag_xmit6); +#endif /* defined(CONFIG_BCM_KF_NBUFF) */ + /** * skb_tx_error - report an sk_buff xmit error * @skb: buffer that triggered an error @@ -806,10 +1557,21 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) { +#if defined(CONFIG_BCM_KF_VLAN) && (defined(CONFIG_BCM_VLAN) || defined(CONFIG_BCM_VLAN_MODULE)) + int i; +#endif + +#if defined(CONFIG_BCM_KF_WL) + memset(new->pktc_cb, 0, sizeof(new->pktc_cb)); +#endif + new->tstamp = old->tstamp; /* We do not copy old->sk */ new->dev = old->dev; memcpy(new->cb, old->cb, sizeof(old->cb)); +#if IS_ENABLED(CONFIG_AVM_NET_SKB_INPUT_DEV) + new->input_dev = old->input_dev; +#endif skb_dst_copy(new, old); #ifdef CONFIG_XFRM new->sp = secpath_get(old->sp); @@ -820,6 +1582,11 @@ * It is not yet because we do not want to have a 16 bit hole */ new->queue_mapping = old->queue_mapping; +#if defined(CONFIG_BCM_KF_NBUFF) + new->queue = old->queue; + new->priority = old->priority; + new->recycle_and_rnr_flags |= old->recycle_and_rnr_flags & SKB_RNR_FLAGS; +#endif memcpy(&new->headers_start, &old->headers_start, offsetof(struct sk_buff, headers_end) - @@ -827,18 +1594,26 @@ CHECK_SKB_FIELD(protocol); CHECK_SKB_FIELD(csum); CHECK_SKB_FIELD(hash); +#if defined(CONFIG_BCM_KF_NBUFF) +#else CHECK_SKB_FIELD(priority); +#endif CHECK_SKB_FIELD(skb_iif); CHECK_SKB_FIELD(vlan_proto); CHECK_SKB_FIELD(vlan_tci); + CHECK_SKB_FIELD(fritz_os_mark); CHECK_SKB_FIELD(transport_header); CHECK_SKB_FIELD(network_header); CHECK_SKB_FIELD(mac_header); + CHECK_SKB_FIELD(inner_protocol); CHECK_SKB_FIELD(inner_transport_header); CHECK_SKB_FIELD(inner_network_header); CHECK_SKB_FIELD(inner_mac_header); +#if defined(CONFIG_BCM_KF_NBUFF) +#else CHECK_SKB_FIELD(mark); +#endif #ifdef CONFIG_NETWORK_SECMARK CHECK_SKB_FIELD(secmark); #endif @@ -848,12 +1623,48 @@ #ifdef CONFIG_XPS CHECK_SKB_FIELD(sender_cpu); #endif +#if defined(CONFIG_BCM_KF_BLOG) && defined(CONFIG_BLOG) + new->tunl = old->tunl; +#endif +#if defined(CONFIG_BCM_KF_BLOG) && defined(CONFIG_BLOG) + blog_xfer(new, old); /* CONFIG_BLOG: transfers blog ownership */ +#endif + +#if (defined(CONFIG_BCM_KF_USBNET) && defined(CONFIG_BCM_USBNET_ACCELERATION)) + if (old->clone_fc_head) { + /* here we expect old->data > old->clone_fc_head, if for some reason this is not + * true we still need to set new->clone_fc_head. + * skb_avail_headroom , will check for this error + */ + new->clone_fc_head = new->data - (int)(old->data - old->clone_fc_head); + } +#endif + +#if defined(CONFIG_BCM_KF_NBUFF) + new->vtag_word = old->vtag_word; + new->tc_word = old->tc_word; +#if defined(CONFIG_BCM_KF_VLAN) && (defined(CONFIG_BCM_VLAN) || defined(CONFIG_BCM_VLAN_MODULE)) + new->vlan_count = old->vlan_count; + new->vlan_tpid = old->vlan_tpid; + for (i = 0; i < SKB_VLAN_MAX_TAGS; i++) { + new->vlan_header[i] = old->vlan_header[i]; + } + new->rxdev = old->rxdev; +#endif /* BCM_VLAN */ + new->in_dev = old->in_dev; +#else /* CONFIG_BCM_KF_NBUFF */ + #ifdef CONFIG_NET_SCHED CHECK_SKB_FIELD(tc_index); #ifdef CONFIG_NET_CLS_ACT CHECK_SKB_FIELD(tc_verd); #endif #endif +#endif /* CONFIG_BCM_KF_NBUFF */ + +#if defined(CONFIG_BCM_KF_WL) + new->pktc_flags = old->pktc_flags; +#endif } @@ -863,6 +1674,9 @@ */ static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) { +#if defined(CONFIG_BCM_KF_VLAN) && (defined(CONFIG_BCM_VLAN) || defined(CONFIG_BCM_VLAN_MODULE)) + int i; +#endif #define C(x) n->x = skb->x n->next = n->prev = NULL; @@ -883,6 +1697,42 @@ C(head_frag); C(data); C(truesize); + +#if (defined(CONFIG_BCM_KF_USBNET) && defined(CONFIG_BCM_USBNET_ACCELERATION)) + n->clone_wr_head = NULL; + skb->clone_wr_head = NULL; + C(clone_fc_head); +#endif + +#if defined (CONFIG_BCM_KF_BPM_BUF_TRACKING) + GBPM_INC_REF(skb->data); + KERN_GBPM_TRACK_SKB(skb, GBPM_VAL_COPY_SRC, 0); + KERN_GBPM_TRACK_SKB(skb, GBPM_VAL_COPY_DST, 0); +#endif + +#if defined(CONFIG_BCM_KF_NBUFF) + C(recycle_hook); + C(recycle_context); + n->recycle_flags = skb->recycle_flags & SKB_NO_RECYCLE; +#if defined(CONFIG_BCM_KF_VLAN) && (defined(CONFIG_BCM_VLAN) || defined(CONFIG_BCM_VLAN_MODULE)) + C(vlan_count); + C(vlan_tpid); + C(cfi_save); + C(vlan_tci); + C(vlan_proto); + for (i = 0; iusers, 1); atomic_inc(&(skb_shinfo(skb)->dataref)); @@ -904,8 +1754,45 @@ */ struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) { +#if defined(CONFIG_BCM_KF_NBUFF) + struct sk_buff *skb; + unsigned int recycle_flags; + unsigned long recycle_context; + RecycleFuncP recycle_hook; + + skb_release_all(dst); + + /* Need to retain the recycle flags, context & hook of dst to free it + * into proper pool. */ + recycle_flags = dst->recycle_flags & SKB_RECYCLE; + recycle_hook = dst->recycle_hook; + recycle_context = dst->recycle_context; + +#if defined(CONFIG_BCM_KF_BLOG) && defined(CONFIG_BLOG) + if(dst->blog_p) + blog_skip(dst, blog_skip_reason_skb_morph); +#endif + + if (unlikely((src->recycle_flags & SKB_DATA_RECYCLE) && + ((recycle_hook != src->recycle_hook) || + (recycle_context != src->recycle_context)))) + { + /* free the skb->head from src and reallocate from kernel + * if pskb_expand_head returns fail, unhandled error will be triggered. + * so BUG_ON here. */ + BUG_ON(pskb_expand_head(src, 0, 0, GFP_ATOMIC)); + } + + skb = __skb_clone(dst, src); + + dst->recycle_flags |= recycle_flags; + dst->recycle_hook = recycle_hook; + dst->recycle_context = recycle_context; + return skb; +#else /* CONFIG_BCM_KF_NBUFF */ skb_release_all(dst); return __skb_clone(dst, src); +#endif /* CONFIG_BCM_KF_NBUFF */ } EXPORT_SYMBOL_GPL(skb_morph); @@ -1009,7 +1896,9 @@ kmemcheck_annotate_bitfield(n, flags1); n->fclone = SKB_FCLONE_UNAVAILABLE; } - +#if defined(CONFIG_BCM_KF_BLOG) && defined(CONFIG_BLOG) + n->blog_p = NULL; +#endif return __skb_clone(n, skb); } EXPORT_SYMBOL(skb_clone); @@ -1029,7 +1918,11 @@ skb->inner_mac_header += off; } +#if !defined(CONFIG_BCM_MPTCP) || !defined(CONFIG_BCM_KF_MPTCP) static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) +#else +void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) +#endif { __copy_skb_header(new, old); @@ -1081,6 +1974,10 @@ BUG(); copy_skb_header(n, skb); + +#if defined (CONFIG_BCM_KF_BPM_BUF_TRACKING) + KERN_GBPM_TRACK_SKB(skb, GBPM_VAL_COPY_SRC, 1); +#endif return n; } EXPORT_SYMBOL(skb_copy); @@ -1144,6 +2041,11 @@ } copy_skb_header(n, skb); + +#if defined (CONFIG_BCM_KF_BPM_BUF_TRACKING) + KERN_GBPM_TRACK_SKB(skb, GBPM_VAL_COPY_SRC, 2); +#endif + out: return n; } @@ -1172,6 +2074,9 @@ u8 *data; int size = nhead + skb_end_offset(skb) + ntail; long off; +#if (defined(CONFIG_BCM_KF_USBNET) && defined(CONFIG_BCM_USBNET_ACCELERATION)) + int clone_fc_len = 0; +#endif BUG_ON(nhead < 0); @@ -1220,7 +2125,18 @@ skb->head = data; skb->head_frag = 0; +#if (defined(CONFIG_BCM_KF_USBNET) && defined(CONFIG_BCM_USBNET_ACCELERATION)) + if (skb->clone_fc_head) + clone_fc_len = skb->data - skb->clone_fc_head; +#endif + skb->data += off; + +#if (defined(CONFIG_BCM_KF_USBNET) && defined(CONFIG_BCM_USBNET_ACCELERATION)) + if (skb->clone_fc_head) + skb->clone_fc_head = skb->data - clone_fc_len; +#endif + #ifdef NET_SKBUFF_DATA_USES_OFFSET skb->end = size; off = nhead; @@ -1232,6 +2148,20 @@ skb->cloned = 0; skb->hdr_len = 0; skb->nohdr = 0; + +#if defined(CONFIG_BCM_KF_NBUFF) + /* Clear Data recycle as this buffer was allocated via kmalloc. + * Note that skb_release_data might have already cleared it but it is + * not guaranteed. If the buffer is cloned, then skb_release_data + * does not clear the buffer. The original data buffer will be freed + * when the cloned skb is freed */ + skb->recycle_flags &= SKB_DATA_NO_RECYCLE; + /* The data buffer of this skb is not pre-allocated any more + * even though the skb itself is pre-allocated, + * dirty_p pertains to previous buffer so clear it */ + skb_shinfo(skb)->dirty_p = NULL; +#endif + atomic_set(&skb_shinfo(skb)->dataref, 1); return 0; @@ -1318,6 +2248,10 @@ skb_headers_offset_update(n, newheadroom - oldheadroom); +#if defined (CONFIG_BCM_KF_BPM_BUF_TRACKING) + KERN_GBPM_TRACK_SKB(skb, GBPM_VAL_COPY_SRC, 3); +#endif + return n; } EXPORT_SYMBOL(skb_copy_expand); @@ -3090,6 +4024,9 @@ nskb->truesize += skb_end_offset(nskb) - hsize; skb_release_head_state(nskb); __skb_push(nskb, doffset); +#if defined(CONFIG_BCM_KF_BLOG) && defined(CONFIG_BLOG) + blog_skip(nskb, blog_skip_reason_skb_segment); +#endif } else { nskb = __alloc_skb(hsize + doffset + headroom, GFP_ATOMIC, skb_alloc_rx_flag(head_skb), @@ -4189,6 +5126,22 @@ ipvs_reset(skb); skb_orphan(skb); + +#if defined(CONFIG_BCM_KF_BLOG) && defined(CONFIG_BLOG) +/* + * when using containers in some network configurations, the packets + * will be treated as L2 flows while learning. Local aceleration + * is not supported for L2 flows so skip it. + * + * note: we try to add containers MAC address to Host MAC table (using + * netdev_notfier) to treat flows terminating in containers as L3 flows, + * but this check helps to catch any unknown configurations +*/ + if(skb->blog_p && skb->blog_p->l2_mode) + { + blog_skip(skb, blog_skip_reason_scrub_pkt); /* No local accel in l2 mode */ + } +#endif skb->mark = 0; } EXPORT_SYMBOL_GPL(skb_scrub_packet); @@ -4257,6 +5210,11 @@ vhdr = (struct vlan_hdr *)skb->data; vlan_tci = ntohs(vhdr->h_vlan_TCI); +#if defined(CONFIG_BCM_KF_NBUFF) +#if defined(CONFIG_BCM_KF_VLAN) && (defined(CONFIG_BCM_VLAN) || defined(CONFIG_BCM_VLAN_MODULE)) + skb->cfi_save = vlan_tci & VLAN_CFI_MASK; +#endif +#endif __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); skb_pull_rcsum(skb, VLAN_HLEN);