/**************************************************************************************** * ddc_sar_init.c * DDC UR8 SAR implementation. * * 2006 (c) Texas Instruments Inc. * * 8/18/2006 AV & Greg Guyotte Adapted the original version from Adam2. * 9/15/2006 AV Adding support for Tx channel close. * 9/25/2006 AV Adding support for Rx channel close. * 10/8/2006 AV Modifying the implementation of the cpsarRxBDProc() * to simplify the implementation, by taking out the * multipacket support for now. * 10/9/2006 AV Adding support for the doing the EOI's separately. * 10/12/2006 AV & GSG Fixing cpsarRxBDProc() for multiple BD support. * 10/19/2006 AV Fixing cpsarRxBDProc() to handle data corruption. * 10/27/2006 AV CQ11118:Consolidating the teardown functions for Tx and Rx. * 10/29/2006 AV Updating the DDC_malloc_rxbuffer() API to pass in the size. * 10/31/2006 AV Removing the support for servicing multiple interrupts in * the cpsarRxBDProc(). * 11/30/2006 EP CQ11196: Fixed multiple TX channel issue in GetTxBD(). * 12/07/2006 EP Enabled priorities for SAR channel 0. * 01/09/2007 EP Added multiple BD recycling in RxBDProc(). * 01/16/2007 MK CQ11287: Freeing buffer if the channel is not open in cpsarRxBDProc(). * 02/14/2007 EP CQ11442: dev_kfree_skb() cannot be called when interrupt is disabled. * 02/15/2007 MT Invalidated cache for each BD before recycling in multiBD case. ****************************************************************************************/ #include "ddc_cpsardrv.h" #define MAX_TX_Q 4 //The queue 0 and 1 are used by CPMAC. 2 and 3 are available for us. static Cppi4BD *TxSavedBD[MAX_TX_Q] = {NULL, NULL, NULL, NULL}; #define MAX_RX_Q 8 //The queue 0 and 1 are used by CPMAC. 2-7 are available for us. static Cppi4BD *RxSavedBD[MAX_RX_Q] = {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}; #ifdef SAR_DBG #define DBG_PRINT DDA_printf #else #define DBG_PRINT(szFmt, args...) #endif typedef enum{ RX_TEARDOWN, TX_TEARDOWN }ddc_sar_teardown_t; int ddc_sar_do_tear_down(CpsarDDCObj *hDDC, Cppi4BD *InputBD, ddc_sar_teardown_t txrx); int ddc_cpsar_check_for_rx_error(CpsarDDCObj *hDDC, Cppi4BD *sopBD, unsigned int channel); /** * DDC_cpsarSend * - queues a packet onto a hardware queue * - supports multifragment packets * - no channel validation performed in the interest of speed * - sendArgs Bit Mapping as follows: * * (Note: Bits 15-0 map directly to CPPI4 BD Word 2 bits 31-16.) * 17-16 Priority Queue. 0-3. (Only valid for Ch 0) * 15-12 Packet Type * 11 Descriptor Type (0 = Host, 1 = Embedded Mode) * 10-8 Additional Buffer Count (0-7). Set to 0 for Host Mode. * 7 Reserved * 6-4 Protocol Specific Region Offset. Set to 0 for Host Mode. * 3-0 Reserved. */ int DDC_cpsarSend(CpsarDDCObj *hDDC, DDC_NetPktObj *pkt, int channel, void * sendArgs) { int retVal = CPSAR_SUCCESS; Cppi4BD *currBD, *sopBD; CpsarTxCppiCh *txCppi; DDC_NetBufObj *bufList; unsigned int lockKey, cnt; int flag = 0; bufList = pkt->bufList; /* Get handle to the buffer array */ /* * \note: The critical section is required to protect the "txCppi" data * structure which is being used by this function and by cpsarTxBDProc(). * Since the function requires to protect the resources in many places in the * function below, it would be wise to get and release the crit section only * once rather than doing it in many places below. Interrupt protection is * sought here so that this function or the cpsarTxBDProc() function can be * invoked in any context and still will be safe to do so. */ PAL_osProtectEntry(PAL_OSPROTECT_INTERRUPT, &lockKey); txCppi = &hDDC->txCppi[channel]; currBD = txCppi->bdPoolHead; if (currBD == NULL) { #ifdef CPSAR_DDC_GETSTATS txCppi->outOfTxBD++; #endif retVal = CPSAR_ERR_TX_OUT_OF_BD; flag = 1; goto Exit_DDC_cpsarSend; } DBG_PRINT("%s: currBD = 0x%p \n", __FUNCTION__, currBD); currBD = (void *)CPSAR_PHYS_2_VIRT(currBD); sopBD = currBD; /* set the channel number in the SOP tag only */ currBD->tags = channel << 20; /* Store the OS pkt pointer for use in SendComplete, so that I can free multiple fragments. */ currBD->pktToken = pkt->pktToken; /* Multiple Tx BD for the packet to be sent */ for (cnt = 0; cnt < pkt->numBufs; cnt++) { /* Populate the BD contents to be added to the TX list */ currBD->nextPktPtr = 0; currBD->pktInfo = (((unsigned int)sendArgs << 16) | pkt->pktLength); currBD->buf0Ptr = CPSAR_VIRT_2_PHYS((Int32 *)bufList->dataPtr); currBD->off_bLen = bufList->length; currBD->bufToken = bufList->bufToken; if (cnt == (pkt->numBufs -1)) { /* last fragment */ txCppi->bdPoolHead = currBD->nextBDPtr; currBD->nextBDPtr = 0; sopBD->txEopBD = (void *) currBD; CPSAR_CACHE_WRITEBACK(currBD, CPPI4_BD_LENGTH_FOR_CACHE); } else { /* start or middle of packet */ currBD->nextBDPtr = (void *) CPSAR_VIRT_2_PHYS(currBD->nextBDPtr); CPSAR_CACHE_WRITEBACK(currBD, CPPI4_BD_LENGTH_FOR_CACHE); currBD = currBD->nextBDPtr; if (currBD) { currBD = (void *)CPSAR_PHYS_2_VIRT(currBD); } else { #ifdef CPSAR_DDC_GETSTATS txCppi->outOfTxBD++; #endif retVal = CPSAR_ERR_TX_OUT_OF_BD; flag = 2; goto Exit_DDC_cpsarSend; } } bufList++; } /* Call PAL to send the packet, last param is priority */ if (channel == 0) { PAL_cppi4TxPush(txCppi->cppi4TxChHnd, (void *)CPSAR_VIRT_2_PHYS(sopBD), (((unsigned int)sendArgs >> 16) & 3), NULL); } else { PAL_cppi4TxPush(txCppi->cppi4TxChHnd, (void *)CPSAR_VIRT_2_PHYS(sopBD), 0, NULL); } DBG_PRINT("%s: sopBD = 0x%p \n",__FUNCTION__, CPSAR_VIRT_2_PHYS(sopBD)); Exit_DDC_cpsarSend: /* End of Send Critical Section */ PAL_osProtectExit(PAL_OSPROTECT_INTERRUPT,lockKey); DBG_PRINT("%s: flag = %d \n", __FUNCTION__, flag); return (flag); } /** * cpsarTxPktProcess * - transmit ISR, registered with interrupt controller * - Invokes cpsarTxBDProc, checks return to see if EOQ was reached * - if EOQ not reached, don't EOI, the calling task has to re-call this routine. */ int cpsarTxPktProcess (CpsarDDCObj *hDDC, int tx_queue, int pktsToProcess, int * pktsServiced) { Bool isEOQ; *pktsServiced = cpsarTxBDProc(hDDC, tx_queue, pktsToProcess, &isEOQ); #ifdef NO_EP_SAR /* Decide whether to EOI or retrigger the interrupt */ if (isEOQ == True) { ddc_sar_do_tx_eoi(hDDC, tx_queue); } #endif #ifdef CPSAR_DDC_GETSTATS hDDC->txIntCount[tx_queue]++; #endif return (isEOQ); } /************************************************************ NEED FN HEADER ************************************************************/ inline Cppi4BD *cpsarGetTxBD (unsigned int tx_queue) { Cppi4BD *currBD; volatile unsigned int popBD; volatile unsigned int *p = (volatile unsigned int *)0xa306f000; currBD = TxSavedBD[tx_queue]; DBG_PRINT("%s: currBD = 0x%p \n", __FUNCTION__, currBD); if (!currBD) { p += tx_queue; popBD = *p; DBG_PRINT("%s: popBD = 0x%p \n", __FUNCTION__, popBD); currBD = (popBD & 0x7FFFFFFF) ? (Cppi4BD *) popBD : NULL; } if (currBD) { currBD = CPSAR_PHYS_2_VIRT (currBD); CPSAR_CACHE_INVALIDATE(currBD, CPPI4_BD_LENGTH_FOR_CACHE); DBG_PRINT("%s: currBD->nextPktPtr = 0x%p \n", __FUNCTION__, currBD->nextPktPtr); /* Save only if the currBD is not a teardown descriptor. */ if(!((unsigned int)currBD & 0x1)) TxSavedBD[tx_queue] = (void *) currBD->nextPktPtr; else TxSavedBD[tx_queue] = NULL; } return (currBD); } /** * cpsarTxBDProc * - CPSAR DDC TX Buffer Descriptor processing * - processes transmit completed packets and returns the handles to DDA layer * - 'queue' (0-7) indicates the completion queue to process * - channel and queue validations should not be performed for speed * * \note returns number of pkts processed and returns false in isEOQ if pkt * completion processing pending. */ int cpsarTxBDProc(CpsarDDCObj *hDDC, unsigned int tx_queue, int pktsToProcess, Bool *isEOQ) { Cppi4BD *currBD, *eopBD; CpsarTxCppiCh *txCppi; unsigned int pktsProcessed = 0, lockKey; unsigned int portChannel = 0; unsigned int ch_mask; #if 0 /*Move inside the while loop*/ //PAL_osProtectEntry(PAL_OSPROTECT_INTERRUPT, &lockKey); #endif /* Get TX BD list to process */ /* This 0x7fffffff test is necessary because currBD value may be 0x80000000 */ while ((currBD = cpsarGetTxBD(tx_queue)) && (unsigned int)currBD & 0x7fffffff) { DBG_PRINT("%s: currBD = 0x%p \n", __FUNCTION__, currBD); /* check for teardown condition here. This allows packets to be processed normally up until the teardown descriptor shows up */ if (unlikely((unsigned int)currBD & 0x1)) { if (ddc_sar_do_tear_down(hDDC, currBD, TX_TEARDOWN) < 0) { DDA_printf("%s: Got incorrect teardown BD %p \n", __FUNCTION__, currBD); } //We must do an eoi before we go any further. currBD = 0; break; } DBG_PRINT("%s: currBD = 0x%p \n", __FUNCTION__, currBD); /* The invalidate must be here because invalidating a teardown descriptor, which is only 2 words long, can have side effects on adjacent memory */ CPSAR_CACHE_INVALIDATE(currBD, CPPI4_BD_LENGTH_FOR_CACHE); /* get associated SAR channel, and obtain pointer to channel struct. Note that with the CPPI4 architecture, packets from different SAR channels may be processed on the same completion queue. */ portChannel = (currBD->tags&0x03f00000) >> 20; ch_mask = 1 << portChannel; /* The EOP BD pointer was saved in the descriptor by Send */ eopBD = currBD->txEopBD; /* Single packet TX complete notify */ if(hDDC->txIsOpen & ch_mask) { DDC_send_complete(hDDC->hDDA, (DDC_NetDataToken)currBD->pktToken, 1, (void *)portChannel); DBG_PRINT("%s: Send complete called with currBD = 0x%p \n", __FUNCTION__, currBD); } #ifndef NO_EP_SAR else { DDA_printf("%s: Warning: Channel %d is not open.\n", __FUNCTION__, portChannel); } #endif /* Return TX BD's to the software list - this is protected by critical section */ PAL_osProtectEntry(PAL_OSPROTECT_INTERRUPT, &lockKey); txCppi = &hDDC->txCppi[portChannel]; eopBD->nextBDPtr = (void *) txCppi->bdPoolHead; txCppi->bdPoolHead = currBD; PAL_osProtectExit(PAL_OSPROTECT_INTERRUPT,lockKey); if (++pktsProcessed == pktsToProcess) break; } /* End of while loop */ DBG_PRINT("%s: Reached end of while \n", __FUNCTION__); //if ((unsigned int)currBD & 0x7fffffff) if (currBD) { /* Save off last BD on a per queue basis. The next call to TxPop will retrieve this BD. */ TxSavedBD[tx_queue] = currBD; DBG_PRINT("%s: Had to save 0x%p on the saved BD list\n", __FUNCTION__, currBD); /* Indicate to caller that we did not process the entire list */ *isEOQ = False; } else *isEOQ = True; #ifdef CPSAR_DDC_GETSTATS if(pktsProcessed == 0) hDDC->txEmptyIntCount[tx_queue]++; #endif #if 0 //PAL_osProtectExit(PAL_OSPROTECT_INTERRUPT,lockKey); #endif return (pktsProcessed); } /** * cpsarRxPktProcess * - receive ISR, registered with interrupt controller * - Invokes cpsarRxBDProc, checks to see if EOQ was reached * */ int cpsarRxPktProcess (CpsarDDCObj *hDDC, int rxQueue, int pktsPending, int * pktsServiced) { Bool isEOQ; DBG_PRINT("%s: called with rxQueue = %d \n", __FUNCTION__, rxQueue); *pktsServiced = cpsarRxBDProc(hDDC, rxQueue, pktsPending, &isEOQ); #ifdef NO_EP_SAR /* Decide whether to EOI or retrigger the interrupt */ if (isEOQ == True) { ddc_sar_do_rx_eoi(hDDC, rxQueue); } #endif #ifdef CPSAR_DDC_GETSTATS hDDC->rxIntCount[rxQueue]++; #endif return (isEOQ); } /************************************************************ NEED FN HEADER ************************************************************/ inline Cppi4BD *cpsarGetRxBD (unsigned int rxQueue) { Cppi4BD *currBD; volatile unsigned int *RxQue = (volatile unsigned int *) (0xa306f800 + (0x10 * rxQueue)); currBD = RxSavedBD[rxQueue]; DBG_PRINT("%s: currBD = 0x%p \n", __FUNCTION__, currBD); if (!currBD) { unsigned int popBD; popBD = *RxQue; DBG_PRINT("%s: popBD = 0x%p \n", __FUNCTION__, popBD); currBD = (popBD & 0x7FFFFFFF) ? (Cppi4BD *) popBD : NULL; } RxSavedBD[rxQueue] = NULL; return (currBD); } /** * cpsarRxBDProc * - CPSAR DDC RX Buffer Descriptor processing * - processes received packets and passes them to DDA layer * - requeues the buffer descriptor to the receive pool * - channel and queue validations should not be performed for speed * * \note return number of pkts processed and returns TRUE in isEOQ and FALSE * if pkt processing not completed. */ int cpsarRxBDProc(CpsarDDCObj *hDDC, unsigned int rxQueue, int pktsToBeProcessed, Bool *isEOQ) { CpsarRxCppiCh *rxCppi; Cppi4BD *sopBD = CPSAR_PHYS_2_VIRT(0); Uint32 pktsProcessed = 0; DDC_NetPktObj *currPkt; Uint32 recycle_pkt = 0; char *newBuffer; DDC_NetDataToken newBufToken; DDC_NetBufObj *rxBufObj; Bool processing_done = FALSE; *isEOQ = True; sopBD = cpsarGetRxBD (rxQueue); sopBD = CPSAR_PHYS_2_VIRT(sopBD); while (sopBD != CPSAR_PHYS_2_VIRT(0)) { unsigned int portChannel; unsigned int ch_mask; Cppi4BD *nextPktBD = NULL; sopBD = CPSAR_PHYS_2_VIRT(sopBD); recycle_pkt = 0; processing_done = TRUE; /* AV: Handle teardown */ /* check for teardown condition here. This allows packets to be processed normally up until the teardown descriptor shows up */ if (unlikely((unsigned int)sopBD & 0x1)) { if (ddc_sar_do_tear_down(hDDC, sopBD, RX_TEARDOWN) < 0) DDA_printf("%s: Got incorrect teardown BD %p \n", __FUNCTION__, sopBD); sopBD = CPSAR_PHYS_2_VIRT(0); /* Go back and read from the hardware. */ break; } CPSAR_CACHE_INVALIDATE (sopBD, CPPI4_BD_LENGTH_FOR_CACHE); nextPktBD = (Cppi4BD *)(CPSAR_PHYS_2_VIRT (sopBD->nextPktPtr)); portChannel = (sopBD->tags & 0x03f00000) >> 20; rxCppi = &hDDC->rxCppi[portChannel]; ch_mask = 1 << portChannel; currPkt = &rxCppi->pktQueue[0]; /* Check for errors reported by the SAR. */ if (unlikely((sopBD->pktInfo & 0x00800000) || (sopBD->pktInfo == 0))) { ddc_cpsar_check_for_rx_error(hDDC, sopBD, portChannel); goto Recycle_CpsarRxBDProc; } /* Get a local reference to the DDC buffer structure before walking list */ rxBufObj = &currPkt->bufList[0]; rxBufObj->dataPtr = (char *) sopBD->dataPtr; rxBufObj->length = sopBD->off_bLen & CPPI4_BD_BUF_SIZE_MASK; rxBufObj->bufToken = sopBD->bufToken; currPkt->pktToken = currPkt->bufList->bufToken; currPkt->numBufs = 1; currPkt->pktLength = (sopBD->pktInfo & CPPI4_BD_PKT_LENGTH_MASK); newBuffer = (char *)DDC_malloc_rxbuffer(hDDC->hDDA, (DDC_NetDataToken *)&newBufToken, rxCppi->chInfo.RxBufSize, portChannel); if (newBuffer == NULL) { #ifdef CPSAR_DDC_GETSTATS ++rxCppi->outOfRxBuffers; #endif recycle_pkt = 1; goto Recycle_CpsarRxBDProc; } /* Send packet to DDA. Assuming that the NetPktObj that is sent can be * reused upon return of this call. */ if(hDDC->rxIsOpen & ch_mask) { DDC_sar_receive(hDDC->hDDA, &rxCppi->pktQueue[0], (void *)portChannel, (void *) sopBD); } else { #ifndef NO_EP_SAR DDA_printf("%s: Warning: Channel %d is not open.\n", __FUNCTION__, portChannel); #endif DDC_free_rxbuffer(hDDC->hDDA, currPkt->pktToken, portChannel); } sopBD->buf0Ptr = CPSAR_VIRT_2_PHYS (newBuffer); sopBD->dataPtr = (Ptr) newBuffer; sopBD->bufToken = newBufToken; Recycle_CpsarRxBDProc: do { Cppi4BD *sopBDNext; sopBDNext = sopBD->nextBDPtr; sopBD->nextPktPtr = 0; sopBD->pktInfo = 0; sopBD->off_bLen = rxCppi->chInfo.RxBufSize & 0xffff; sopBD->nextBDPtr = 0; CPSAR_CACHE_WRITEBACK (sopBD, CPPI4_BD_LENGTH_FOR_CACHE); PAL_cppi4RxFreeBDPush (rxCppi->cppi4RxChHnd,(Uint32 *) PAL_CPPI4_VIRT_2_PHYS (sopBD), NULL); if (sopBDNext) { ++rxCppi->rxMultiBDs; sopBD = CPSAR_PHYS_2_VIRT (sopBDNext); CPSAR_CACHE_INVALIDATE (sopBD, CPPI4_BD_LENGTH_FOR_CACHE); } else { break; } } while (1); sopBD = nextPktBD; if(recycle_pkt) break; if(++pktsProcessed >= pktsToBeProcessed) break; } /* End of while receive packet processing loop */ /* Save the BD if the processing ran out of ServiceMax */ if((sopBD != CPSAR_PHYS_2_VIRT(0))) { RxSavedBD[rxQueue] = sopBD; DBG_PRINT("%s: Saving the BD 0x%p on RxSavedBD[rxQueue] pkts done = %d recycle_pkt = %d processing_done = %d\n", __FUNCTION__, RxSavedBD[rxQueue], pktsProcessed, recycle_pkt, processing_done); *isEOQ = False; } #ifdef CPSAR_DDC_GETSTATS if (!(recycle_pkt) && (pktsProcessed == 0)) hDDC->rxEmptyIntCount[rxQueue]++; #endif return (pktsProcessed); } /************************************************************ NEED FN HEADER ************************************************************/ int ddc_cpsar_check_for_rx_error(CpsarDDCObj *hDDC, Cppi4BD *sopBD, unsigned int channel) { unsigned int errors, count; int *p; CpsarRxCppiCh *rxCppi = &hDDC->rxCppi[channel]; for(count = 0, p = (int*)sopBD ; count < 8; count++) DBG_PRINT("currBD[%d] = %#8x \n", count, *p++); /* update error statistics */ errors = (sopBD->pktInfo & 0x00070000)>>16; DBG_PRINT("%s: RX Error %d \n",__FUNCTION__, sopBD->pktInfo); if (errors & (1<rxCrcErrors++; if (errors & (1<rxLengthErrors++; if (errors & (1<rxAbortErrors++; return 0; } /************************************************************ NEED FN HEADER ************************************************************/ int ddc_sar_do_tx_eoi(CpsarDDCObj *hDDC, unsigned int tx_queue) { volatile unsigned int *eoi_reg = (volatile unsigned int *)0xa3069008; //PAL_cppi4TxEOI(txCppi->cppi4TxChHnd, NULL); #ifdef NO_EP_SAR *eoi_reg |= tx_queue; #else *eoi_reg = tx_queue; #endif DBG_PRINT("%s: Did EOI txQueue = %d \n", __FUNCTION__, tx_queue); return 0; } /************************************************************ NEED FN HEADER ************************************************************/ int ddc_sar_do_rx_eoi(CpsarDDCObj *hDDC, unsigned int rxQueue) { volatile unsigned int *eoi_reg = (volatile unsigned int *)0xa3069048; //PAL_cppi4RxEOI(rxCppi->cppi4RxChHnd, NULL); *eoi_reg = rxQueue; DBG_PRINT("%s: Did EOI rxQueue = %d \n", __FUNCTION__, rxQueue); return 0; } /************************************************************ NEED FN HEADER ************************************************************/ int ddc_sar_do_tear_down(CpsarDDCObj *hDDC, Cppi4BD *InputBD, ddc_sar_teardown_t txrx) { Cppi4BD * TeardownBD; Cppi4BD * currBD = InputBD; unsigned int BD; unsigned int ctr, ch_mask; CpsarTxCppiCh *txCppi; CpsarRxCppiCh *rxCppi; unsigned int *TD_pending = NULL; if(txrx == RX_TEARDOWN) TD_pending = &hDDC->rxTeardownPending; else TD_pending = &hDDC->txTeardownPending; /* strip off the LSB which is set to 1, so I can read the BD tags */ currBD = (Cppi4BD *)( (unsigned int)currBD & 0xfffffffe ); DBG_PRINT("%s: currBD = 0x%p txrx = %d\n", __FUNCTION__, currBD, txrx); /* determine which channel was torndown and clear pending flag. Assuming Rx channels = Tx channels. */ for (ctr = 0; ctr < CPSAR_MAX_RX_CHANNELS; ctr++) { ch_mask = 1 << ctr; if(*TD_pending & ch_mask) { if(txrx == RX_TEARDOWN) { rxCppi = &hDDC->rxCppi[ctr]; TeardownBD = (Cppi4BD *)PAL_cppi4GetRxTeardownBD(rxCppi->cppi4RxChHnd,NULL); } else { txCppi = &hDDC->txCppi[ctr]; TeardownBD = (Cppi4BD *)PAL_cppi4GetTxTeardownBD(txCppi->cppi4TxChHnd,NULL); } BD = (unsigned int)(0xffffff00 & ((unsigned int)TeardownBD)); DBG_PRINT("\n%s: Got teardown BD 0x%p for channel %d and BD %#x \n", __FUNCTION__, TeardownBD, ctr, BD); if(BD == (unsigned int)(0xffffff00 & ((unsigned int)currBD))) { /* Clear the teardown flag. */ *TD_pending &= ~ch_mask; DDA_printf("%s: teardown complete for channel %d TD_pending = %d \n", __FUNCTION__, ctr, *TD_pending); break; } } } return ((ctr < CPSAR_MAX_RX_CHANNELS ? ctr : -1)); }