-/*\r
- * Copyright (C) 2017-2021 THL A29 Limited, a Tencent company.\r
- * All rights reserved.\r
- *\r
- * Redistribution and use in source and binary forms, with or without\r
- * modification, are permitted provided that the following conditions are met:\r
- *\r
- * 1. Redistributions of source code must retain the above copyright notice, this\r
- * list of conditions and the following disclaimer.\r
- * 2. Redistributions in binary form must reproduce the above copyright notice,\r
- * this list of conditions and the following disclaimer in the documentation\r
- * and/or other materials provided with the distribution.\r
- *\r
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND\r
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\r
- * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR\r
- * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\r
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\r
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\r
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\r
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
- *\r
- */\r
-#include <assert.h>\r
-#include <unistd.h>\r
-#include <sys/mman.h>\r
-#include <errno.h>\r
- \r
-#include <rte_common.h>\r
-#include <rte_byteorder.h>\r
-#include <rte_log.h>\r
-#include <rte_memory.h>\r
-#include <rte_memcpy.h>\r
-#include <rte_memzone.h>\r
-#include <rte_config.h>\r
-#include <rte_eal.h>\r
-#include <rte_pci.h>\r
-#include <rte_mbuf.h>\r
-#include <rte_lcore.h>\r
-#include <rte_launch.h>\r
-#include <rte_ethdev.h>\r
-#include <rte_debug.h>\r
-#include <rte_ether.h>\r
-#include <rte_malloc.h>\r
-#include <rte_cycles.h>\r
-#include <rte_timer.h>\r
-#include <rte_thash.h>\r
-#include <rte_ip.h>\r
-#include <rte_tcp.h>\r
-#include <rte_udp.h>\r
-\r
-#include "ff_dpdk_if.h"\r
-#include "ff_dpdk_pcap.h"\r
-#include "ff_dpdk_kni.h"\r
-#include "ff_config.h"\r
-#include "ff_veth.h"\r
-#include "ff_host_interface.h"\r
-#include "ff_msg.h"\r
-#include "ff_api.h"\r
-#include "ff_memory.h"\r
-\r
-#define PAGE_SIZE 4096\r
-#define PAGE_SHIFT 12\r
-#define PAGE_MASK (PAGE_SIZE - 1)\r
-#define trunc_page(x) ((x) & ~PAGE_MASK)\r
-#define round_page(x) (((x) + PAGE_MASK) & ~PAGE_MASK)\r
-\r
-extern struct rte_mempool *pktmbuf_pool[NB_SOCKETS];\r
-extern struct lcore_conf lcore_conf;\r
-\r
-//struct ff_tx_offload;\r
-\r
-// ff_ref_pool allocate rte_mbuf without data space, which data point to bsd mbuf's data address.\r
-static struct rte_mempool *ff_ref_pool[NB_SOCKETS];\r
-\r
-#define Head_INC(h) {\\r
- if ( ++h >= TX_QUEUE_SIZE ) \\r
- h = 0;\\r
- };\r
-\r
-#define Head_DEC(h) do{\\r
- if ( --h < 0 ) \\r
- h = TX_QUEUE_SIZE-1;\\r
- }while(0);\r
-\r
-// bsd mbuf was moved into nic_tx_ring from tmp_tables, after rte_eth_tx_burst() succeed.\r
-static struct mbuf_txring nic_tx_ring[RTE_MAX_ETHPORTS];\r
-static inline int ff_txring_enqueue(struct mbuf_txring* q, void *p, int seg_num);\r
-static inline void ff_txring_init(struct mbuf_txring* r, uint32_t len);\r
-\r
-typedef struct _list_manager_s\r
-{\r
- uint64_t *ele; \r
- int size; \r
- //int FreeNum; \r
- int top;\r
-}StackList_t;\r
-\r
-static StackList_t ff_mpage_ctl = {0};\r
-static uint64_t ff_page_start = (uint64_t)NULL, ff_page_end = (uint64_t)NULL;\r
-static phys_addr_t *ff_mpage_phy = NULL;\r
-\r
-static inline void *stklist_pop(StackList_t *p);\r
-static inline int stklist_push(StackList_t * p, uint64_t val);\r
-\r
-static int stklist_init(StackList_t*p, int size)\r
-{\r
- \r
- int i = 0;\r
- \r
- if (p==NULL || size<=0){\r
- return -1;\r
- }\r
- p->size = size;\r
- p->top = 0;\r
- if ( posix_memalign((void**)&p->ele, sizeof(uint64_t), sizeof(uint64_t)*size) != 0)\r
- return -2;\r
- \r
- return 0;\r
-}\r
-\r
-static inline void *stklist_pop(StackList_t *p)\r
-{\r
- int head = 0;\r
- \r
- if (p==NULL)\r
- return NULL;\r
-\r
- if (p->top > 0 ){\r
- return (void*)p->ele[--p->top];\r
- }\r
- else\r
- return NULL;\r
-}\r
-\r
-//id: the id of element to be freed.\r
-//return code: -1: faile; >=0:OK.\r
-static inline int stklist_push(StackList_t *p, const uint64_t val){\r
- int tail = 0;\r
- \r
- if (p==NULL)\r
- return -1;\r
- if (p->top < p->size){\r
- p->ele[p->top++] = val;\r
- return 0;\r
- }\r
- else\r
- return -1;\r
-}\r
-\r
-static inline int stklist_size(StackList_t * p)\r
-{\r
- return p->size;\r
-}\r
-\r
-// set (void*) to rte_mbuf's priv_data.\r
-static inline int ff_mbuf_set_uint64(struct rte_mbuf* p, uint64_t data)\r
-{\r
- if (rte_pktmbuf_priv_size(p->pool) >= sizeof(uint64_t))\r
- *((uint64_t*)(p+1)) = data;\r
- return 0;\r
-}\r
-\r
-/*************************\r
-* if mbuf has num segment in all, Dev's sw_ring will use num descriptions. ff_txring also use num segments as below:\r
-* <--- num-1 ---->|ptr| head |\r
-* ----------------------------------------------\r
-* | 0 | 0 | ..............| 0 | p | XXX | \r
-*-----------------------------------------------\r
-*************************/\r
-static inline int ff_txring_enqueue(struct mbuf_txring* q, void *p, int seg_num)\r
-{\r
- int i = 0;\r
- for ( i=0; i<seg_num-1; i++){\r
- if ( q->m_table[q->head] ){\r
- ff_mbuf_free(q->m_table[q->head]);\r
- q->m_table[q->head] = NULL;\r
- }\r
- Head_INC(q->head);\r
- }\r
- if ( q->m_table[q->head] )\r
- ff_mbuf_free(q->m_table[q->head]);\r
- q->m_table[q->head] = p;\r
- Head_INC(q->head);\r
- \r
- return 0;\r
-}\r
-\r
-// pop out from head-1 .\r
-static inline int ff_txring_pop(struct mbuf_txring* q, int num)\r
-{\r
- int i = 0;\r
-\r
- for (i=0; i<num; i++){\r
- Head_DEC(q->head);\r
- if ( (i==0 && q->m_table[q->head]==NULL) || (i>0 && q->m_table[q->head]!=NULL) ){\r
- rte_panic("ff_txring_pop fatal error!");\r
- }\r
- if ( q->m_table[q->head] != NULL ){\r
- ff_mbuf_free(q->m_table[q->head]);\r
- q->m_table[q->head] = NULL;\r
- }\r
- } \r
-}\r
-\r
-static inline void ff_txring_init(struct mbuf_txring* q, uint32_t num)\r
-{\r
- memset(q, 0, sizeof(struct mbuf_txring)*num);\r
-}\r
-\r
-void ff_init_ref_pool(int nb_mbuf, int socketid)\r
-{\r
- char s[64] = {0};\r
- \r
- if (ff_ref_pool[socketid] != NULL) {\r
- return;\r
- }\r
- snprintf(s, sizeof(s), "ff_ref_pool_%d", socketid);\r
- if (rte_eal_process_type() == RTE_PROC_PRIMARY) {\r
- ff_ref_pool[socketid] = rte_pktmbuf_pool_create(s, nb_mbuf, MEMPOOL_CACHE_SIZE, 0, 0, socketid);\r
- } else {\r
- ff_ref_pool[socketid] = rte_mempool_lookup(s);\r
- }\r
-}\r
-\r
-int ff_mmap_init()\r
-{\r
- int err = 0;\r
- int i = 0;\r
- uint64_t virt_addr = (uint64_t)NULL;\r
- phys_addr_t phys_addr = 0;\r
- uint64_t bsd_memsz = (ff_global_cfg.freebsd.mem_size << 20);\r
- unsigned int bsd_pagesz = 0;\r
- \r
- ff_page_start = (uint64_t)mmap( NULL, bsd_memsz, PROT_READ | PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_POPULATE, -1, 0);\r
- if (ff_page_start == (uint64_t)-1){\r
- rte_panic("ff_mmap_init get ff_page_start failed, err=%d.\n", errno);\r
- return -1;\r
- }\r
- \r
- if ( mlock((void*)ff_page_start, bsd_memsz)<0 ) {\r
- rte_panic("mlock failed, err=%d.\n", errno);\r
- return -1;\r
- }\r
- ff_page_end = ff_page_start + bsd_memsz;\r
- bsd_pagesz = (bsd_memsz>>12);\r
- rte_log(RTE_LOG_INFO, RTE_LOGTYPE_USER1, "ff_mmap_init mmap %d pages, %d MB.\n", bsd_pagesz, ff_global_cfg.freebsd.mem_size);\r
- printf("ff_mmap_init mem[0x%lx:0x%lx]\n", ff_page_start, ff_page_end);\r
-\r
- if (posix_memalign((void**)&ff_mpage_phy, sizeof(phys_addr_t), bsd_pagesz*sizeof(phys_addr_t))!=0){\r
- rte_panic("posix_memalign get ff_mpage_phy failed, err=%d.\n", errno);\r
- return -1;\r
- }\r
- \r
- stklist_init(&ff_mpage_ctl, bsd_pagesz);\r
- \r
- for (i=0; i<bsd_pagesz; i++ ){\r
- virt_addr = ff_page_start + PAGE_SIZE*i;\r
- memset((void*)virt_addr, 0, PAGE_SIZE);\r
- \r
- stklist_push( &ff_mpage_ctl, virt_addr);\r
- ff_mpage_phy[i] = rte_mem_virt2phy((const void*)virt_addr);\r
- if ( ff_mpage_phy[i] == RTE_BAD_IOVA ){\r
- rte_panic("rte_mem_virt2phy return invalid address.");\r
- return -1;\r
- }\r
- }\r
-\r
- ff_txring_init(&nic_tx_ring[0], RTE_MAX_ETHPORTS);\r
- \r
- return 0;\r
-}\r
-\r
-// 1: vma in fstack page table; 0: vma not in fstack pages, in DPDK pool.\r
-static inline int ff_chk_vma(const uint64_t virtaddr)\r
-{\r
- return !!( virtaddr > ff_page_start && virtaddr < ff_page_end );\r
-}\r
-\r
-/*\r
- * Get physical address of any mapped virtual address in the current process.\r
- */\r
-static inline uint64_t ff_mem_virt2phy(const void* virtaddr)\r
-{\r
- uint64_t addr = 0;\r
- uint32_t pages = 0;\r
-\r
- pages = (((uint64_t)virtaddr - (uint64_t)ff_page_start)>>PAGE_SHIFT);\r
- if (pages >= stklist_size(&ff_mpage_ctl)){\r
- rte_panic("ff_mbuf_virt2phy get invalid pages %d.", pages);\r
- return -1;\r
- }\r
- \r
- addr = ff_mpage_phy[pages] + ((const uint64_t)virtaddr & PAGE_MASK);\r
- return addr;\r
-}\r
-\r
-void *ff_mem_get_page()\r
-{\r
- return (void*)stklist_pop(&ff_mpage_ctl);\r
-}\r
-\r
-int ff_mem_free_addr(void *p)\r
-{\r
- stklist_push(&ff_mpage_ctl, (const uint64_t)p);\r
- return 0;\r
-}\r
-\r
-static inline void ff_offload_set(struct ff_dpdk_if_context *ctx, void *m, struct rte_mbuf *head)\r
-{\r
- void *data = NULL;\r
- struct ff_tx_offload offload = {0};\r
- \r
- ff_mbuf_tx_offload(m, &offload);\r
- data = rte_pktmbuf_mtod(head, void*);\r
-\r
- if (offload.ip_csum) {\r
- /* ipv6 not supported yet */\r
- struct rte_ipv4_hdr *iph;\r
- int iph_len;\r
- iph = (struct rte_ipv4_hdr *)(data + RTE_ETHER_HDR_LEN);\r
- iph_len = (iph->version_ihl & 0x0f) << 2;\r
-\r
- head->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4;\r
- head->l2_len = RTE_ETHER_HDR_LEN;\r
- head->l3_len = iph_len;\r
- }\r
-\r
- if (ctx->hw_features.tx_csum_l4) {\r
- struct rte_ipv4_hdr *iph;\r
- int iph_len;\r
- iph = (struct rte_ipv4_hdr *)(data + RTE_ETHER_HDR_LEN);\r
- iph_len = (iph->version_ihl & 0x0f) << 2;\r
-\r
- if (offload.tcp_csum) {\r
- head->ol_flags |= PKT_TX_TCP_CKSUM;\r
- head->l2_len = RTE_ETHER_HDR_LEN;\r
- head->l3_len = iph_len;\r
- }\r
-\r
- /*\r
- * TCP segmentation offload.\r
- *\r
- * - set the PKT_TX_TCP_SEG flag in mbuf->ol_flags (this flag\r
- * implies PKT_TX_TCP_CKSUM)\r
- * - set the flag PKT_TX_IPV4 or PKT_TX_IPV6\r
- * - if it's IPv4, set the PKT_TX_IP_CKSUM flag and\r
- * write the IP checksum to 0 in the packet\r
- * - fill the mbuf offload information: l2_len,\r
- * l3_len, l4_len, tso_segsz\r
- * - calculate the pseudo header checksum without taking ip_len\r
- * in account, and set it in the TCP header. Refer to\r
- * rte_ipv4_phdr_cksum() and rte_ipv6_phdr_cksum() that can be\r
- * used as helpers.\r
- */\r
- if (offload.tso_seg_size) {\r
- struct rte_tcp_hdr *tcph;\r
- int tcph_len;\r
- tcph = (struct rte_tcp_hdr *)((char *)iph + iph_len);\r
- tcph_len = (tcph->data_off & 0xf0) >> 2;\r
- tcph->cksum = rte_ipv4_phdr_cksum(iph, PKT_TX_TCP_SEG);\r
-\r
- head->ol_flags |= PKT_TX_TCP_SEG;\r
- head->l4_len = tcph_len;\r
- head->tso_segsz = offload.tso_seg_size;\r
- }\r
-\r
- if (offload.udp_csum) {\r
- head->ol_flags |= PKT_TX_UDP_CKSUM;\r
- head->l2_len = RTE_ETHER_HDR_LEN;\r
- head->l3_len = iph_len;\r
- }\r
- }\r
-}\r
-\r
-// create rte_buf refer to data which is transmit from bsd stack by EXT_CLUSTER.\r
-static inline struct rte_mbuf* ff_extcl_to_rte(void *m )\r
-{\r
- struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id];\r
- struct rte_mbuf *src_mbuf = NULL;\r
- struct rte_mbuf *p_head = NULL;\r
-\r
- src_mbuf = (struct rte_mbuf*)ff_rte_frm_extcl(m);\r
- if ( NULL==src_mbuf ){\r
- return NULL;\r
- }\r
- p_head = rte_pktmbuf_clone(src_mbuf, mbuf_pool);\r
- if (p_head == NULL){\r
- return NULL;\r
- }\r
- \r
- return p_head;\r
-}\r
-\r
-// create rte_mbuf refer to data in bsd mbuf.\r
-static inline struct rte_mbuf* ff_bsd_to_rte(void *m, int total)\r
-{\r
- struct rte_mempool *mbuf_pool = ff_ref_pool[lcore_conf.socket_id];\r
- struct rte_mbuf *p_head = NULL;\r
- struct rte_mbuf *cur = NULL, *prev = NULL, *tmp=NULL;\r
- void *data = NULL;\r
- void *p_bsdbuf = NULL;\r
- unsigned len = 0;\r
- \r
- p_head = rte_pktmbuf_alloc(mbuf_pool);\r
- if (p_head == NULL){\r
- return NULL;\r
- }\r
- p_head->pkt_len = total;\r
- p_head->nb_segs = 0;\r
- cur = p_head;\r
- p_bsdbuf = m;\r
- while ( p_bsdbuf ){\r
- if (cur == NULL) {\r
- cur = rte_pktmbuf_alloc(mbuf_pool);\r
- if (cur == NULL) {\r
- rte_pktmbuf_free(p_head);\r
- return NULL;\r
- }\r
- }\r
- ff_next_mbuf(&p_bsdbuf, &data, &len); // p_bsdbuf move to next mbuf.\r
- cur->buf_addr = data;\r
- cur->buf_iova = ff_mem_virt2phy((const void*)(cur->buf_addr));\r
- cur->data_off = 0;\r
- cur->data_len = len; \r
-\r
- p_head->nb_segs++;\r
- if (prev != NULL) {\r
- prev->next = cur;\r
- }\r
- prev = cur;\r
- cur = NULL;\r
- }\r
- \r
- return p_head;\r
-}\r
-\r
-int ff_if_send_onepkt(struct ff_dpdk_if_context *ctx, void *m, int total)\r
-{\r
- struct rte_mbuf *head = NULL;\r
- void *src_buf = NULL;\r
- void *p_data = NULL;\r
- struct lcore_conf *qconf = NULL;\r
- unsigned len = 0;\r
-\r
- if ( !m ){\r
- rte_log(RTE_LOG_CRIT, RTE_LOGTYPE_USER1, "ff_dpdk_if_send_ex input invalid NULL address.");\r
- return 0;\r
- }\r
- p_data = ff_mbuf_mtod(m);\r
- if ( ff_chk_vma((uint64_t)p_data)){\r
- head = ff_bsd_to_rte(m, total);\r
- }\r
- else if ( (head = ff_extcl_to_rte(m)) == NULL ){\r
- rte_panic("data address 0x%lx is out of page bound or not malloced by DPDK recver.", (uint64_t)p_data);\r
- return 0;\r
- }\r
- \r
- if (head == NULL){\r
- rte_log(RTE_LOG_CRIT, RTE_LOGTYPE_USER1, "ff_if_send_onepkt call ff_bsd_to_rte failed.");\r
- ff_mbuf_free(m);\r
- return 0;\r
- }\r
- \r
- ff_offload_set(ctx, m, head);\r
- qconf = &lcore_conf;\r
- len = qconf->tx_mbufs[ctx->port_id].len;\r
- qconf->tx_mbufs[ctx->port_id].m_table[len] = head;\r
- qconf->tx_mbufs[ctx->port_id].bsd_m_table[len] = m;\r
- len++;\r
-\r
- return len;\r
-}\r
-\r
-int ff_enq_tx_bsdmbuf(uint8_t portid, void *p_mbuf, int nb_segs)\r
-{\r
- return ff_txring_enqueue(&nic_tx_ring[portid], p_mbuf, nb_segs);\r
-}\r
-\r
+/*
+ * Copyright (C) 2017-2021 THL A29 Limited, a Tencent company.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#include <assert.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <errno.h>
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_memzone.h>
+#include <rte_config.h>
+#include <rte_eal.h>
+#include <rte_pci.h>
+#include <rte_mbuf.h>
+#include <rte_lcore.h>
+#include <rte_launch.h>
+#include <rte_ethdev.h>
+#include <rte_debug.h>
+#include <rte_ether.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_timer.h>
+#include <rte_thash.h>
+#include <rte_ip.h>
+#include <rte_tcp.h>
+#include <rte_udp.h>
+
+#include "ff_dpdk_if.h"
+#include "ff_dpdk_pcap.h"
+#include "ff_dpdk_kni.h"
+#include "ff_config.h"
+#include "ff_veth.h"
+#include "ff_host_interface.h"
+#include "ff_msg.h"
+#include "ff_api.h"
+#include "ff_memory.h"
+
+#define PAGE_SIZE 4096
+#define PAGE_SHIFT 12
+#define PAGE_MASK (PAGE_SIZE - 1)
+#define trunc_page(x) ((x) & ~PAGE_MASK)
+#define round_page(x) (((x) + PAGE_MASK) & ~PAGE_MASK)
+
+extern struct rte_mempool *pktmbuf_pool[NB_SOCKETS];
+extern struct lcore_conf lcore_conf;
+
+//struct ff_tx_offload;
+
+// ff_ref_pool allocate rte_mbuf without data space, which data point to bsd mbuf's data address.
+static struct rte_mempool *ff_ref_pool[NB_SOCKETS];
+
+#define Head_INC(h) {\
+ if ( ++h >= TX_QUEUE_SIZE ) \
+ h = 0;\
+ };
+
+#define Head_DEC(h) do{\
+ if ( --h < 0 ) \
+ h = TX_QUEUE_SIZE-1;\
+ }while(0);
+
+// bsd mbuf was moved into nic_tx_ring from tmp_tables, after rte_eth_tx_burst() succeed.
+static struct mbuf_txring nic_tx_ring[RTE_MAX_ETHPORTS];
+static inline int ff_txring_enqueue(struct mbuf_txring* q, void *p, int seg_num);
+static inline void ff_txring_init(struct mbuf_txring* r, uint32_t len);
+
+typedef struct _list_manager_s
+{
+ uint64_t *ele;
+ int size;
+ //int FreeNum;
+ int top;
+}StackList_t;
+
+static StackList_t ff_mpage_ctl = {0};
+static uint64_t ff_page_start = (uint64_t)NULL, ff_page_end = (uint64_t)NULL;
+static phys_addr_t *ff_mpage_phy = NULL;
+
+static inline void *stklist_pop(StackList_t *p);
+static inline int stklist_push(StackList_t * p, uint64_t val);
+
+static int stklist_init(StackList_t*p, int size)
+{
+
+ int i = 0;
+
+ if (p==NULL || size<=0){
+ return -1;
+ }
+ p->size = size;
+ p->top = 0;
+ if ( posix_memalign((void**)&p->ele, sizeof(uint64_t), sizeof(uint64_t)*size) != 0)
+ return -2;
+
+ return 0;
+}
+
+static inline void *stklist_pop(StackList_t *p)
+{
+ int head = 0;
+
+ if (p==NULL)
+ return NULL;
+
+ if (p->top > 0 ){
+ return (void*)p->ele[--p->top];
+ }
+ else
+ return NULL;
+}
+
+//id: the id of element to be freed.
+//return code: -1: faile; >=0:OK.
+static inline int stklist_push(StackList_t *p, const uint64_t val){
+ int tail = 0;
+
+ if (p==NULL)
+ return -1;
+ if (p->top < p->size){
+ p->ele[p->top++] = val;
+ return 0;
+ }
+ else
+ return -1;
+}
+
+static inline int stklist_size(StackList_t * p)
+{
+ return p->size;
+}
+
+// set (void*) to rte_mbuf's priv_data.
+static inline int ff_mbuf_set_uint64(struct rte_mbuf* p, uint64_t data)
+{
+ if (rte_pktmbuf_priv_size(p->pool) >= sizeof(uint64_t))
+ *((uint64_t*)(p+1)) = data;
+ return 0;
+}
+
+/*************************
+* if mbuf has num segment in all, Dev's sw_ring will use num descriptions. ff_txring also use num segments as below:
+* <--- num-1 ---->|ptr| head |
+* ----------------------------------------------
+* | 0 | 0 | ..............| 0 | p | XXX |
+*-----------------------------------------------
+*************************/
+static inline int ff_txring_enqueue(struct mbuf_txring* q, void *p, int seg_num)
+{
+ int i = 0;
+ for ( i=0; i<seg_num-1; i++){
+ if ( q->m_table[q->head] ){
+ ff_mbuf_free(q->m_table[q->head]);
+ q->m_table[q->head] = NULL;
+ }
+ Head_INC(q->head);
+ }
+ if ( q->m_table[q->head] )
+ ff_mbuf_free(q->m_table[q->head]);
+ q->m_table[q->head] = p;
+ Head_INC(q->head);
+
+ return 0;
+}
+
+// pop out from head-1 .
+static inline int ff_txring_pop(struct mbuf_txring* q, int num)
+{
+ int i = 0;
+
+ for (i=0; i<num; i++){
+ Head_DEC(q->head);
+ if ( (i==0 && q->m_table[q->head]==NULL) || (i>0 && q->m_table[q->head]!=NULL) ){
+ rte_panic("ff_txring_pop fatal error!");
+ }
+ if ( q->m_table[q->head] != NULL ){
+ ff_mbuf_free(q->m_table[q->head]);
+ q->m_table[q->head] = NULL;
+ }
+ }
+}
+
+static inline void ff_txring_init(struct mbuf_txring* q, uint32_t num)
+{
+ memset(q, 0, sizeof(struct mbuf_txring)*num);
+}
+
+void ff_init_ref_pool(int nb_mbuf, int socketid)
+{
+ char s[64] = {0};
+
+ if (ff_ref_pool[socketid] != NULL) {
+ return;
+ }
+ snprintf(s, sizeof(s), "ff_ref_pool_%d", socketid);
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ ff_ref_pool[socketid] = rte_pktmbuf_pool_create(s, nb_mbuf, MEMPOOL_CACHE_SIZE, 0, 0, socketid);
+ } else {
+ ff_ref_pool[socketid] = rte_mempool_lookup(s);
+ }
+}
+
+int ff_mmap_init()
+{
+ int err = 0;
+ int i = 0;
+ uint64_t virt_addr = (uint64_t)NULL;
+ phys_addr_t phys_addr = 0;
+ uint64_t bsd_memsz = (ff_global_cfg.freebsd.mem_size << 20);
+ unsigned int bsd_pagesz = 0;
+
+ ff_page_start = (uint64_t)mmap( NULL, bsd_memsz, PROT_READ | PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_POPULATE, -1, 0);
+ if (ff_page_start == (uint64_t)-1){
+ rte_panic("ff_mmap_init get ff_page_start failed, err=%d.\n", errno);
+ return -1;
+ }
+
+ if ( mlock((void*)ff_page_start, bsd_memsz)<0 ) {
+ rte_panic("mlock failed, err=%d.\n", errno);
+ return -1;
+ }
+ ff_page_end = ff_page_start + bsd_memsz;
+ bsd_pagesz = (bsd_memsz>>12);
+ rte_log(RTE_LOG_INFO, RTE_LOGTYPE_USER1, "ff_mmap_init mmap %d pages, %d MB.\n", bsd_pagesz, ff_global_cfg.freebsd.mem_size);
+ printf("ff_mmap_init mem[0x%lx:0x%lx]\n", ff_page_start, ff_page_end);
+
+ if (posix_memalign((void**)&ff_mpage_phy, sizeof(phys_addr_t), bsd_pagesz*sizeof(phys_addr_t))!=0){
+ rte_panic("posix_memalign get ff_mpage_phy failed, err=%d.\n", errno);
+ return -1;
+ }
+
+ stklist_init(&ff_mpage_ctl, bsd_pagesz);
+
+ for (i=0; (unsigned)i<bsd_pagesz; i++ ){
+ virt_addr = ff_page_start + PAGE_SIZE*i;
+ memset((void*)virt_addr, 0, PAGE_SIZE);
+
+ stklist_push( &ff_mpage_ctl, virt_addr);
+ ff_mpage_phy[i] = rte_mem_virt2phy((const void*)virt_addr);
+ if ( ff_mpage_phy[i] == RTE_BAD_IOVA ){
+ rte_panic("rte_mem_virt2phy return invalid address.");
+ return -1;
+ }
+ }
+
+ ff_txring_init(&nic_tx_ring[0], RTE_MAX_ETHPORTS);
+
+ return 0;
+}
+
+// 1: vma in fstack page table; 0: vma not in fstack pages, in DPDK pool.
+static inline int ff_chk_vma(const uint64_t virtaddr)
+{
+ return !!( virtaddr > ff_page_start && virtaddr < ff_page_end );
+}
+
+/*
+ * Get physical address of any mapped virtual address in the current process.
+ */
+static inline uint64_t ff_mem_virt2phy(const void* virtaddr)
+{
+ uint64_t addr = 0;
+ uint32_t pages = 0;
+
+ pages = (((uint64_t)virtaddr - (uint64_t)ff_page_start)>>PAGE_SHIFT);
+ if (pages >= (uint32_t)stklist_size(&ff_mpage_ctl)){
+ rte_panic("ff_mbuf_virt2phy get invalid pages %d.", pages);
+ return -1;
+ }
+
+ addr = ff_mpage_phy[pages] + ((const uint64_t)virtaddr & PAGE_MASK);
+ return addr;
+}
+
+void *ff_mem_get_page()
+{
+ return (void*)stklist_pop(&ff_mpage_ctl);
+}
+
+int ff_mem_free_addr(void *p)
+{
+ stklist_push(&ff_mpage_ctl, (const uint64_t)p);
+ return 0;
+}
+
+static inline void ff_offload_set(struct ff_dpdk_if_context *ctx, void *m, struct rte_mbuf *head)
+{
+ void *data = NULL;
+ struct ff_tx_offload offload = {0};
+
+ ff_mbuf_tx_offload(m, &offload);
+ data = rte_pktmbuf_mtod(head, void*);
+
+ if (offload.ip_csum) {
+ /* ipv6 not supported yet */
+ struct rte_ipv4_hdr *iph;
+ int iph_len;
+ iph = (struct rte_ipv4_hdr *)(data + RTE_ETHER_HDR_LEN);
+ iph_len = (iph->version_ihl & 0x0f) << 2;
+
+ head->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_IPV4;
+ head->l2_len = RTE_ETHER_HDR_LEN;
+ head->l3_len = iph_len;
+ }
+
+ if (ctx->hw_features.tx_csum_l4) {
+ struct rte_ipv4_hdr *iph;
+ int iph_len;
+ iph = (struct rte_ipv4_hdr *)(data + RTE_ETHER_HDR_LEN);
+ iph_len = (iph->version_ihl & 0x0f) << 2;
+
+ if (offload.tcp_csum) {
+ head->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM;
+ head->l2_len = RTE_ETHER_HDR_LEN;
+ head->l3_len = iph_len;
+ }
+
+ /*
+ * TCP segmentation offload.
+ *
+ * - set the PKT_TX_TCP_SEG flag in mbuf->ol_flags (this flag
+ * implies PKT_TX_TCP_CKSUM)
+ * - set the flag PKT_TX_IPV4 or PKT_TX_IPV6
+ * - if it's IPv4, set the PKT_TX_IP_CKSUM flag and
+ * write the IP checksum to 0 in the packet
+ * - fill the mbuf offload information: l2_len,
+ * l3_len, l4_len, tso_segsz
+ * - calculate the pseudo header checksum without taking ip_len
+ * in account, and set it in the TCP header. Refer to
+ * rte_ipv4_phdr_cksum() and rte_ipv6_phdr_cksum() that can be
+ * used as helpers.
+ */
+ if (offload.tso_seg_size) {
+ struct rte_tcp_hdr *tcph;
+ int tcph_len;
+ tcph = (struct rte_tcp_hdr *)((char *)iph + iph_len);
+ tcph_len = (tcph->data_off & 0xf0) >> 2;
+ tcph->cksum = rte_ipv4_phdr_cksum(iph, RTE_MBUF_F_TX_TCP_SEG);
+
+ head->ol_flags |= RTE_MBUF_F_TX_TCP_SEG;
+ head->l4_len = tcph_len;
+ head->tso_segsz = offload.tso_seg_size;
+ }
+
+ if (offload.udp_csum) {
+ head->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM;
+ head->l2_len = RTE_ETHER_HDR_LEN;
+ head->l3_len = iph_len;
+ }
+ }
+}
+
+// create rte_buf refer to data which is transmit from bsd stack by EXT_CLUSTER.
+static inline struct rte_mbuf* ff_extcl_to_rte(void *m )
+{
+ struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id];
+ struct rte_mbuf *src_mbuf = NULL;
+ struct rte_mbuf *p_head = NULL;
+
+ src_mbuf = (struct rte_mbuf*)ff_rte_frm_extcl(m);
+ if ( NULL==src_mbuf ){
+ return NULL;
+ }
+ p_head = rte_pktmbuf_clone(src_mbuf, mbuf_pool);
+ if (p_head == NULL){
+ return NULL;
+ }
+
+ return p_head;
+}
+
+// create rte_mbuf refer to data in bsd mbuf.
+static inline struct rte_mbuf* ff_bsd_to_rte(void *m, int total)
+{
+ struct rte_mempool *mbuf_pool = ff_ref_pool[lcore_conf.socket_id];
+ struct rte_mbuf *p_head = NULL;
+ struct rte_mbuf *cur = NULL, *prev = NULL, *tmp=NULL;
+ void *data = NULL;
+ void *p_bsdbuf = NULL;
+ unsigned len = 0;
+
+ p_head = rte_pktmbuf_alloc(mbuf_pool);
+ if (p_head == NULL){
+ return NULL;
+ }
+ p_head->pkt_len = total;
+ p_head->nb_segs = 0;
+ cur = p_head;
+ p_bsdbuf = m;
+ while ( p_bsdbuf ){
+ if (cur == NULL) {
+ cur = rte_pktmbuf_alloc(mbuf_pool);
+ if (cur == NULL) {
+ rte_pktmbuf_free(p_head);
+ return NULL;
+ }
+ }
+ ff_next_mbuf(&p_bsdbuf, &data, &len); // p_bsdbuf move to next mbuf.
+ cur->buf_addr = data;
+ cur->buf_iova = ff_mem_virt2phy((const void*)(cur->buf_addr));
+ cur->data_off = 0;
+ cur->data_len = len;
+
+ p_head->nb_segs++;
+ if (prev != NULL) {
+ prev->next = cur;
+ }
+ prev = cur;
+ cur = NULL;
+ }
+
+ return p_head;
+}
+
+int ff_if_send_onepkt(struct ff_dpdk_if_context *ctx, void *m, int total)
+{
+ struct rte_mbuf *head = NULL;
+ void *src_buf = NULL;
+ void *p_data = NULL;
+ struct lcore_conf *qconf = NULL;
+ unsigned len = 0;
+
+ if ( !m ){
+ rte_log(RTE_LOG_CRIT, RTE_LOGTYPE_USER1, "ff_dpdk_if_send_ex input invalid NULL address.");
+ return 0;
+ }
+ p_data = ff_mbuf_mtod(m);
+ if ( ff_chk_vma((uint64_t)p_data)){
+ head = ff_bsd_to_rte(m, total);
+ }
+ else if ( (head = ff_extcl_to_rte(m)) == NULL ){
+ rte_panic("data address 0x%lx is out of page bound or not malloced by DPDK recver.", (uint64_t)p_data);
+ return 0;
+ }
+
+ if (head == NULL){
+ rte_log(RTE_LOG_CRIT, RTE_LOGTYPE_USER1, "ff_if_send_onepkt call ff_bsd_to_rte failed.");
+ ff_mbuf_free(m);
+ return 0;
+ }
+
+ ff_offload_set(ctx, m, head);
+ qconf = &lcore_conf;
+ len = qconf->tx_mbufs[ctx->port_id].len;
+ qconf->tx_mbufs[ctx->port_id].m_table[len] = head;
+ qconf->tx_mbufs[ctx->port_id].bsd_m_table[len] = m;
+ len++;
+
+ return len;
+}
+
+int ff_enq_tx_bsdmbuf(uint8_t portid, void *p_mbuf, int nb_segs)
+{
+ return ff_txring_enqueue(&nic_tx_ring[portid], p_mbuf, nb_segs);
+}
+
projects / 0xmirror / f-stack.git / commitdiff