From 650685c1650a08ec88f3f084b3fa6ebdfae99073 Mon Sep 17 00:00:00 2001 From: sephe Date: Fri, 11 Nov 2016 07:41:38 +0000 Subject: [PATCH] MFC 308166,308167 308166 hyperv/hn: Move TSO packet fixup to an earlier place for if_transmit. While TSO packet header may be still cache-hot. Sponsored by: Microsoft Differential Revision: https://reviews.freebsd.org/D8393 308167 hyperv/hn: Directly fill chimney sending buffer for small packets. Sponsored by: Microsoft Differential Revision: https://reviews.freebsd.org/D8394 git-svn-id: svn://svn.freebsd.org/base/stable/10@308513 ccf9f872-aa2e-dd11-9fc8-001c23d0bc1f --- sys/dev/hyperv/netvsc/if_hn.c | 182 ++++++++++++++++++++++++---------- 1 file changed, 132 insertions(+), 50 deletions(-) diff --git a/sys/dev/hyperv/netvsc/if_hn.c b/sys/dev/hyperv/netvsc/if_hn.c index 27289c843..1731c975c 100644 --- a/sys/dev/hyperv/netvsc/if_hn.c +++ b/sys/dev/hyperv/netvsc/if_hn.c @@ -550,6 +550,80 @@ hn_chim_free(struct hn_softc *sc, uint32_t chim_idx) atomic_clear_long(&sc->hn_chim_bmap[idx], mask); } +#if defined(INET6) || defined(INET) +/* + * NOTE: If this function failed, the m_head would be freed. + */ +static __inline struct mbuf * +hn_tso_fixup(struct mbuf *m_head) +{ + struct ether_vlan_header *evl; + struct tcphdr *th; + int ehlen; + + KASSERT(M_WRITABLE(m_head), ("TSO mbuf not writable")); + +#define PULLUP_HDR(m, len) \ +do { \ + if (__predict_false((m)->m_len < (len))) { \ + (m) = m_pullup((m), (len)); \ + if ((m) == NULL) \ + return (NULL); \ + } \ +} while (0) + + PULLUP_HDR(m_head, sizeof(*evl)); + evl = mtod(m_head, struct ether_vlan_header *); + if (evl->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) + ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + else + ehlen = ETHER_HDR_LEN; + +#ifdef INET + if (m_head->m_pkthdr.csum_flags & CSUM_IP_TSO) { + struct ip *ip; + int iphlen; + + PULLUP_HDR(m_head, ehlen + sizeof(*ip)); + ip = mtodo(m_head, ehlen); + iphlen = ip->ip_hl << 2; + + PULLUP_HDR(m_head, ehlen + iphlen + sizeof(*th)); + th = mtodo(m_head, ehlen + iphlen); + + ip->ip_len = 0; + ip->ip_sum = 0; + th->th_sum = in_pseudo(ip->ip_src.s_addr, + ip->ip_dst.s_addr, htons(IPPROTO_TCP)); + } +#endif +#if defined(INET6) && defined(INET) + else +#endif +#ifdef INET6 + { + struct ip6_hdr *ip6; + + PULLUP_HDR(m_head, ehlen + sizeof(*ip6)); + ip6 = mtodo(m_head, ehlen); + if (ip6->ip6_nxt != IPPROTO_TCP) { + m_freem(m_head); + return (NULL); + } + + PULLUP_HDR(m_head, ehlen + sizeof(*ip6) + sizeof(*th)); + th = mtodo(m_head, ehlen + sizeof(*ip6)); + + ip6->ip6_plen = 0; + th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); + } +#endif + return (m_head); + +#undef PULLUP_HDR +} +#endif /* INET6 || INET */ + static int hn_set_rxfilter(struct hn_softc *sc) { @@ -1348,15 +1422,28 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) struct mbuf *m_head = *m_head0; struct rndis_packet_msg *pkt; uint32_t *pi_data; + void *chim = NULL; int pktlen; - /* - * extension points to the area reserved for the - * rndis_filter_packet, which is placed just after - * the netvsc_packet (and rppi struct, if present; - * length is updated later). - */ pkt = txd->rndis_pkt; + if (m_head->m_pkthdr.len + HN_RNDIS_PKT_LEN < txr->hn_chim_size) { + /* + * This packet is small enough to fit into a chimney sending + * buffer. Try allocating one chimney sending buffer now. + */ + txr->hn_tx_chimney_tried++; + txd->chim_index = hn_chim_alloc(txr->hn_sc); + if (txd->chim_index != HN_NVS_CHIM_IDX_INVALID) { + chim = txr->hn_sc->hn_chim + + (txd->chim_index * txr->hn_sc->hn_chim_szmax); + /* + * Directly fill the chimney sending buffer w/ the + * RNDIS packet message. + */ + pkt = chim; + } + } + pkt->rm_type = REMOTE_NDIS_PACKET_MSG; pkt->rm_len = sizeof(*pkt) + m_head->m_pkthdr.len; pkt->rm_dataoffset = sizeof(*pkt); @@ -1386,32 +1473,10 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { #if defined(INET6) || defined(INET) - struct ether_vlan_header *eh; - int ether_len; - - /* - * XXX need m_pullup and use mtodo - */ - eh = mtod(m_head, struct ether_vlan_header*); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) - ether_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - else - ether_len = ETHER_HDR_LEN; - pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, NDIS_LSO2_INFO_SIZE, NDIS_PKTINFO_TYPE_LSO); #ifdef INET if (m_head->m_pkthdr.csum_flags & CSUM_IP_TSO) { - struct ip *ip = - (struct ip *)(m_head->m_data + ether_len); - unsigned long iph_len = ip->ip_hl << 2; - struct tcphdr *th = - (struct tcphdr *)((caddr_t)ip + iph_len); - - ip->ip_len = 0; - ip->ip_sum = 0; - th->th_sum = in_pseudo(ip->ip_src.s_addr, - ip->ip_dst.s_addr, htons(IPPROTO_TCP)); *pi_data = NDIS_LSO2_INFO_MAKEIPV4(0, m_head->m_pkthdr.tso_segsz); } @@ -1421,12 +1486,6 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) #endif #ifdef INET6 { - struct ip6_hdr *ip6 = (struct ip6_hdr *) - (m_head->m_data + ether_len); - struct tcphdr *th = (struct tcphdr *)(ip6 + 1); - - ip6->ip6_plen = 0; - th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); *pi_data = NDIS_LSO2_INFO_MAKEIPV6(0, m_head->m_pkthdr.tso_segsz); } @@ -1457,26 +1516,25 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) pkt->rm_pktinfooffset = hn_rndis_pktmsg_offset(pkt->rm_pktinfooffset); /* - * Chimney send, if the packet could fit into one chimney buffer. + * Fast path: Chimney sending. */ - if (pkt->rm_len < txr->hn_chim_size) { - txr->hn_tx_chimney_tried++; - txd->chim_index = hn_chim_alloc(txr->hn_sc); - if (txd->chim_index != HN_NVS_CHIM_IDX_INVALID) { - uint8_t *dest = txr->hn_sc->hn_chim + - (txd->chim_index * txr->hn_sc->hn_chim_szmax); + if (chim != NULL) { + KASSERT(txd->chim_index != HN_NVS_CHIM_IDX_INVALID, + ("chimney buffer is not used")); + KASSERT(pkt == chim, ("RNDIS pkt not in chimney buffer")); - memcpy(dest, pkt, pktlen); - dest += pktlen; - m_copydata(m_head, 0, m_head->m_pkthdr.len, dest); + m_copydata(m_head, 0, m_head->m_pkthdr.len, + ((uint8_t *)chim) + pktlen); - txd->chim_size = pkt->rm_len; - txr->hn_gpa_cnt = 0; - txr->hn_tx_chimney++; - txr->hn_sendpkt = hn_txpkt_chim; - goto done; - } + txd->chim_size = pkt->rm_len; + txr->hn_gpa_cnt = 0; + txr->hn_tx_chimney++; + txr->hn_sendpkt = hn_txpkt_chim; + goto done; } + KASSERT(txd->chim_index == HN_NVS_CHIM_IDX_INVALID, + ("chimney buffer is used")); + KASSERT(pkt == txd->rndis_pkt, ("RNDIS pkt not in txdesc")); error = hn_txdesc_dmamap_load(txr, txd, &m_head, segs, &nsegs); if (error) { @@ -3306,6 +3364,16 @@ hn_start_locked(struct hn_tx_ring *txr, int len) return 1; } +#if defined(INET6) || defined(INET) + if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { + m_head = hn_tso_fixup(m_head); + if (__predict_false(m_head == NULL)) { + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + continue; + } + } +#endif + txd = hn_txdesc_get(txr); if (txd == NULL) { txr->hn_no_txdescs++; @@ -3468,6 +3536,20 @@ hn_transmit(struct ifnet *ifp, struct mbuf *m) struct hn_tx_ring *txr; int error, idx = 0; +#if defined(INET6) || defined(INET) + /* + * Perform TSO packet header fixup now, since the TSO + * packet header should be cache-hot. + */ + if (m->m_pkthdr.csum_flags & CSUM_TSO) { + m = hn_tso_fixup(m); + if (__predict_false(m == NULL)) { + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + return EIO; + } + } +#endif + /* * Select the TX ring based on flowid */ -- 2.45.0