From b38bde736ac458e833959192183fce8344d10f8c Mon Sep 17 00:00:00 2001 From: glebius Date: Wed, 22 Jan 2020 05:53:16 +0000 Subject: [PATCH] Make tcp_output() require network epoch. Enter the epoch before calling into tcp_output() from those functions, that didn't do that before. This eliminates a bunch of epoch recursions in TCP. --- sys/netinet/tcp_output.c | 4 +--- sys/netinet/tcp_stacks/bbr.c | 5 ++--- sys/netinet/tcp_stacks/rack.c | 7 +++---- sys/netinet/tcp_timer.c | 9 +++++++-- sys/netinet/tcp_usrreq.c | 32 ++++++++++++++++++++++++-------- sys/netinet/toecore.c | 2 +- 6 files changed, 38 insertions(+), 21 deletions(-) diff --git a/sys/netinet/tcp_output.c b/sys/netinet/tcp_output.c index 49fbc591849..c0480830910 100644 --- a/sys/netinet/tcp_output.c +++ b/sys/netinet/tcp_output.c @@ -193,7 +193,6 @@ cc_after_idle(struct tcpcb *tp) int tcp_output(struct tcpcb *tp) { - struct epoch_tracker et; struct socket *so = tp->t_inpcb->inp_socket; int32_t len; uint32_t recwin, sendwin; @@ -233,6 +232,7 @@ tcp_output(struct tcpcb *tp) const bool hw_tls = false; #endif + NET_EPOCH_ASSERT(); INP_WLOCK_ASSERT(tp->t_inpcb); #ifdef TCP_OFFLOAD @@ -1372,7 +1372,6 @@ tcp_output(struct tcpcb *tp) * m->m_pkthdr.len should have been set before checksum calculation, * because in6_cksum() need it. */ - NET_EPOCH_ENTER(et); #ifdef INET6 if (isipv6) { /* @@ -1458,7 +1457,6 @@ tcp_output(struct tcpcb *tp) mtu = tp->t_inpcb->inp_route.ro_rt->rt_mtu; } #endif /* INET */ - NET_EPOCH_EXIT(et); out: /* diff --git a/sys/netinet/tcp_stacks/bbr.c b/sys/netinet/tcp_stacks/bbr.c index d8bcaadf583..97a50e280e0 100644 --- a/sys/netinet/tcp_stacks/bbr.c +++ b/sys/netinet/tcp_stacks/bbr.c @@ -12091,7 +12091,6 @@ bbr_window_update_needed(struct tcpcb *tp, struct socket *so, uint32_t recwin, i static int bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv) { - struct epoch_tracker et; struct socket *so; int32_t len; uint32_t cts; @@ -13938,7 +13937,6 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv) * m->m_pkthdr.len should have been set before cksum calcuration, * because in6_cksum() need it. */ - NET_EPOCH_ENTER(et); #ifdef INET6 if (isipv6) { /* @@ -14016,7 +14014,6 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv) mtu = inp->inp_route.ro_rt->rt_mtu; } #endif /* INET */ - NET_EPOCH_EXIT(et); out: if (lgb) { @@ -14465,6 +14462,8 @@ bbr_output(struct tcpcb *tp) struct timeval tv; struct tcp_bbr *bbr; + NET_EPOCH_ASSERT(); + bbr = (struct tcp_bbr *)tp->t_fb_ptr; INP_WLOCK_ASSERT(tp->t_inpcb); (void)tcp_get_usecs(&tv); diff --git a/sys/netinet/tcp_stacks/rack.c b/sys/netinet/tcp_stacks/rack.c index bc9766e461e..ff72566daf2 100644 --- a/sys/netinet/tcp_stacks/rack.c +++ b/sys/netinet/tcp_stacks/rack.c @@ -8091,7 +8091,6 @@ rack_get_pacing_delay(struct tcp_rack *rack, struct tcpcb *tp, uint32_t len) static int rack_output(struct tcpcb *tp) { - struct epoch_tracker et; struct socket *so; uint32_t recwin, sendwin; uint32_t sb_offset; @@ -8155,8 +8154,10 @@ rack_output(struct tcpcb *tp) #ifdef KERN_TLS hw_tls = (so->so_snd.sb_flags & SB_TLS_IFNET) != 0; #endif - + + NET_EPOCH_ASSERT(); INP_WLOCK_ASSERT(inp); + #ifdef TCP_OFFLOAD if (tp->t_flags & TF_TOE) return (tcp_offload_output(tp)); @@ -9734,7 +9735,6 @@ rack_output(struct tcpcb *tp) * m->m_pkthdr.len should have been set before cksum calcuration, * because in6_cksum() need it. */ - NET_EPOCH_ENTER(et); #ifdef INET6 if (isipv6) { /* @@ -9812,7 +9812,6 @@ rack_output(struct tcpcb *tp) mtu = inp->inp_route.ro_rt->rt_mtu; } #endif /* INET */ - NET_EPOCH_EXIT(et); out: if (lgb) { diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c index d0ace8ec7a4..c3fc0c4183d 100644 --- a/sys/netinet/tcp_timer.c +++ b/sys/netinet/tcp_timer.c @@ -251,6 +251,7 @@ int tcp_totbackoff = 2559; /* sum of tcp_backoff[] */ void tcp_timer_delack(void *xtp) { + struct epoch_tracker et; struct tcpcb *tp = xtp; struct inpcb *inp; CURVNET_SET(tp->t_vnet); @@ -272,8 +273,10 @@ tcp_timer_delack(void *xtp) } tp->t_flags |= TF_ACKNOW; TCPSTAT_INC(tcps_delack); + NET_EPOCH_ENTER(et); (void) tp->t_fb->tfb_tcp_output(tp); INP_WUNLOCK(inp); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); } @@ -570,7 +573,9 @@ tcp_timer_persist(void *xtp) } tcp_setpersist(tp); tp->t_flags |= TF_FORCEDATA; + NET_EPOCH_ENTER(et); (void) tp->t_fb->tfb_tcp_output(tp); + NET_EPOCH_EXIT(et); tp->t_flags &= ~TF_FORCEDATA; #ifdef TCPDEBUG @@ -824,9 +829,9 @@ tcp_timer_rexmt(void * xtp) tp->t_rtttime = 0; cc_cong_signal(tp, NULL, CC_RTO); - + NET_EPOCH_ENTER(et); (void) tp->t_fb->tfb_tcp_output(tp); - + NET_EPOCH_EXIT(et); #ifdef TCPDEBUG if (tp != NULL && (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)) tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0, diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c index 8c4f0185772..1f374d34f7d 100644 --- a/sys/netinet/tcp_usrreq.c +++ b/sys/netinet/tcp_usrreq.c @@ -531,6 +531,7 @@ tcp6_usr_listen(struct socket *so, int backlog, struct thread *td) static int tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) { + struct epoch_tracker et; int error = 0; struct inpcb *inp; struct tcpcb *tp = NULL; @@ -571,7 +572,9 @@ tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) goto out; #endif tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); + NET_EPOCH_ENTER(et); error = tp->t_fb->tfb_tcp_output(tp); + NET_EPOCH_EXIT(et); out: TCPDEBUG2(PRU_CONNECT); TCP_PROBE2(debug__user, tp, PRU_CONNECT); @@ -584,6 +587,7 @@ tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) static int tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) { + struct epoch_tracker et; int error = 0; struct inpcb *inp; struct tcpcb *tp = NULL; @@ -654,7 +658,9 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) (error = tcp_offload_connect(so, nam)) == 0) goto out; #endif + NET_EPOCH_ENTER(et); error = tp->t_fb->tfb_tcp_output(tp); + NET_EPOCH_EXIT(et); goto out; } else { if ((inp->inp_vflag & INP_IPV6) == 0) { @@ -677,8 +683,9 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) goto out; #endif tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); + NET_EPOCH_ENTER(et); error = tp->t_fb->tfb_tcp_output(tp); - + NET_EPOCH_EXIT(et); out: /* * If the implicit bind in the connect call fails, restore @@ -882,6 +889,7 @@ tcp_usr_shutdown(struct socket *so) static int tcp_usr_rcvd(struct socket *so, int flags) { + struct epoch_tracker et; struct inpcb *inp; struct tcpcb *tp = NULL; int error = 0; @@ -911,8 +919,9 @@ tcp_usr_rcvd(struct socket *so, int flags) tcp_offload_rcvd(tp); else #endif + NET_EPOCH_ENTER(et); tp->t_fb->tfb_tcp_output(tp); - + NET_EPOCH_EXIT(et); out: TCPDEBUG2(PRU_RCVD); TCP_PROBE2(debug__user, tp, PRU_RCVD); @@ -953,8 +962,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, * We require the pcbinfo "read lock" if we will close the socket * as part of this call. */ - if (flags & PRUS_EOF) - NET_EPOCH_ENTER(et); + NET_EPOCH_ENTER(et); inp = sotoinpcb(so); KASSERT(inp != NULL, ("tcp_usr_send: inp == NULL")); INP_WLOCK(inp); @@ -1240,14 +1248,14 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, TCP_PROBE2(debug__user, tp, (flags & PRUS_OOB) ? PRU_SENDOOB : ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND)); INP_WUNLOCK(inp); - if (flags & PRUS_EOF) - NET_EPOCH_EXIT(et); + NET_EPOCH_EXIT(et); return (error); } static int tcp_usr_ready(struct socket *so, struct mbuf *m, int count) { + struct epoch_tracker et; struct inpcb *inp; struct tcpcb *tp; int error; @@ -1264,8 +1272,11 @@ tcp_usr_ready(struct socket *so, struct mbuf *m, int count) SOCKBUF_LOCK(&so->so_snd); error = sbready(&so->so_snd, m, count); SOCKBUF_UNLOCK(&so->so_snd); - if (error == 0) + if (error == 0) { + NET_EPOCH_ENTER(et); error = tp->t_fb->tfb_tcp_output(tp); + NET_EPOCH_EXIT(et); + } INP_WUNLOCK(inp); return (error); @@ -1921,8 +1932,13 @@ tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp tp->t_flags |= TF_NOPUSH; else if (tp->t_flags & TF_NOPUSH) { tp->t_flags &= ~TF_NOPUSH; - if (TCPS_HAVEESTABLISHED(tp->t_state)) + if (TCPS_HAVEESTABLISHED(tp->t_state)) { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); error = tp->t_fb->tfb_tcp_output(tp); + NET_EPOCH_EXIT(et); + } } goto unlock_and_done; diff --git a/sys/netinet/toecore.c b/sys/netinet/toecore.c index 656c20e0cad..bbca6cc3ec8 100644 --- a/sys/netinet/toecore.c +++ b/sys/netinet/toecore.c @@ -503,6 +503,7 @@ void toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) { + NET_EPOCH_ASSERT(); INP_WLOCK_ASSERT(inp); if (!(inp->inp_flags & INP_DROPPED)) { @@ -527,7 +528,6 @@ toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) (void) tp->t_fb->tfb_tcp_output(tp); } else { - NET_EPOCH_ASSERT(); tp = tcp_drop(tp, err); if (tp == NULL) INP_WLOCK(inp); /* re-acquire */ -- 2.45.0