sys/rpc/clnt_vc.c

   1 /*      $NetBSD: clnt_vc.c,v 1.4 2000/07/14 08:40:42 fvdl Exp $ */
   2
   3 /*-
   4  * SPDX-License-Identifier: BSD-3-Clause
   5  *
   6  * Copyright (c) 2009, Sun Microsystems, Inc.
   7  * All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions are met:
  11  * - Redistributions of source code must retain the above copyright notice,
  12  *   this list of conditions and the following disclaimer.
  13  * - Redistributions in binary form must reproduce the above copyright notice,
  14  *   this list of conditions and the following disclaimer in the documentation
  15  *   and/or other materials provided with the distribution.
  16  * - Neither the name of Sun Microsystems, Inc. nor the names of its
  17  *   contributors may be used to endorse or promote products derived
  18  *   from this software without specific prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  24  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30  * POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #if defined(LIBC_SCCS) && !defined(lint)
  34 static char *sccsid2 = "@(#)clnt_tcp.c 1.37 87/10/05 Copyr 1984 Sun Micro";
  35 static char *sccsid = "@(#)clnt_tcp.c   2.2 88/08/01 4.0 RPCSRC";
  36 static char sccsid3[] = "@(#)clnt_vc.c 1.19 89/03/16 Copyr 1988 Sun Micro";
  37 #endif
  38 #include <sys/cdefs.h>
  39 __FBSDID("$FreeBSD$");
  40
  41 /*
  42  * clnt_tcp.c, Implements a TCP/IP based, client side RPC.
  43  *
  44  * Copyright (C) 1984, Sun Microsystems, Inc.
  45  *
  46  * TCP based RPC supports 'batched calls'.
  47  * A sequence of calls may be batched-up in a send buffer.  The rpc call
  48  * return immediately to the client even though the call was not necessarily
  49  * sent.  The batching occurs if the results' xdr routine is NULL (0) AND
  50  * the rpc timeout value is zero (see clnt.h, rpc).
  51  *
  52  * Clients should NOT casually batch calls that in fact return results; that is,
  53  * the server side should be aware that a call is batched and not produce any
  54  * return message.  Batched calls that produce many result messages can
  55  * deadlock (netlock) the client and the server....
  56  *
  57  * Now go hang yourself.
  58  */
  59
  60 #include "opt_kern_tls.h"
  61
  62 #include <sys/param.h>
  63 #include <sys/systm.h>
  64 #include <sys/kernel.h>
  65 #include <sys/kthread.h>
  66 #include <sys/ktls.h>
  67 #include <sys/lock.h>
  68 #include <sys/malloc.h>
  69 #include <sys/mbuf.h>
  70 #include <sys/mutex.h>
  71 #include <sys/pcpu.h>
  72 #include <sys/proc.h>
  73 #include <sys/protosw.h>
  74 #include <sys/socket.h>
  75 #include <sys/socketvar.h>
  76 #include <sys/sx.h>
  77 #include <sys/syslog.h>
  78 #include <sys/time.h>
  79 #include <sys/uio.h>
  80
  81 #include <net/vnet.h>
  82
  83 #include <netinet/tcp.h>
  84
  85 #include <rpc/rpc.h>
  86 #include <rpc/rpc_com.h>
  87 #include <rpc/krpc.h>
  88 #include <rpc/rpcsec_tls.h>
  89
  90 struct cmessage {
  91         struct cmsghdr cmsg;
  92         struct cmsgcred cmcred;
  93 };
  94
  95 static enum clnt_stat clnt_vc_call(CLIENT *, struct rpc_callextra *,
  96     rpcproc_t, struct mbuf *, struct mbuf **, struct timeval);
  97 static void clnt_vc_geterr(CLIENT *, struct rpc_err *);
  98 static bool_t clnt_vc_freeres(CLIENT *, xdrproc_t, void *);
  99 static void clnt_vc_abort(CLIENT *);
 100 static bool_t clnt_vc_control(CLIENT *, u_int, void *);
 101 static void clnt_vc_close(CLIENT *);
 102 static void clnt_vc_destroy(CLIENT *);
 103 static bool_t time_not_ok(struct timeval *);
 104 static int clnt_vc_soupcall(struct socket *so, void *arg, int waitflag);
 105 static void clnt_vc_dotlsupcall(void *data);
 106
 107 static struct clnt_ops clnt_vc_ops = {
 108         .cl_call =      clnt_vc_call,
 109         .cl_abort =     clnt_vc_abort,
 110         .cl_geterr =    clnt_vc_geterr,
 111         .cl_freeres =   clnt_vc_freeres,
 112         .cl_close =     clnt_vc_close,
 113         .cl_destroy =   clnt_vc_destroy,
 114         .cl_control =   clnt_vc_control
 115 };
 116
 117 static void clnt_vc_upcallsdone(struct ct_data *);
 118
 119 static int      fake_wchan;
 120
 121 /*
 122  * Create a client handle for a connection.
 123  * Default options are set, which the user can change using clnt_control()'s.
 124  * The rpc/vc package does buffering similar to stdio, so the client
 125  * must pick send and receive buffer sizes, 0 => use the default.
 126  * NB: fd is copied into a private area.
 127  * NB: The rpch->cl_auth is set null authentication. Caller may wish to
 128  * set this something more useful.
 129  *
 130  * fd should be an open socket
 131  */
 132 CLIENT *
 133 clnt_vc_create(
 134         struct socket *so,              /* open file descriptor */
 135         struct sockaddr *raddr,         /* servers address */
 136         const rpcprog_t prog,           /* program number */
 137         const rpcvers_t vers,           /* version number */
 138         size_t sendsz,                  /* buffer recv size */
 139         size_t recvsz,                  /* buffer send size */
 140         int intrflag)                   /* interruptible */
 141 {
 142         CLIENT *cl;                     /* client handle */
 143         struct ct_data *ct = NULL;      /* client handle */
 144         struct timeval now;
 145         struct rpc_msg call_msg;
 146         static uint32_t disrupt;
 147         struct __rpc_sockinfo si;
 148         XDR xdrs;
 149         int error, interrupted, one = 1, sleep_flag;
 150         struct sockopt sopt;
 151
 152         if (disrupt == 0)
 153                 disrupt = (uint32_t)(long)raddr;
 154
 155         cl = (CLIENT *)mem_alloc(sizeof (*cl));
 156         ct = (struct ct_data *)mem_alloc(sizeof (*ct));
 157
 158         mtx_init(&ct->ct_lock, "ct->ct_lock", NULL, MTX_DEF);
 159         ct->ct_threads = 0;
 160         ct->ct_closing = FALSE;
 161         ct->ct_closed = FALSE;
 162         ct->ct_upcallrefs = 0;
 163         ct->ct_rcvstate = RPCRCVSTATE_NORMAL;
 164
 165         if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
 166                 error = soconnect(so, raddr, curthread);
 167                 SOCK_LOCK(so);
 168                 interrupted = 0;
 169                 sleep_flag = PSOCK;
 170                 if (intrflag != 0)
 171                         sleep_flag |= PCATCH;
 172                 while ((so->so_state & SS_ISCONNECTING)
 173                     && so->so_error == 0) {
 174                         error = msleep(&so->so_timeo, SOCK_MTX(so),
 175                             sleep_flag, "connec", 0);
 176                         if (error) {
 177                                 if (error == EINTR || error == ERESTART)
 178                                         interrupted = 1;
 179                                 break;
 180                         }
 181                 }
 182                 if (error == 0) {
 183                         error = so->so_error;
 184                         so->so_error = 0;
 185                 }
 186                 SOCK_UNLOCK(so);
 187                 if (error) {
 188                         if (!interrupted)
 189                                 so->so_state &= ~SS_ISCONNECTING;
 190                         rpc_createerr.cf_stat = RPC_SYSTEMERROR;
 191                         rpc_createerr.cf_error.re_errno = error;
 192                         goto err;
 193                 }
 194         }
 195
 196         if (!__rpc_socket2sockinfo(so, &si)) {
 197                 goto err;
 198         }
 199
 200         if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
 201                 bzero(&sopt, sizeof(sopt));
 202                 sopt.sopt_dir = SOPT_SET;
 203                 sopt.sopt_level = SOL_SOCKET;
 204                 sopt.sopt_name = SO_KEEPALIVE;
 205                 sopt.sopt_val = &one;
 206                 sopt.sopt_valsize = sizeof(one);
 207                 sosetopt(so, &sopt);
 208         }
 209
 210         if (so->so_proto->pr_protocol == IPPROTO_TCP) {
 211                 bzero(&sopt, sizeof(sopt));
 212                 sopt.sopt_dir = SOPT_SET;
 213                 sopt.sopt_level = IPPROTO_TCP;
 214                 sopt.sopt_name = TCP_NODELAY;
 215                 sopt.sopt_val = &one;
 216                 sopt.sopt_valsize = sizeof(one);
 217                 sosetopt(so, &sopt);
 218         }
 219
 220         ct->ct_closeit = FALSE;
 221
 222         /*
 223          * Set up private data struct
 224          */
 225         ct->ct_socket = so;
 226         ct->ct_wait.tv_sec = -1;
 227         ct->ct_wait.tv_usec = -1;
 228         memcpy(&ct->ct_addr, raddr, raddr->sa_len);
 229
 230         /*
 231          * Initialize call message
 232          */
 233         getmicrotime(&now);
 234         ct->ct_xid = ((uint32_t)++disrupt) ^ __RPC_GETXID(&now);
 235         call_msg.rm_xid = ct->ct_xid;
 236         call_msg.rm_direction = CALL;
 237         call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
 238         call_msg.rm_call.cb_prog = (uint32_t)prog;
 239         call_msg.rm_call.cb_vers = (uint32_t)vers;
 240
 241         /*
 242          * pre-serialize the static part of the call msg and stash it away
 243          */
 244         xdrmem_create(&xdrs, ct->ct_mcallc, MCALL_MSG_SIZE,
 245             XDR_ENCODE);
 246         if (! xdr_callhdr(&xdrs, &call_msg)) {
 247                 if (ct->ct_closeit) {
 248                         soclose(ct->ct_socket);
 249                 }
 250                 goto err;
 251         }
 252         ct->ct_mpos = XDR_GETPOS(&xdrs);
 253         XDR_DESTROY(&xdrs);
 254         ct->ct_waitchan = "rpcrecv";
 255         ct->ct_waitflag = 0;
 256
 257         /*
 258          * Create a client handle which uses xdrrec for serialization
 259          * and authnone for authentication.
 260          */
 261         sendsz = __rpc_get_t_size(si.si_af, si.si_proto, (int)sendsz);
 262         recvsz = __rpc_get_t_size(si.si_af, si.si_proto, (int)recvsz);
 263         error = soreserve(ct->ct_socket, sendsz, recvsz);
 264         if (error != 0) {
 265                 if (ct->ct_closeit) {
 266                         soclose(ct->ct_socket);
 267                 }
 268                 goto err;
 269         }
 270         cl->cl_refs = 1;
 271         cl->cl_ops = &clnt_vc_ops;
 272         cl->cl_private = ct;
 273         cl->cl_auth = authnone_create();
 274
 275         SOCKBUF_LOCK(&ct->ct_socket->so_rcv);
 276         soupcall_set(ct->ct_socket, SO_RCV, clnt_vc_soupcall, ct);
 277         SOCKBUF_UNLOCK(&ct->ct_socket->so_rcv);
 278
 279         ct->ct_raw = NULL;
 280         ct->ct_record = NULL;
 281         ct->ct_record_resid = 0;
 282         ct->ct_sslrefno = 0;
 283         TAILQ_INIT(&ct->ct_pending);
 284         return (cl);
 285
 286 err:
 287         mtx_destroy(&ct->ct_lock);
 288         mem_free(ct, sizeof (struct ct_data));
 289         mem_free(cl, sizeof (CLIENT));
 290
 291         return ((CLIENT *)NULL);
 292 }
 293
 294 static enum clnt_stat
 295 clnt_vc_call(
 296         CLIENT          *cl,            /* client handle */
 297         struct rpc_callextra *ext,      /* call metadata */
 298         rpcproc_t       proc,           /* procedure number */
 299         struct mbuf     *args,          /* pointer to args */
 300         struct mbuf     **resultsp,     /* pointer to results */
 301         struct timeval  utimeout)
 302 {
 303         struct ct_data *ct = (struct ct_data *) cl->cl_private;
 304         AUTH *auth;
 305         struct rpc_err *errp;
 306         enum clnt_stat stat;
 307         XDR xdrs;
 308         struct rpc_msg reply_msg;
 309         bool_t ok;
 310         int nrefreshes = 2;             /* number of times to refresh cred */
 311         struct timeval timeout;
 312         uint32_t xid;
 313         struct mbuf *mreq = NULL, *results;
 314         struct ct_request *cr;
 315         int error, maxextsiz, trycnt;
 316 #ifdef KERN_TLS
 317         u_int maxlen;
 318 #endif
 319
 320         cr = malloc(sizeof(struct ct_request), M_RPC, M_WAITOK);
 321
 322         mtx_lock(&ct->ct_lock);
 323
 324         if (ct->ct_closing || ct->ct_closed) {
 325                 mtx_unlock(&ct->ct_lock);
 326                 free(cr, M_RPC);
 327                 return (RPC_CANTSEND);
 328         }
 329         ct->ct_threads++;
 330
 331         if (ext) {
 332                 auth = ext->rc_auth;
 333                 errp = &ext->rc_err;
 334         } else {
 335                 auth = cl->cl_auth;
 336                 errp = &ct->ct_error;
 337         }
 338
 339         cr->cr_mrep = NULL;
 340         cr->cr_error = 0;
 341
 342         if (ct->ct_wait.tv_usec == -1) {
 343                 timeout = utimeout;     /* use supplied timeout */
 344         } else {
 345                 timeout = ct->ct_wait;  /* use default timeout */
 346         }
 347
 348         /*
 349          * After 15sec of looping, allow it to return RPC_CANTSEND, which will
 350          * cause the clnt_reconnect layer to create a new TCP connection.
 351          */
 352         trycnt = 15 * hz;
 353 call_again:
 354         mtx_assert(&ct->ct_lock, MA_OWNED);
 355         if (ct->ct_closing || ct->ct_closed) {
 356                 ct->ct_threads--;
 357                 wakeup(ct);
 358                 mtx_unlock(&ct->ct_lock);
 359                 free(cr, M_RPC);
 360                 return (RPC_CANTSEND);
 361         }
 362
 363         ct->ct_xid++;
 364         xid = ct->ct_xid;
 365
 366         mtx_unlock(&ct->ct_lock);
 367
 368         /*
 369          * Leave space to pre-pend the record mark.
 370          */
 371         mreq = m_gethdr(M_WAITOK, MT_DATA);
 372         mreq->m_data += sizeof(uint32_t);
 373         KASSERT(ct->ct_mpos + sizeof(uint32_t) <= MHLEN,
 374             ("RPC header too big"));
 375         bcopy(ct->ct_mcallc, mreq->m_data, ct->ct_mpos);
 376         mreq->m_len = ct->ct_mpos;
 377
 378         /*
 379          * The XID is the first thing in the request.
 380          */
 381         *mtod(mreq, uint32_t *) = htonl(xid);
 382
 383         xdrmbuf_create(&xdrs, mreq, XDR_ENCODE);
 384
 385         errp->re_status = stat = RPC_SUCCESS;
 386
 387         if ((! XDR_PUTINT32(&xdrs, &proc)) ||
 388             (! AUTH_MARSHALL(auth, xid, &xdrs,
 389                 m_copym(args, 0, M_COPYALL, M_WAITOK)))) {
 390                 errp->re_status = stat = RPC_CANTENCODEARGS;
 391                 mtx_lock(&ct->ct_lock);
 392                 goto out;
 393         }
 394         mreq->m_pkthdr.len = m_length(mreq, NULL);
 395
 396         /*
 397          * Prepend a record marker containing the packet length.
 398          */
 399         M_PREPEND(mreq, sizeof(uint32_t), M_WAITOK);
 400         *mtod(mreq, uint32_t *) =
 401                 htonl(0x80000000 | (mreq->m_pkthdr.len - sizeof(uint32_t)));
 402
 403         cr->cr_xid = xid;
 404         mtx_lock(&ct->ct_lock);
 405         /*
 406          * Check to see if the other end has already started to close down
 407          * the connection. The upcall will have set ct_error.re_status
 408          * to RPC_CANTRECV if this is the case.
 409          * If the other end starts to close down the connection after this
 410          * point, it will be detected later when cr_error is checked,
 411          * since the request is in the ct_pending queue.
 412          */
 413         if (ct->ct_error.re_status == RPC_CANTRECV) {
 414                 if (errp != &ct->ct_error) {
 415                         errp->re_errno = ct->ct_error.re_errno;
 416                         errp->re_status = RPC_CANTRECV;
 417                 }
 418                 stat = RPC_CANTRECV;
 419                 goto out;
 420         }
 421
 422         /* For TLS, wait for an upcall to be done, as required. */
 423         while ((ct->ct_rcvstate & (RPCRCVSTATE_NORMAL |
 424             RPCRCVSTATE_NONAPPDATA)) == 0)
 425                 msleep(&ct->ct_rcvstate, &ct->ct_lock, 0, "rpcrcvst", hz);
 426
 427         TAILQ_INSERT_TAIL(&ct->ct_pending, cr, cr_link);
 428         mtx_unlock(&ct->ct_lock);
 429
 430         if (ct->ct_sslrefno != 0) {
 431                 /*
 432                  * Copy the mbuf chain to a chain of ext_pgs mbuf(s)
 433                  * as required by KERN_TLS.
 434                  */
 435                 maxextsiz = TLS_MAX_MSG_SIZE_V10_2;
 436 #ifdef KERN_TLS
 437                 if (rpctls_getinfo(&maxlen, false, false))
 438                         maxextsiz = min(maxextsiz, maxlen);
 439 #endif
 440                 mreq = _rpc_copym_into_ext_pgs(mreq, maxextsiz);
 441         }
 442         /*
 443          * sosend consumes mreq.
 444          */
 445         error = sosend(ct->ct_socket, NULL, NULL, mreq, NULL, 0, curthread);
 446         mreq = NULL;
 447         if (error == EMSGSIZE || (error == ERESTART &&
 448             (ct->ct_waitflag & PCATCH) == 0 && trycnt-- > 0)) {
 449                 SOCKBUF_LOCK(&ct->ct_socket->so_snd);
 450                 sbwait(&ct->ct_socket->so_snd);
 451                 SOCKBUF_UNLOCK(&ct->ct_socket->so_snd);
 452                 AUTH_VALIDATE(auth, xid, NULL, NULL);
 453                 mtx_lock(&ct->ct_lock);
 454                 TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
 455                 /* Sleep for 1 clock tick before trying the sosend() again. */
 456                 msleep(&fake_wchan, &ct->ct_lock, 0, "rpclpsnd", 1);
 457                 goto call_again;
 458         }
 459
 460         reply_msg.acpted_rply.ar_verf.oa_flavor = AUTH_NULL;
 461         reply_msg.acpted_rply.ar_verf.oa_base = cr->cr_verf;
 462         reply_msg.acpted_rply.ar_verf.oa_length = 0;
 463         reply_msg.acpted_rply.ar_results.where = NULL;
 464         reply_msg.acpted_rply.ar_results.proc = (xdrproc_t)xdr_void;
 465
 466         mtx_lock(&ct->ct_lock);
 467         if (error) {
 468                 TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
 469                 errp->re_errno = error;
 470                 errp->re_status = stat = RPC_CANTSEND;
 471                 goto out;
 472         }
 473
 474         /*
 475          * Check to see if we got an upcall while waiting for the
 476          * lock. In both these cases, the request has been removed
 477          * from ct->ct_pending.
 478          */
 479         if (cr->cr_error) {
 480                 TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
 481                 errp->re_errno = cr->cr_error;
 482                 errp->re_status = stat = RPC_CANTRECV;
 483                 goto out;
 484         }
 485         if (cr->cr_mrep) {
 486                 TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
 487                 goto got_reply;
 488         }
 489
 490         /*
 491          * Hack to provide rpc-based message passing
 492          */
 493         if (timeout.tv_sec == 0 && timeout.tv_usec == 0) {
 494                 TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
 495                 errp->re_status = stat = RPC_TIMEDOUT;
 496                 goto out;
 497         }
 498
 499         error = msleep(cr, &ct->ct_lock, ct->ct_waitflag, ct->ct_waitchan,
 500             tvtohz(&timeout));
 501
 502         TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
 503
 504         if (error) {
 505                 /*
 506                  * The sleep returned an error so our request is still
 507                  * on the list. Turn the error code into an
 508                  * appropriate client status.
 509                  */
 510                 errp->re_errno = error;
 511                 switch (error) {
 512                 case EINTR:
 513                         stat = RPC_INTR;
 514                         break;
 515                 case EWOULDBLOCK:
 516                         stat = RPC_TIMEDOUT;
 517                         break;
 518                 default:
 519                         stat = RPC_CANTRECV;
 520                 }
 521                 errp->re_status = stat;
 522                 goto out;
 523         } else {
 524                 /*
 525                  * We were woken up by the upcall.  If the
 526                  * upcall had a receive error, report that,
 527                  * otherwise we have a reply.
 528                  */
 529                 if (cr->cr_error) {
 530                         errp->re_errno = cr->cr_error;
 531                         errp->re_status = stat = RPC_CANTRECV;
 532                         goto out;
 533                 }
 534         }
 535
 536 got_reply:
 537         /*
 538          * Now decode and validate the response. We need to drop the
 539          * lock since xdr_replymsg may end up sleeping in malloc.
 540          */
 541         mtx_unlock(&ct->ct_lock);
 542
 543         if (ext && ext->rc_feedback)
 544                 ext->rc_feedback(FEEDBACK_OK, proc, ext->rc_feedback_arg);
 545
 546         xdrmbuf_create(&xdrs, cr->cr_mrep, XDR_DECODE);
 547         ok = xdr_replymsg(&xdrs, &reply_msg);
 548         cr->cr_mrep = NULL;
 549
 550         if (ok) {
 551                 if ((reply_msg.rm_reply.rp_stat == MSG_ACCEPTED) &&
 552                     (reply_msg.acpted_rply.ar_stat == SUCCESS))
 553                         errp->re_status = stat = RPC_SUCCESS;
 554                 else
 555                         stat = _seterr_reply(&reply_msg, errp);
 556
 557                 if (stat == RPC_SUCCESS) {
 558                         results = xdrmbuf_getall(&xdrs);
 559                         if (!AUTH_VALIDATE(auth, xid,
 560                                 &reply_msg.acpted_rply.ar_verf,
 561                                 &results)) {
 562                                 errp->re_status = stat = RPC_AUTHERROR;
 563                                 errp->re_why = AUTH_INVALIDRESP;
 564                         } else {
 565                                 KASSERT(results,
 566                                     ("auth validated but no result"));
 567                                 *resultsp = results;
 568                         }
 569                 }               /* end successful completion */
 570                 /*
 571                  * If unsuccessful AND error is an authentication error
 572                  * then refresh credentials and try again, else break
 573                  */
 574                 else if (stat == RPC_AUTHERROR)
 575                         /* maybe our credentials need to be refreshed ... */
 576                         if (nrefreshes > 0 &&
 577                             AUTH_REFRESH(auth, &reply_msg)) {
 578                                 nrefreshes--;
 579                                 XDR_DESTROY(&xdrs);
 580                                 mtx_lock(&ct->ct_lock);
 581                                 goto call_again;
 582                         }
 583                 /* end of unsuccessful completion */
 584         }       /* end of valid reply message */
 585         else {
 586                 errp->re_status = stat = RPC_CANTDECODERES;
 587         }
 588         XDR_DESTROY(&xdrs);
 589         mtx_lock(&ct->ct_lock);
 590 out:
 591         mtx_assert(&ct->ct_lock, MA_OWNED);
 592
 593         KASSERT(stat != RPC_SUCCESS || *resultsp,
 594             ("RPC_SUCCESS without reply"));
 595
 596         if (mreq)
 597                 m_freem(mreq);
 598         if (cr->cr_mrep)
 599                 m_freem(cr->cr_mrep);
 600
 601         ct->ct_threads--;
 602         if (ct->ct_closing)
 603                 wakeup(ct);
 604
 605         mtx_unlock(&ct->ct_lock);
 606
 607         if (auth && stat != RPC_SUCCESS)
 608                 AUTH_VALIDATE(auth, xid, NULL, NULL);
 609
 610         free(cr, M_RPC);
 611
 612         return (stat);
 613 }
 614
 615 static void
 616 clnt_vc_geterr(CLIENT *cl, struct rpc_err *errp)
 617 {
 618         struct ct_data *ct = (struct ct_data *) cl->cl_private;
 619
 620         *errp = ct->ct_error;
 621 }
 622
 623 static bool_t
 624 clnt_vc_freeres(CLIENT *cl, xdrproc_t xdr_res, void *res_ptr)
 625 {
 626         XDR xdrs;
 627         bool_t dummy;
 628
 629         xdrs.x_op = XDR_FREE;
 630         dummy = (*xdr_res)(&xdrs, res_ptr);
 631
 632         return (dummy);
 633 }
 634
 635 /*ARGSUSED*/
 636 static void
 637 clnt_vc_abort(CLIENT *cl)
 638 {
 639 }
 640
 641 static bool_t
 642 clnt_vc_control(CLIENT *cl, u_int request, void *info)
 643 {
 644         struct ct_data *ct = (struct ct_data *)cl->cl_private;
 645         void *infop = info;
 646         SVCXPRT *xprt;
 647         uint64_t *p;
 648         int error;
 649         static u_int thrdnum = 0;
 650
 651         mtx_lock(&ct->ct_lock);
 652
 653         switch (request) {
 654         case CLSET_FD_CLOSE:
 655                 ct->ct_closeit = TRUE;
 656                 mtx_unlock(&ct->ct_lock);
 657                 return (TRUE);
 658         case CLSET_FD_NCLOSE:
 659                 ct->ct_closeit = FALSE;
 660                 mtx_unlock(&ct->ct_lock);
 661                 return (TRUE);
 662         default:
 663                 break;
 664         }
 665
 666         /* for other requests which use info */
 667         if (info == NULL) {
 668                 mtx_unlock(&ct->ct_lock);
 669                 return (FALSE);
 670         }
 671         switch (request) {
 672         case CLSET_TIMEOUT:
 673                 if (time_not_ok((struct timeval *)info)) {
 674                         mtx_unlock(&ct->ct_lock);
 675                         return (FALSE);
 676                 }
 677                 ct->ct_wait = *(struct timeval *)infop;
 678                 break;
 679         case CLGET_TIMEOUT:
 680                 *(struct timeval *)infop = ct->ct_wait;
 681                 break;
 682         case CLGET_SERVER_ADDR:
 683                 (void) memcpy(info, &ct->ct_addr, (size_t)ct->ct_addr.ss_len);
 684                 break;
 685         case CLGET_SVC_ADDR:
 686                 /*
 687                  * Slightly different semantics to userland - we use
 688                  * sockaddr instead of netbuf.
 689                  */
 690                 memcpy(info, &ct->ct_addr, ct->ct_addr.ss_len);
 691                 break;
 692         case CLSET_SVC_ADDR:            /* set to new address */
 693                 mtx_unlock(&ct->ct_lock);
 694                 return (FALSE);
 695         case CLGET_XID:
 696                 *(uint32_t *)info = ct->ct_xid;
 697                 break;
 698         case CLSET_XID:
 699                 /* This will set the xid of the NEXT call */
 700                 /* decrement by 1 as clnt_vc_call() increments once */
 701                 ct->ct_xid = *(uint32_t *)info - 1;
 702                 break;
 703         case CLGET_VERS:
 704                 /*
 705                  * This RELIES on the information that, in the call body,
 706                  * the version number field is the fifth field from the
 707                  * beginning of the RPC header. MUST be changed if the
 708                  * call_struct is changed
 709                  */
 710                 *(uint32_t *)info =
 711                     ntohl(*(uint32_t *)(void *)(ct->ct_mcallc +
 712                     4 * BYTES_PER_XDR_UNIT));
 713                 break;
 714
 715         case CLSET_VERS:
 716                 *(uint32_t *)(void *)(ct->ct_mcallc +
 717                     4 * BYTES_PER_XDR_UNIT) =
 718                     htonl(*(uint32_t *)info);
 719                 break;
 720
 721         case CLGET_PROG:
 722                 /*
 723                  * This RELIES on the information that, in the call body,
 724                  * the program number field is the fourth field from the
 725                  * beginning of the RPC header. MUST be changed if the
 726                  * call_struct is changed
 727                  */
 728                 *(uint32_t *)info =
 729                     ntohl(*(uint32_t *)(void *)(ct->ct_mcallc +
 730                     3 * BYTES_PER_XDR_UNIT));
 731                 break;
 732
 733         case CLSET_PROG:
 734                 *(uint32_t *)(void *)(ct->ct_mcallc +
 735                     3 * BYTES_PER_XDR_UNIT) =
 736                     htonl(*(uint32_t *)info);
 737                 break;
 738
 739         case CLSET_WAITCHAN:
 740                 ct->ct_waitchan = (const char *)info;
 741                 break;
 742
 743         case CLGET_WAITCHAN:
 744                 *(const char **) info = ct->ct_waitchan;
 745                 break;
 746
 747         case CLSET_INTERRUPTIBLE:
 748                 if (*(int *) info)
 749                         ct->ct_waitflag = PCATCH;
 750                 else
 751                         ct->ct_waitflag = 0;
 752                 break;
 753
 754         case CLGET_INTERRUPTIBLE:
 755                 if (ct->ct_waitflag)
 756                         *(int *) info = TRUE;
 757                 else
 758                         *(int *) info = FALSE;
 759                 break;
 760
 761         case CLSET_BACKCHANNEL:
 762                 xprt = (SVCXPRT *)info;
 763                 if (ct->ct_backchannelxprt == NULL) {
 764                         xprt->xp_p2 = ct;
 765                         if (ct->ct_sslrefno != 0)
 766                                 xprt->xp_tls = RPCTLS_FLAGS_HANDSHAKE;
 767                         ct->ct_backchannelxprt = xprt;
 768                 }
 769                 break;
 770
 771         case CLSET_TLS:
 772                 p = (uint64_t *)info;
 773                 ct->ct_sslsec = *p++;
 774                 ct->ct_sslusec = *p++;
 775                 ct->ct_sslrefno = *p;
 776                 if (ct->ct_sslrefno != RPCTLS_REFNO_HANDSHAKE) {
 777                         mtx_unlock(&ct->ct_lock);
 778                         /* Start the kthread that handles upcalls. */
 779                         error = kthread_add(clnt_vc_dotlsupcall, ct,
 780                             NULL, NULL, 0, 0, "krpctls%u", thrdnum++);
 781                         if (error != 0)
 782                                 panic("Can't add KRPC thread error %d", error);
 783                 } else
 784                         mtx_unlock(&ct->ct_lock);
 785                 return (TRUE);
 786
 787         case CLSET_BLOCKRCV:
 788                 if (*(int *) info) {
 789                         ct->ct_rcvstate &= ~RPCRCVSTATE_NORMAL;
 790                         ct->ct_rcvstate |= RPCRCVSTATE_TLSHANDSHAKE;
 791                 } else {
 792                         ct->ct_rcvstate &= ~RPCRCVSTATE_TLSHANDSHAKE;
 793                         ct->ct_rcvstate |= RPCRCVSTATE_NORMAL;
 794                 }
 795                 break;
 796
 797         default:
 798                 mtx_unlock(&ct->ct_lock);
 799                 return (FALSE);
 800         }
 801
 802         mtx_unlock(&ct->ct_lock);
 803         return (TRUE);
 804 }
 805
 806 static void
 807 clnt_vc_close(CLIENT *cl)
 808 {
 809         struct ct_data *ct = (struct ct_data *) cl->cl_private;
 810         struct ct_request *cr;
 811
 812         mtx_lock(&ct->ct_lock);
 813
 814         if (ct->ct_closed) {
 815                 mtx_unlock(&ct->ct_lock);
 816                 return;
 817         }
 818
 819         if (ct->ct_closing) {
 820                 while (ct->ct_closing)
 821                         msleep(ct, &ct->ct_lock, 0, "rpcclose", 0);
 822                 KASSERT(ct->ct_closed, ("client should be closed"));
 823                 mtx_unlock(&ct->ct_lock);
 824                 return;
 825         }
 826
 827         if (ct->ct_socket) {
 828                 ct->ct_closing = TRUE;
 829                 mtx_unlock(&ct->ct_lock);
 830
 831                 SOCKBUF_LOCK(&ct->ct_socket->so_rcv);
 832                 if (ct->ct_socket->so_rcv.sb_upcall != NULL) {
 833                         soupcall_clear(ct->ct_socket, SO_RCV);
 834                         clnt_vc_upcallsdone(ct);
 835                 }
 836                 SOCKBUF_UNLOCK(&ct->ct_socket->so_rcv);
 837
 838                 /*
 839                  * Abort any pending requests and wait until everyone
 840                  * has finished with clnt_vc_call.
 841                  */
 842                 mtx_lock(&ct->ct_lock);
 843                 TAILQ_FOREACH(cr, &ct->ct_pending, cr_link) {
 844                         cr->cr_xid = 0;
 845                         cr->cr_error = ESHUTDOWN;
 846                         wakeup(cr);
 847                 }
 848
 849                 while (ct->ct_threads)
 850                         msleep(ct, &ct->ct_lock, 0, "rpcclose", 0);
 851         }
 852
 853         ct->ct_closing = FALSE;
 854         ct->ct_closed = TRUE;
 855         wakeup(&ct->ct_sslrefno);
 856         mtx_unlock(&ct->ct_lock);
 857         wakeup(ct);
 858 }
 859
 860 static void
 861 clnt_vc_destroy(CLIENT *cl)
 862 {
 863         struct ct_data *ct = (struct ct_data *) cl->cl_private;
 864         struct socket *so = NULL;
 865         SVCXPRT *xprt;
 866         enum clnt_stat stat;
 867         uint32_t reterr;
 868
 869         clnt_vc_close(cl);
 870
 871         mtx_lock(&ct->ct_lock);
 872         xprt = ct->ct_backchannelxprt;
 873         ct->ct_backchannelxprt = NULL;
 874         if (xprt != NULL) {
 875                 mtx_unlock(&ct->ct_lock);       /* To avoid a LOR. */
 876                 sx_xlock(&xprt->xp_lock);
 877                 mtx_lock(&ct->ct_lock);
 878                 xprt->xp_p2 = NULL;
 879                 sx_xunlock(&xprt->xp_lock);
 880         }
 881
 882         if (ct->ct_socket) {
 883                 if (ct->ct_closeit) {
 884                         so = ct->ct_socket;
 885                 }
 886         }
 887
 888         /* Wait for the upcall kthread to terminate. */
 889         while ((ct->ct_rcvstate & RPCRCVSTATE_UPCALLTHREAD) != 0)
 890                 msleep(&ct->ct_sslrefno, &ct->ct_lock, 0,
 891                     "clntvccl", hz);
 892         mtx_unlock(&ct->ct_lock);
 893
 894         mtx_destroy(&ct->ct_lock);
 895         if (so) {
 896                 if (ct->ct_sslrefno != 0) {
 897                         /*
 898                          * If the TLS handshake is in progress, the upcall
 899                          * will fail, but the socket should be closed by the
 900                          * daemon, since the connect upcall has just failed.
 901                          */
 902                         if (ct->ct_sslrefno != RPCTLS_REFNO_HANDSHAKE) {
 903                                 /*
 904                                  * If the upcall fails, the socket has
 905                                  * probably been closed via the rpctlscd
 906                                  * daemon having crashed or been
 907                                  * restarted, so ignore return stat.
 908                                  */
 909                                 stat = rpctls_cl_disconnect(ct->ct_sslsec,
 910                                     ct->ct_sslusec, ct->ct_sslrefno,
 911                                     &reterr);
 912                         }
 913                         /* Must sorele() to get rid of reference. */
 914                         CURVNET_SET(so->so_vnet);
 915                         SOCK_LOCK(so);
 916                         sorele(so);
 917                         CURVNET_RESTORE();
 918                 } else {
 919                         soshutdown(so, SHUT_WR);
 920                         soclose(so);
 921                 }
 922         }
 923         m_freem(ct->ct_record);
 924         m_freem(ct->ct_raw);
 925         mem_free(ct, sizeof(struct ct_data));
 926         if (cl->cl_netid && cl->cl_netid[0])
 927                 mem_free(cl->cl_netid, strlen(cl->cl_netid) +1);
 928         if (cl->cl_tp && cl->cl_tp[0])
 929                 mem_free(cl->cl_tp, strlen(cl->cl_tp) +1);
 930         mem_free(cl, sizeof(CLIENT));
 931 }
 932
 933 /*
 934  * Make sure that the time is not garbage.   -1 value is disallowed.
 935  * Note this is different from time_not_ok in clnt_dg.c
 936  */
 937 static bool_t
 938 time_not_ok(struct timeval *t)
 939 {
 940         return (t->tv_sec <= -1 || t->tv_sec > 100000000 ||
 941                 t->tv_usec <= -1 || t->tv_usec > 1000000);
 942 }
 943
 944 int
 945 clnt_vc_soupcall(struct socket *so, void *arg, int waitflag)
 946 {
 947         struct ct_data *ct = (struct ct_data *) arg;
 948         struct uio uio;
 949         struct mbuf *m, *m2, **ctrlp;
 950         struct ct_request *cr;
 951         int error, rcvflag, foundreq;
 952         uint32_t xid_plus_direction[2], header;
 953         SVCXPRT *xprt;
 954         struct cf_conn *cd;
 955         u_int rawlen;
 956         struct cmsghdr *cmsg;
 957         struct tls_get_record tgr;
 958
 959         /*
 960          * RPC-over-TLS needs to block reception during
 961          * upcalls since the upcall will be doing I/O on
 962          * the socket via openssl library calls.
 963          */
 964         mtx_lock(&ct->ct_lock);
 965         if ((ct->ct_rcvstate & (RPCRCVSTATE_NORMAL |
 966             RPCRCVSTATE_NONAPPDATA)) == 0) {
 967                 /* Mark that a socket upcall needs to be done. */
 968                 if ((ct->ct_rcvstate & (RPCRCVSTATE_UPCALLNEEDED |
 969                     RPCRCVSTATE_UPCALLINPROG)) != 0)
 970                         ct->ct_rcvstate |= RPCRCVSTATE_SOUPCALLNEEDED;
 971                 mtx_unlock(&ct->ct_lock);
 972                 return (SU_OK);
 973         }
 974         mtx_unlock(&ct->ct_lock);
 975
 976         /*
 977          * If another thread is already here, it must be in
 978          * soreceive(), so just return to avoid races with it.
 979          * ct_upcallrefs is protected by the SOCKBUF_LOCK(),
 980          * which is held in this function, except when
 981          * soreceive() is called.
 982          */
 983         if (ct->ct_upcallrefs > 0)
 984                 return (SU_OK);
 985         ct->ct_upcallrefs++;
 986
 987         /*
 988          * Read as much as possible off the socket and link it
 989          * onto ct_raw.
 990          */
 991         for (;;) {
 992                 uio.uio_resid = 1000000000;
 993                 uio.uio_td = curthread;
 994                 m2 = m = NULL;
 995                 rcvflag = MSG_DONTWAIT | MSG_SOCALLBCK;
 996                 if (ct->ct_sslrefno != 0 && (ct->ct_rcvstate &
 997                     RPCRCVSTATE_NORMAL) != 0) {
 998                         rcvflag |= MSG_TLSAPPDATA;
 999                         ctrlp = NULL;
1000                 } else
1001                         ctrlp = &m2;
1002                 SOCKBUF_UNLOCK(&so->so_rcv);
1003                 error = soreceive(so, NULL, &uio, &m, ctrlp, &rcvflag);
1004                 SOCKBUF_LOCK(&so->so_rcv);
1005
1006                 if (error == EWOULDBLOCK) {
1007                         /*
1008                          * We must re-test for readability after
1009                          * taking the lock to protect us in the case
1010                          * where a new packet arrives on the socket
1011                          * after our call to soreceive fails with
1012                          * EWOULDBLOCK.
1013                          */
1014                         error = 0;
1015                         if (!soreadable(so))
1016                                 break;
1017                         continue;
1018                 }
1019                 if (error == 0 && m == NULL) {
1020                         /*
1021                          * We must have got EOF trying
1022                          * to read from the stream.
1023                          */
1024                         error = ECONNRESET;
1025                 }
1026
1027                 /*
1028                  * A return of ENXIO indicates that there is a
1029                  * non-application data record at the head of the
1030                  * socket's receive queue, for TLS connections.
1031                  * This record needs to be handled in userland
1032                  * via an SSL_read() call, so do an upcall to the daemon.
1033                  */
1034                 if (ct->ct_sslrefno != 0 && error == ENXIO) {
1035                         /* Disable reception, marking an upcall needed. */
1036                         mtx_lock(&ct->ct_lock);
1037                         ct->ct_rcvstate |= RPCRCVSTATE_UPCALLNEEDED;
1038                         /*
1039                          * If an upcall in needed, wake up the kthread
1040                          * that runs clnt_vc_dotlsupcall().
1041                          */
1042                         wakeup(&ct->ct_sslrefno);
1043                         mtx_unlock(&ct->ct_lock);
1044                         break;
1045                 }
1046                 if (error != 0)
1047                         break;
1048
1049                 /* Process any record header(s). */
1050                 if (m2 != NULL) {
1051                         cmsg = mtod(m2, struct cmsghdr *);
1052                         if (cmsg->cmsg_type == TLS_GET_RECORD &&
1053                             cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) {
1054                                 memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr));
1055                                 /*
1056                                  * This should have been handled by
1057                                  * setting RPCRCVSTATE_UPCALLNEEDED in
1058                                  * ct_rcvstate but if not, all we can do
1059                                  * is toss it away.
1060                                  */
1061                                 if (tgr.tls_type != TLS_RLTYPE_APP) {
1062                                         m_freem(m);
1063                                         m_free(m2);
1064                                         mtx_lock(&ct->ct_lock);
1065                                         ct->ct_rcvstate &=
1066                                             ~RPCRCVSTATE_NONAPPDATA;
1067                                         ct->ct_rcvstate |= RPCRCVSTATE_NORMAL;
1068                                         mtx_unlock(&ct->ct_lock);
1069                                         continue;
1070                                 }
1071                         }
1072                         m_free(m2);
1073                 }
1074
1075                 if (ct->ct_raw != NULL)
1076                         m_last(ct->ct_raw)->m_next = m;
1077                 else
1078                         ct->ct_raw = m;
1079         }
1080         rawlen = m_length(ct->ct_raw, NULL);
1081
1082         /* Now, process as much of ct_raw as possible. */
1083         for (;;) {
1084                 /*
1085                  * If ct_record_resid is zero, we are waiting for a
1086                  * record mark.
1087                  */
1088                 if (ct->ct_record_resid == 0) {
1089                         if (rawlen < sizeof(uint32_t))
1090                                 break;
1091                         m_copydata(ct->ct_raw, 0, sizeof(uint32_t),
1092                             (char *)&header);
1093                         header = ntohl(header);
1094                         ct->ct_record_resid = header & 0x7fffffff;
1095                         ct->ct_record_eor = ((header & 0x80000000) != 0);
1096                         m_adj(ct->ct_raw, sizeof(uint32_t));
1097                         rawlen -= sizeof(uint32_t);
1098                 } else {
1099                         /*
1100                          * Move as much of the record as possible to
1101                          * ct_record.
1102                          */
1103                         if (rawlen == 0)
1104                                 break;
1105                         if (rawlen <= ct->ct_record_resid) {
1106                                 if (ct->ct_record != NULL)
1107                                         m_last(ct->ct_record)->m_next =
1108                                             ct->ct_raw;
1109                                 else
1110                                         ct->ct_record = ct->ct_raw;
1111                                 ct->ct_raw = NULL;
1112                                 ct->ct_record_resid -= rawlen;
1113                                 rawlen = 0;
1114                         } else {
1115                                 m = m_split(ct->ct_raw, ct->ct_record_resid,
1116                                     M_NOWAIT);
1117                                 if (m == NULL)
1118                                         break;
1119                                 if (ct->ct_record != NULL)
1120                                         m_last(ct->ct_record)->m_next =
1121                                             ct->ct_raw;
1122                                 else
1123                                         ct->ct_record = ct->ct_raw;
1124                                 rawlen -= ct->ct_record_resid;
1125                                 ct->ct_record_resid = 0;
1126                                 ct->ct_raw = m;
1127                         }
1128                         if (ct->ct_record_resid > 0)
1129                                 break;
1130
1131                         /*
1132                          * If we have the entire record, see if we can
1133                          * match it to a request.
1134                          */
1135                         if (ct->ct_record_eor) {
1136                                 /*
1137                                  * The XID is in the first uint32_t of
1138                                  * the reply and the message direction
1139                                  * is the second one.
1140                                  */
1141                                 if (ct->ct_record->m_len <
1142                                     sizeof(xid_plus_direction) &&
1143                                     m_length(ct->ct_record, NULL) <
1144                                     sizeof(xid_plus_direction)) {
1145                                         /*
1146                                          * What to do now?
1147                                          * The data in the TCP stream is
1148                                          * corrupted such that there is no
1149                                          * valid RPC message to parse.
1150                                          * I think it best to close this
1151                                          * connection and allow
1152                                          * clnt_reconnect_call() to try
1153                                          * and establish a new one.
1154                                          */
1155                                         printf("clnt_vc_soupcall: "
1156                                             "connection data corrupted\n");
1157                                         error = ECONNRESET;
1158                                         goto wakeup_all;
1159                                 }
1160                                 m_copydata(ct->ct_record, 0,
1161                                     sizeof(xid_plus_direction),
1162                                     (char *)xid_plus_direction);
1163                                 xid_plus_direction[0] =
1164                                     ntohl(xid_plus_direction[0]);
1165                                 xid_plus_direction[1] =
1166                                     ntohl(xid_plus_direction[1]);
1167                                 /* Check message direction. */
1168                                 if (xid_plus_direction[1] == CALL) {
1169                                         /* This is a backchannel request. */
1170                                         mtx_lock(&ct->ct_lock);
1171                                         xprt = ct->ct_backchannelxprt;
1172                                         if (xprt == NULL) {
1173                                                 mtx_unlock(&ct->ct_lock);
1174                                                 /* Just throw it away. */
1175                                                 m_freem(ct->ct_record);
1176                                                 ct->ct_record = NULL;
1177                                         } else {
1178                                                 cd = (struct cf_conn *)
1179                                                     xprt->xp_p1;
1180                                                 m2 = cd->mreq;
1181                                                 /*
1182                                                  * The requests are chained
1183                                                  * in the m_nextpkt list.
1184                                                  */
1185                                                 while (m2 != NULL &&
1186                                                     m2->m_nextpkt != NULL)
1187                                                         /* Find end of list. */
1188                                                         m2 = m2->m_nextpkt;
1189                                                 if (m2 != NULL)
1190                                                         m2->m_nextpkt =
1191                                                             ct->ct_record;
1192                                                 else
1193                                                         cd->mreq =
1194                                                             ct->ct_record;
1195                                                 ct->ct_record->m_nextpkt =
1196                                                     NULL;
1197                                                 ct->ct_record = NULL;
1198                                                 xprt_active(xprt);
1199                                                 mtx_unlock(&ct->ct_lock);
1200                                         }
1201                                 } else {
1202                                         mtx_lock(&ct->ct_lock);
1203                                         foundreq = 0;
1204                                         TAILQ_FOREACH(cr, &ct->ct_pending,
1205                                             cr_link) {
1206                                                 if (cr->cr_xid ==
1207                                                     xid_plus_direction[0]) {
1208                                                         /*
1209                                                          * This one
1210                                                          * matches. We leave
1211                                                          * the reply mbuf in
1212                                                          * cr->cr_mrep. Set
1213                                                          * the XID to zero so
1214                                                          * that we will ignore
1215                                                          * any duplicated
1216                                                          * replies.
1217                                                          */
1218                                                         cr->cr_xid = 0;
1219                                                         cr->cr_mrep =
1220                                                             ct->ct_record;
1221                                                         cr->cr_error = 0;
1222                                                         foundreq = 1;
1223                                                         wakeup(cr);
1224                                                         break;
1225                                                 }
1226                                         }
1227                                         mtx_unlock(&ct->ct_lock);
1228
1229                                         if (!foundreq)
1230                                                 m_freem(ct->ct_record);
1231                                         ct->ct_record = NULL;
1232                                 }
1233                         }
1234                 }
1235         }
1236
1237         if (error != 0) {
1238         wakeup_all:
1239                 /*
1240                  * This socket is broken, so mark that it cannot
1241                  * receive and fail all RPCs waiting for a reply
1242                  * on it, so that they will be retried on a new
1243                  * TCP connection created by clnt_reconnect_X().
1244                  */
1245                 mtx_lock(&ct->ct_lock);
1246                 ct->ct_error.re_status = RPC_CANTRECV;
1247                 ct->ct_error.re_errno = error;
1248                 TAILQ_FOREACH(cr, &ct->ct_pending, cr_link) {
1249                         cr->cr_error = error;
1250                         wakeup(cr);
1251                 }
1252                 mtx_unlock(&ct->ct_lock);
1253         }
1254
1255         ct->ct_upcallrefs--;
1256         if (ct->ct_upcallrefs < 0)
1257                 panic("rpcvc upcall refcnt");
1258         if (ct->ct_upcallrefs == 0)
1259                 wakeup(&ct->ct_upcallrefs);
1260         return (SU_OK);
1261 }
1262
1263 /*
1264  * Wait for all upcalls in progress to complete.
1265  */
1266 static void
1267 clnt_vc_upcallsdone(struct ct_data *ct)
1268 {
1269
1270         SOCKBUF_LOCK_ASSERT(&ct->ct_socket->so_rcv);
1271
1272         while (ct->ct_upcallrefs > 0)
1273                 (void) msleep(&ct->ct_upcallrefs,
1274                     SOCKBUF_MTX(&ct->ct_socket->so_rcv), 0, "rpcvcup", 0);
1275 }
1276
1277 /*
1278  * Do a TLS upcall to the rpctlscd daemon, as required.
1279  * This function runs as a kthread.
1280  */
1281 static void
1282 clnt_vc_dotlsupcall(void *data)
1283 {
1284         struct ct_data *ct = (struct ct_data *)data;
1285         enum clnt_stat ret;
1286         uint32_t reterr;
1287
1288         mtx_lock(&ct->ct_lock);
1289         ct->ct_rcvstate |= RPCRCVSTATE_UPCALLTHREAD;
1290         while (!ct->ct_closed) {
1291                 if ((ct->ct_rcvstate & RPCRCVSTATE_UPCALLNEEDED) != 0) {
1292                         ct->ct_rcvstate &= ~RPCRCVSTATE_UPCALLNEEDED;
1293                         ct->ct_rcvstate |= RPCRCVSTATE_UPCALLINPROG;
1294                         if (ct->ct_sslrefno != 0 && ct->ct_sslrefno !=
1295                             RPCTLS_REFNO_HANDSHAKE) {
1296                                 mtx_unlock(&ct->ct_lock);
1297                                 ret = rpctls_cl_handlerecord(ct->ct_sslsec,
1298                                     ct->ct_sslusec, ct->ct_sslrefno, &reterr);
1299                                 mtx_lock(&ct->ct_lock);
1300                         }
1301                         ct->ct_rcvstate &= ~RPCRCVSTATE_UPCALLINPROG;
1302                         if (ret == RPC_SUCCESS && reterr == RPCTLSERR_OK)
1303                                 ct->ct_rcvstate |= RPCRCVSTATE_NORMAL;
1304                         else
1305                                 ct->ct_rcvstate |= RPCRCVSTATE_NONAPPDATA;
1306                         wakeup(&ct->ct_rcvstate);
1307                 }
1308                 if ((ct->ct_rcvstate & RPCRCVSTATE_SOUPCALLNEEDED) != 0) {
1309                         ct->ct_rcvstate &= ~RPCRCVSTATE_SOUPCALLNEEDED;
1310                         mtx_unlock(&ct->ct_lock);
1311                         SOCKBUF_LOCK(&ct->ct_socket->so_rcv);
1312                         clnt_vc_soupcall(ct->ct_socket, ct, M_NOWAIT);
1313                         SOCKBUF_UNLOCK(&ct->ct_socket->so_rcv);
1314                         mtx_lock(&ct->ct_lock);
1315                 }
1316                 msleep(&ct->ct_sslrefno, &ct->ct_lock, 0, "clntvcdu", hz);
1317         }
1318         ct->ct_rcvstate &= ~RPCRCVSTATE_UPCALLTHREAD;
1319         wakeup(&ct->ct_sslrefno);
1320         mtx_unlock(&ct->ct_lock);
1321         kthread_exit();
1322 }