2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2008 Isilon Inc http://www.isilon.com/
5 * Authors: Doug Rabson <dfr@rabson.org>
6 * Developed with Red Inc: Alfred Perlstein <alfred@freebsd.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 /* Modified from the kernel GSSAPI code for RPC-over-TLS. */
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
35 #include "opt_kern_tls.h"
37 #include <sys/param.h>
38 #include <sys/capsicum.h>
40 #include <sys/filedesc.h>
41 #include <sys/kernel.h>
43 #include <sys/malloc.h>
45 #include <sys/mutex.h>
48 #include <sys/socketvar.h>
49 #include <sys/syscall.h>
50 #include <sys/syscallsubr.h>
51 #include <sys/sysent.h>
52 #include <sys/sysproto.h>
55 #include <rpc/rpc_com.h>
56 #include <rpc/rpcsec_tls.h>
60 #include <vm/vm_param.h>
68 static struct syscall_helper_data rpctls_syscalls[] = {
69 SYSCALL_INIT_HELPER(rpctls_syscall),
73 static CLIENT *rpctls_connect_handle;
74 static struct mtx rpctls_connect_lock;
75 static struct socket *rpctls_connect_so = NULL;
76 static CLIENT *rpctls_connect_cl = NULL;
77 static CLIENT *rpctls_server_handle;
78 static struct mtx rpctls_server_lock;
79 static struct socket *rpctls_server_so = NULL;
80 static SVCXPRT *rpctls_server_xprt = NULL;
81 static struct opaque_auth rpctls_null_verf;
83 static CLIENT *rpctls_connect_client(void);
84 static CLIENT *rpctls_server_client(void);
85 static enum clnt_stat rpctls_server(SVCXPRT *xprt, struct socket *so,
86 uint32_t *flags, uint64_t *sslp,
87 uid_t *uid, int *ngrps, gid_t **gids);
94 error = syscall_helper_register(rpctls_syscalls, SY_THR_STATIC_KLD);
96 printf("rpctls_init: cannot register syscall\n");
99 mtx_init(&rpctls_connect_lock, "rpctls_connect_lock", NULL,
101 mtx_init(&rpctls_server_lock, "rpctls_server_lock", NULL,
103 rpctls_null_verf.oa_flavor = AUTH_NULL;
104 rpctls_null_verf.oa_base = RPCTLS_START_STRING;
105 rpctls_null_verf.oa_length = strlen(RPCTLS_START_STRING);
110 sys_rpctls_syscall(struct thread *td, struct rpctls_syscall_args *uap)
112 struct sockaddr_un sun;
113 struct netconfig *nconf;
117 char path[MAXPATHLEN];
118 int fd = -1, error, try_count;
119 CLIENT *cl, *oldcl, *concl;
121 struct timeval timeo;
126 error = priv_check(td, PRIV_NFS_DAEMON);
131 case RPCTLS_SYSC_CLSETPATH:
132 error = copyinstr(uap->path, path, sizeof(path), NULL);
136 if (rpctls_getinfo(&maxlen, false, false))
140 if (error == 0 && (strlen(path) + 1 > sizeof(sun.sun_path) ||
146 sun.sun_family = AF_LOCAL;
147 strlcpy(sun.sun_path, path, sizeof(sun.sun_path));
148 sun.sun_len = SUN_LEN(&sun);
150 nconf = getnetconfigent("local");
151 cl = clnt_reconnect_create(nconf,
152 (struct sockaddr *)&sun, RPCTLSCD, RPCTLSCDVERS,
153 RPC_MAXDATASIZE, RPC_MAXDATASIZE);
155 * The number of retries defaults to INT_MAX, which
156 * effectively means an infinite, uninterruptable loop.
157 * Set the try_count to 1 so that no retries of the
158 * RPC occur. Since it is an upcall to a local daemon,
159 * requests should not be lost and doing one of these
160 * RPCs multiple times is not correct.
161 * If the server is not working correctly, the
162 * daemon can get stuck in SSL_connect() trying
163 * to read data from the socket during the upcall.
164 * Set a timeout (currently 15sec) and assume the
165 * daemon is hung when the timeout occurs.
169 CLNT_CONTROL(cl, CLSET_RETRIES, &try_count);
172 CLNT_CONTROL(cl, CLSET_TIMEOUT, &timeo);
177 mtx_lock(&rpctls_connect_lock);
178 oldcl = rpctls_connect_handle;
179 rpctls_connect_handle = cl;
180 mtx_unlock(&rpctls_connect_lock);
187 case RPCTLS_SYSC_SRVSETPATH:
188 error = copyinstr(uap->path, path, sizeof(path), NULL);
192 if (rpctls_getinfo(&maxlen, false, false))
196 if (error == 0 && (strlen(path) + 1 > sizeof(sun.sun_path) ||
202 sun.sun_family = AF_LOCAL;
203 strlcpy(sun.sun_path, path, sizeof(sun.sun_path));
204 sun.sun_len = SUN_LEN(&sun);
206 nconf = getnetconfigent("local");
207 cl = clnt_reconnect_create(nconf,
208 (struct sockaddr *)&sun, RPCTLSSD, RPCTLSSDVERS,
209 RPC_MAXDATASIZE, RPC_MAXDATASIZE);
211 * The number of retries defaults to INT_MAX, which
212 * effectively means an infinite, uninterruptable loop.
213 * Set the try_count to 1 so that no retries of the
214 * RPC occur. Since it is an upcall to a local daemon,
215 * requests should not be lost and doing one of these
216 * RPCs multiple times is not correct.
217 * Set a timeout (currently 15sec) and assume that
218 * the daemon is hung if a timeout occurs.
222 CLNT_CONTROL(cl, CLSET_RETRIES, &try_count);
225 CLNT_CONTROL(cl, CLSET_TIMEOUT, &timeo);
230 mtx_lock(&rpctls_server_lock);
231 oldcl = rpctls_server_handle;
232 rpctls_server_handle = cl;
233 mtx_unlock(&rpctls_server_lock);
240 case RPCTLS_SYSC_CLSHUTDOWN:
241 mtx_lock(&rpctls_connect_lock);
242 oldcl = rpctls_connect_handle;
243 rpctls_connect_handle = NULL;
244 mtx_unlock(&rpctls_connect_lock);
251 case RPCTLS_SYSC_SRVSHUTDOWN:
252 mtx_lock(&rpctls_server_lock);
253 oldcl = rpctls_server_handle;
254 rpctls_server_handle = NULL;
255 mtx_unlock(&rpctls_server_lock);
262 case RPCTLS_SYSC_CLSOCKET:
263 mtx_lock(&rpctls_connect_lock);
264 so = rpctls_connect_so;
265 rpctls_connect_so = NULL;
266 concl = rpctls_connect_cl;
267 rpctls_connect_cl = NULL;
268 mtx_unlock(&rpctls_connect_lock);
270 error = falloc(td, &fp, &fd, 0);
273 * Set ssl refno so that clnt_vc_destroy() will
274 * not close the socket and will leave that for
279 ssl[2] = RPCTLS_REFNO_HANDSHAKE;
280 CLNT_CONTROL(concl, CLSET_TLS, ssl);
281 finit(fp, FREAD | FWRITE, DTYPE_SOCKET, so,
283 fdrop(fp, td); /* Drop fp reference. */
284 td->td_retval[0] = fd;
289 case RPCTLS_SYSC_SRVSOCKET:
290 mtx_lock(&rpctls_server_lock);
291 so = rpctls_server_so;
292 rpctls_server_so = NULL;
293 xprt = rpctls_server_xprt;
294 rpctls_server_xprt = NULL;
295 mtx_unlock(&rpctls_server_lock);
297 error = falloc(td, &fp, &fd, 0);
300 * Once this file descriptor is associated
301 * with the socket, it cannot be closed by
302 * the server side krpc code (svc_vc.c).
305 sx_xlock(&xprt->xp_lock);
306 xprt->xp_tls = RPCTLS_FLAGS_HANDSHFAIL;
307 sx_xunlock(&xprt->xp_lock);
308 finit(fp, FREAD | FWRITE, DTYPE_SOCKET, so,
310 fdrop(fp, td); /* Drop fp reference. */
311 td->td_retval[0] = fd;
324 * Acquire the rpctls_connect_handle and return it with a reference count,
325 * if it is available.
328 rpctls_connect_client(void)
332 mtx_lock(&rpctls_connect_lock);
333 cl = rpctls_connect_handle;
336 mtx_unlock(&rpctls_connect_lock);
341 * Acquire the rpctls_server_handle and return it with a reference count,
342 * if it is available.
345 rpctls_server_client(void)
349 mtx_lock(&rpctls_server_lock);
350 cl = rpctls_server_handle;
353 mtx_unlock(&rpctls_server_lock);
357 /* Do an upcall for a new socket connect using TLS. */
359 rpctls_connect(CLIENT *newclient, char *certname, struct socket *so,
360 uint64_t *sslp, uint32_t *reterr)
362 struct rpctlscd_connect_arg arg;
363 struct rpctlscd_connect_res res;
364 struct rpc_callextra ext;
365 struct timeval utimeout;
369 static bool rpctls_connect_busy = false;
371 cl = rpctls_connect_client();
373 return (RPC_AUTHERROR);
375 /* First, do the AUTH_TLS NULL RPC. */
376 memset(&ext, 0, sizeof(ext));
377 utimeout.tv_sec = 30;
378 utimeout.tv_usec = 0;
379 ext.rc_auth = authtls_create();
380 stat = clnt_call_private(newclient, &ext, NULLPROC, (xdrproc_t)xdr_void,
381 NULL, (xdrproc_t)xdr_void, NULL, utimeout);
382 AUTH_DESTROY(ext.rc_auth);
383 if (stat == RPC_AUTHERROR)
385 if (stat != RPC_SUCCESS)
386 return (RPC_SYSTEMERROR);
388 /* Serialize the connect upcalls. */
389 mtx_lock(&rpctls_connect_lock);
390 while (rpctls_connect_busy)
391 msleep(&rpctls_connect_busy, &rpctls_connect_lock, PVFS,
393 rpctls_connect_busy = true;
394 rpctls_connect_so = so;
395 rpctls_connect_cl = newclient;
396 mtx_unlock(&rpctls_connect_lock);
398 /* Temporarily block reception during the handshake upcall. */
400 CLNT_CONTROL(newclient, CLSET_BLOCKRCV, &val);
402 /* Do the connect handshake upcall. */
403 if (certname != NULL) {
404 arg.certname.certname_len = strlen(certname);
405 arg.certname.certname_val = certname;
407 arg.certname.certname_len = 0;
408 stat = rpctlscd_connect_1(&arg, &res, cl);
409 if (stat == RPC_SUCCESS) {
410 *reterr = res.reterr;
411 if (res.reterr == 0) {
416 } else if (stat == RPC_TIMEDOUT) {
418 * Do a shutdown on the socket, since the daemon is probably
419 * stuck in SSL_connect() trying to read the socket.
420 * Do not soclose() the socket, since the daemon will close()
421 * the socket after SSL_connect() returns an error.
423 soshutdown(so, SHUT_RD);
427 /* Unblock reception. */
429 CLNT_CONTROL(newclient, CLSET_BLOCKRCV, &val);
431 /* Once the upcall is done, the daemon is done with the fp and so. */
432 mtx_lock(&rpctls_connect_lock);
433 rpctls_connect_so = NULL;
434 rpctls_connect_cl = NULL;
435 rpctls_connect_busy = false;
436 wakeup(&rpctls_connect_busy);
437 mtx_unlock(&rpctls_connect_lock);
442 /* Do an upcall to handle an non-application data record using TLS. */
444 rpctls_cl_handlerecord(uint64_t sec, uint64_t usec, uint64_t ssl,
447 struct rpctlscd_handlerecord_arg arg;
448 struct rpctlscd_handlerecord_res res;
452 cl = rpctls_connect_client();
454 *reterr = RPCTLSERR_NOSSL;
455 return (RPC_SUCCESS);
458 /* Do the handlerecord upcall. */
462 stat = rpctlscd_handlerecord_1(&arg, &res, cl);
464 if (stat == RPC_SUCCESS)
465 *reterr = res.reterr;
470 rpctls_srv_handlerecord(uint64_t sec, uint64_t usec, uint64_t ssl,
473 struct rpctlssd_handlerecord_arg arg;
474 struct rpctlssd_handlerecord_res res;
478 cl = rpctls_server_client();
480 *reterr = RPCTLSERR_NOSSL;
481 return (RPC_SUCCESS);
484 /* Do the handlerecord upcall. */
488 stat = rpctlssd_handlerecord_1(&arg, &res, cl);
490 if (stat == RPC_SUCCESS)
491 *reterr = res.reterr;
495 /* Do an upcall to shut down a socket using TLS. */
497 rpctls_cl_disconnect(uint64_t sec, uint64_t usec, uint64_t ssl,
500 struct rpctlscd_disconnect_arg arg;
501 struct rpctlscd_disconnect_res res;
505 cl = rpctls_connect_client();
507 *reterr = RPCTLSERR_NOSSL;
508 return (RPC_SUCCESS);
511 /* Do the disconnect upcall. */
515 stat = rpctlscd_disconnect_1(&arg, &res, cl);
517 if (stat == RPC_SUCCESS)
518 *reterr = res.reterr;
523 rpctls_srv_disconnect(uint64_t sec, uint64_t usec, uint64_t ssl,
526 struct rpctlssd_disconnect_arg arg;
527 struct rpctlssd_disconnect_res res;
531 cl = rpctls_server_client();
533 *reterr = RPCTLSERR_NOSSL;
534 return (RPC_SUCCESS);
537 /* Do the disconnect upcall. */
541 stat = rpctlssd_disconnect_1(&arg, &res, cl);
543 if (stat == RPC_SUCCESS)
544 *reterr = res.reterr;
548 /* Do an upcall for a new server socket using TLS. */
549 static enum clnt_stat
550 rpctls_server(SVCXPRT *xprt, struct socket *so, uint32_t *flags, uint64_t *sslp,
551 uid_t *uid, int *ngrps, gid_t **gids)
555 struct rpctlssd_connect_res res;
559 static bool rpctls_server_busy = false;
561 cl = rpctls_server_client();
563 return (RPC_SYSTEMERROR);
565 /* Serialize the server upcalls. */
566 mtx_lock(&rpctls_server_lock);
567 while (rpctls_server_busy)
568 msleep(&rpctls_server_busy, &rpctls_server_lock, PVFS,
570 rpctls_server_busy = true;
571 rpctls_server_so = so;
572 rpctls_server_xprt = xprt;
573 mtx_unlock(&rpctls_server_lock);
575 /* Do the server upcall. */
576 res.gid.gid_val = NULL;
577 stat = rpctlssd_connect_1(NULL, &res, cl);
578 if (stat == RPC_SUCCESS) {
583 if ((*flags & (RPCTLS_FLAGS_CERTUSER |
584 RPCTLS_FLAGS_DISABLED)) == RPCTLS_FLAGS_CERTUSER) {
585 *ngrps = res.gid.gid_len;
587 *gids = gidp = mem_alloc(*ngrps * sizeof(gid_t));
588 gidv = res.gid.gid_val;
589 for (i = 0; i < *ngrps; i++)
592 } else if (stat == RPC_TIMEDOUT) {
594 * Do a shutdown on the socket, since the daemon is probably
595 * stuck in SSL_accept() trying to read the socket.
596 * Do not soclose() the socket, since the daemon will close()
597 * the socket after SSL_accept() returns an error.
599 soshutdown(so, SHUT_RD);
602 mem_free(res.gid.gid_val, 0);
604 /* Once the upcall is done, the daemon is done with the fp and so. */
605 mtx_lock(&rpctls_server_lock);
606 rpctls_server_so = NULL;
607 rpctls_server_xprt = NULL;
608 rpctls_server_busy = false;
609 wakeup(&rpctls_server_busy);
610 mtx_unlock(&rpctls_server_lock);
616 * Handle the NULL RPC with authentication flavor of AUTH_TLS.
617 * This is a STARTTLS command, so do the upcall to the rpctlssd daemon,
618 * which will do the TLS handshake.
621 _svcauth_rpcsec_tls(struct svc_req *rqst, struct rpc_msg *msg)
636 /* Initialize reply. */
637 rqst->rq_verf = rpctls_null_verf;
639 /* Check client credentials. */
640 if (rqst->rq_cred.oa_length != 0 ||
641 msg->rm_call.cb_verf.oa_length != 0 ||
642 msg->rm_call.cb_verf.oa_flavor != AUTH_NULL)
643 return (AUTH_BADCRED);
645 if (rqst->rq_proc != NULLPROC)
646 return (AUTH_REJECTEDCRED);
650 if (rpctls_getinfo(&maxlen, false, true))
654 return (AUTH_REJECTEDCRED);
657 * Disable reception for the krpc so that the TLS handshake can
658 * be done on the socket in the rpctlssd daemon.
660 xprt = rqst->rq_xprt;
661 sx_xlock(&xprt->xp_lock);
662 xprt->xp_dontrcv = TRUE;
663 sx_xunlock(&xprt->xp_lock);
666 * Send the reply to the NULL RPC with AUTH_TLS, which is the
667 * STARTTLS command for Sun RPC.
669 call_stat = svc_sendreply(rqst, (xdrproc_t)xdr_void, NULL);
671 sx_xlock(&xprt->xp_lock);
672 xprt->xp_dontrcv = FALSE;
673 sx_xunlock(&xprt->xp_lock);
674 xprt_active(xprt); /* Harmless if already active. */
675 return (AUTH_REJECTEDCRED);
678 /* Do an upcall to do the TLS handshake. */
679 stat = rpctls_server(xprt, xprt->xp_socket, &flags,
680 ssl, &uid, &ngrps, &gidp);
682 /* Re-enable reception on the socket within the krpc. */
683 sx_xlock(&xprt->xp_lock);
684 xprt->xp_dontrcv = FALSE;
685 if (stat == RPC_SUCCESS) {
686 xprt->xp_tls = flags;
687 xprt->xp_sslsec = ssl[0];
688 xprt->xp_sslusec = ssl[1];
689 xprt->xp_sslrefno = ssl[2];
690 if ((flags & (RPCTLS_FLAGS_CERTUSER |
691 RPCTLS_FLAGS_DISABLED)) == RPCTLS_FLAGS_CERTUSER) {
692 xprt->xp_ngrps = ngrps;
694 xprt->xp_gidp = gidp;
697 sx_xunlock(&xprt->xp_lock);
698 xprt_active(xprt); /* Harmless if already active. */
700 return (RPCSEC_GSS_NODISPATCH);
704 * Get kern.ipc.tls.enable and kern.ipc.tls.maxlen.
707 rpctls_getinfo(u_int *maxlenp, bool rpctlscd_run, bool rpctlssd_run)
716 siz = sizeof(enable);
717 error = kernel_sysctlbyname(curthread, "kern.ipc.tls.enable",
718 &enable, &siz, NULL, 0, NULL, 0);
721 siz = sizeof(maxlen);
722 error = kernel_sysctlbyname(curthread, "kern.ipc.tls.maxlen",
723 &maxlen, &siz, NULL, 0, NULL, 0);
726 if (rpctlscd_run && rpctls_connect_handle == NULL)
728 if (rpctlssd_run && rpctls_server_handle == NULL)