2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2008 Isilon Inc http://www.isilon.com/
5 * Authors: Doug Rabson <dfr@rabson.org>
6 * Developed with Red Inc: Alfred Perlstein <alfred@freebsd.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * Extensively modified from /usr/src/usr.sbin/gssd.c r344402 for
32 * the client side of kernel RPC-over-TLS by Rick Macklem.
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
38 #include <sys/param.h>
39 #include <sys/types.h>
40 #include <sys/queue.h>
41 #include <sys/linker.h>
42 #include <sys/module.h>
44 #include <sys/sysctl.h>
45 #include <sys/syslog.h>
60 #include <rpc/rpc_com.h>
61 #include <rpc/rpcsec_tls.h>
63 #include <openssl/opensslconf.h>
64 #include <openssl/bio.h>
65 #include <openssl/ssl.h>
66 #include <openssl/err.h>
67 #include <openssl/x509v3.h>
70 #include "rpc.tlscommon.h"
72 #ifndef _PATH_RPCTLSCDSOCK
73 #define _PATH_RPCTLSCDSOCK "/var/run/rpc.tlsclntd.sock"
75 #ifndef _PATH_CERTANDKEY
76 #define _PATH_CERTANDKEY "/etc/rpc.tlsclntd/"
78 #ifndef _PATH_RPCTLSCDPID
79 #define _PATH_RPCTLSCDPID "/var/run/rpc.tlsclntd.pid"
82 /* Global variables also used by rpc.tlscommon.c. */
83 int rpctls_debug_level;
85 SSL_CTX *rpctls_ctx = NULL;
86 const char *rpctls_verify_cafile = NULL;
87 const char *rpctls_verify_capath = NULL;
88 char *rpctls_crlfile = NULL;
89 bool rpctls_cert = false;
90 bool rpctls_gothup = false;
91 struct ssl_list rpctls_ssllist;
93 static struct pidfh *rpctls_pfh = NULL;
94 static const char *rpctls_certdir = _PATH_CERTANDKEY;
95 static const char *rpctls_ciphers = NULL;
96 static uint64_t rpctls_ssl_refno = 0;
97 static uint64_t rpctls_ssl_sec = 0;
98 static uint64_t rpctls_ssl_usec = 0;
99 static int rpctls_tlsvers = TLS1_3_VERSION;
101 static void rpctlscd_terminate(int);
102 static SSL_CTX *rpctls_setupcl_ssl(void);
103 static SSL *rpctls_connect(SSL_CTX *ctx, int s, char *certname,
104 u_int certlen, X509 **certp);
105 static void rpctls_huphandler(int sig __unused);
107 extern void rpctlscd_1(struct svc_req *rqstp, SVCXPRT *transp);
109 static struct option longopts[] = {
110 { "usetls1_2", no_argument, NULL, '2' },
111 { "certdir", required_argument, NULL, 'D' },
112 { "ciphers", required_argument, NULL, 'C' },
113 { "debuglevel", no_argument, NULL, 'd' },
114 { "verifylocs", required_argument, NULL, 'l' },
115 { "mutualverf", no_argument, NULL, 'm' },
116 { "verifydir", required_argument, NULL, 'p' },
117 { "crl", required_argument, NULL, 'r' },
118 { "verbose", no_argument, NULL, 'v' },
123 main(int argc, char **argv)
126 * We provide an RPC service on a local-domain socket. The
127 * kernel rpctls code will upcall to this daemon to do the initial
130 struct sockaddr_un sun;
137 size_t tls_enable_len;
139 /* Check that another rpctlscd isn't already running. */
140 rpctls_pfh = pidfile_open(_PATH_RPCTLSCDPID, 0600, &otherpid);
141 if (rpctls_pfh == NULL) {
143 errx(1, "rpctlscd already running, pid: %d.", otherpid);
144 warn("cannot open or create pidfile");
147 /* Check to see that the ktls is enabled. */
148 tls_enable_len = sizeof(tls_enable);
149 if (sysctlbyname("kern.ipc.tls.enable", &tls_enable, &tls_enable_len,
150 NULL, 0) != 0 || !tls_enable)
151 errx(1, "Kernel TLS not enabled");
153 /* Get the time when this daemon is started. */
154 gettimeofday(&tm, &tz);
155 rpctls_ssl_sec = tm.tv_sec;
156 rpctls_ssl_usec = tm.tv_usec;
158 rpctls_verbose = false;
159 while ((ch = getopt_long(argc, argv, "2C:D:dl:mp:r:v", longopts,
163 rpctls_tlsvers = TLS1_2_VERSION;
166 rpctls_ciphers = optarg;
169 rpctls_certdir = optarg;
172 rpctls_debug_level++;
175 rpctls_verify_cafile = optarg;
181 rpctls_verify_capath = optarg;
184 rpctls_crlfile = optarg;
187 rpctls_verbose = true;
190 fprintf(stderr, "usage: %s "
192 "[-C/--ciphers available_ciphers] "
193 "[-D/--certdir certdir] [-d/--debuglevel] "
194 "[-l/--verifylocs CAfile] [-m/--mutualverf] "
195 "[-p/--verifydir CApath] [-r/--crl CRLfile] "
196 "[-v/--verbose]\n", argv[0]);
201 if (rpctls_crlfile != NULL && rpctls_verify_cafile == NULL &&
202 rpctls_verify_capath == NULL)
203 errx(1, "-r requires the -l <CAfile> and/or "
204 "-p <CApath> options");
206 if (modfind("krpc") < 0) {
207 /* Not present in kernel, try loading it */
208 if (kldload("krpc") < 0 || modfind("krpc") < 0)
209 errx(1, "Kernel RPC is not available");
213 * Set up the SSL_CTX *.
214 * Do it now, before daemonizing, in case the private key
215 * is encrypted and requires a passphrase to be entered.
217 rpctls_ctx = rpctls_setupcl_ssl();
218 if (rpctls_ctx == NULL) {
219 if (rpctls_debug_level == 0) {
220 syslog(LOG_ERR, "Can't set up TLS context");
223 err(1, "Can't set up TLS context");
225 LIST_INIT(&rpctls_ssllist);
227 if (!rpctls_debug_level) {
228 if (daemon(0, 0) != 0)
229 err(1, "Can't daemonize");
230 signal(SIGINT, SIG_IGN);
231 signal(SIGQUIT, SIG_IGN);
232 signal(SIGHUP, SIG_IGN);
234 signal(SIGTERM, rpctlscd_terminate);
235 signal(SIGPIPE, SIG_IGN);
236 signal(SIGHUP, rpctls_huphandler);
238 pidfile_write(rpctls_pfh);
240 memset(&sun, 0, sizeof sun);
241 sun.sun_family = AF_LOCAL;
242 unlink(_PATH_RPCTLSCDSOCK);
243 strcpy(sun.sun_path, _PATH_RPCTLSCDSOCK);
244 sun.sun_len = SUN_LEN(&sun);
245 fd = socket(AF_LOCAL, SOCK_STREAM, 0);
247 if (rpctls_debug_level == 0) {
248 syslog(LOG_ERR, "Can't create local rpctlscd socket");
251 err(1, "Can't create local rpctlscd socket");
253 oldmask = umask(S_IXUSR|S_IRWXG|S_IRWXO);
254 if (bind(fd, (struct sockaddr *)&sun, sun.sun_len) < 0) {
255 if (rpctls_debug_level == 0) {
256 syslog(LOG_ERR, "Can't bind local rpctlscd socket");
259 err(1, "Can't bind local rpctlscd socket");
262 if (listen(fd, SOMAXCONN) < 0) {
263 if (rpctls_debug_level == 0) {
265 "Can't listen on local rpctlscd socket");
268 err(1, "Can't listen on local rpctlscd socket");
270 xprt = svc_vc_create(fd, RPC_MAXDATASIZE, RPC_MAXDATASIZE);
272 if (rpctls_debug_level == 0) {
274 "Can't create transport for local rpctlscd socket");
277 err(1, "Can't create transport for local rpctlscd socket");
279 if (!svc_reg(xprt, RPCTLSCD, RPCTLSCDVERS, rpctlscd_1, NULL)) {
280 if (rpctls_debug_level == 0) {
282 "Can't register service for local rpctlscd socket");
285 err(1, "Can't register service for local rpctlscd socket");
288 if (rpctls_syscall(RPCTLS_SYSC_CLSETPATH, _PATH_RPCTLSCDSOCK) < 0) {
289 if (rpctls_debug_level == 0) {
291 "Can't set upcall socket path errno=%d", errno);
294 err(1, "Can't set upcall socket path");
299 rpctls_syscall(RPCTLS_SYSC_CLSHUTDOWN, "");
301 SSL_CTX_free(rpctls_ctx);
307 rpctlscd_null_1_svc(__unused void *argp, __unused void *result,
308 __unused struct svc_req *rqstp)
311 rpctls_verbose_out("rpctlscd_null: done\n");
316 rpctlscd_connect_1_svc(struct rpctlscd_connect_arg *argp,
317 struct rpctlscd_connect_res *result, __unused struct svc_req *rqstp)
321 struct ssl_entry *newslp;
324 rpctls_verbose_out("rpctlsd_connect: started\n");
325 /* Get the socket fd from the kernel. */
326 s = rpctls_syscall(RPCTLS_SYSC_CLSOCKET, "");
328 result->reterr = RPCTLSERR_NOSOCKET;
332 /* Do a TLS connect handshake. */
333 ssl = rpctls_connect(rpctls_ctx, s, argp->certname.certname_val,
334 argp->certname.certname_len, &cert);
336 rpctls_verbose_out("rpctlsd_connect: can't do TLS "
338 result->reterr = RPCTLSERR_NOSSL;
340 result->reterr = RPCTLSERR_OK;
341 result->sec = rpctls_ssl_sec;
342 result->usec = rpctls_ssl_usec;
343 result->ssl = ++rpctls_ssl_refno;
344 /* Hard to believe this will ever wrap around.. */
345 if (rpctls_ssl_refno == 0)
346 result->ssl = ++rpctls_ssl_refno;
351 * For RPC-over-TLS, this upcall is expected
352 * to close off the socket.
358 /* Maintain list of all current SSL *'s */
359 newslp = malloc(sizeof(*newslp));
360 newslp->refno = rpctls_ssl_refno;
362 newslp->shutoff = false;
365 LIST_INSERT_HEAD(&rpctls_ssllist, newslp, next);
370 rpctlscd_handlerecord_1_svc(struct rpctlscd_handlerecord_arg *argp,
371 struct rpctlscd_handlerecord_res *result, __unused struct svc_req *rqstp)
373 struct ssl_entry *slp;
378 if (argp->sec == rpctls_ssl_sec && argp->usec ==
380 LIST_FOREACH(slp, &rpctls_ssllist, next) {
381 if (slp->refno == argp->ssl)
387 rpctls_verbose_out("rpctlscd_handlerecord fd=%d\n",
390 * An SSL_read() of 0 bytes should fail, but it should
391 * handle the non-application data record before doing so.
393 ret = SSL_read(slp->ssl, &junk, 0);
395 /* Check to see if this was a close alert. */
396 ret = SSL_get_shutdown(slp->ssl);
397 if ((ret & (SSL_SENT_SHUTDOWN |
398 SSL_RECEIVED_SHUTDOWN)) == SSL_RECEIVED_SHUTDOWN)
399 SSL_shutdown(slp->ssl);
401 if (rpctls_debug_level == 0)
402 syslog(LOG_ERR, "SSL_read returned %d", ret);
404 fprintf(stderr, "SSL_read returned %d\n", ret);
406 result->reterr = RPCTLSERR_OK;
408 result->reterr = RPCTLSERR_NOSSL;
413 rpctlscd_disconnect_1_svc(struct rpctlscd_disconnect_arg *argp,
414 struct rpctlscd_disconnect_res *result, __unused struct svc_req *rqstp)
416 struct ssl_entry *slp;
420 if (argp->sec == rpctls_ssl_sec && argp->usec ==
422 LIST_FOREACH(slp, &rpctls_ssllist, next) {
423 if (slp->refno == argp->ssl)
429 rpctls_verbose_out("rpctlscd_disconnect: fd=%d closed\n",
431 LIST_REMOVE(slp, next);
433 ret = SSL_get_shutdown(slp->ssl);
435 * Do an SSL_shutdown() unless a close alert has
438 if ((ret & SSL_SENT_SHUTDOWN) == 0)
439 SSL_shutdown(slp->ssl);
442 if (slp->cert != NULL)
443 X509_free(slp->cert);
445 * For RPC-over-TLS, this upcall is expected
446 * to close off the socket.
449 shutdown(slp->s, SHUT_WR);
452 result->reterr = RPCTLSERR_OK;
454 result->reterr = RPCTLSERR_NOCLOSE;
459 rpctlscd_1_freeresult(__unused SVCXPRT *transp, __unused xdrproc_t xdr_result,
460 __unused caddr_t result)
467 rpctlscd_terminate(int sig __unused)
470 rpctls_syscall(RPCTLS_SYSC_CLSHUTDOWN, "");
471 pidfile_remove(rpctls_pfh);
476 rpctls_setupcl_ssl(void)
484 SSL_load_error_strings();
485 OpenSSL_add_all_algorithms();
487 ctx = SSL_CTX_new(TLS_client_method());
489 rpctls_verbose_out("rpctls_setupcl_ssl: SSL_CTX_new "
493 SSL_CTX_set_ecdh_auto(ctx, 1);
495 if (rpctls_ciphers != NULL) {
497 * Set available ciphers, since KERN_TLS only supports a
500 ret = SSL_CTX_set_ciphersuites(ctx, rpctls_ciphers);
502 rpctls_verbose_out("rpctls_setupcl_ssl: "
503 "SSL_CTX_set_ciphersuites failed: %s\n",
511 * If rpctls_cert is true, a certificate and key exists in
512 * rpctls_certdir, so that it can do mutual authentication.
515 /* Get the cert.pem and certkey.pem files. */
516 len = strlcpy(path, rpctls_certdir, sizeof(path));
517 rlen = sizeof(path) - len;
518 if (strlcpy(&path[len], "cert.pem", rlen) != 8) {
522 ret = SSL_CTX_use_certificate_file(ctx, path,
525 rpctls_verbose_out("rpctls_setupcl_ssl: can't use "
526 "certificate file path=%s ret=%d\n", path, ret);
530 if (strlcpy(&path[len], "certkey.pem", rlen) != 11) {
534 ret = SSL_CTX_use_PrivateKey_file(ctx, path,
537 rpctls_verbose_out("rpctls_setupcl_ssl: Can't use "
538 "private key path=%s ret=%d\n", path, ret);
544 if (rpctls_verify_cafile != NULL || rpctls_verify_capath != NULL) {
545 if (rpctls_crlfile != NULL) {
546 ret = rpctls_loadcrlfile(ctx);
548 rpctls_verbose_out("rpctls_setupcl_ssl: "
549 "Load CRLfile failed\n");
554 #if OPENSSL_VERSION_NUMBER >= 0x30000000
556 if (rpctls_verify_cafile != NULL)
557 ret = SSL_CTX_load_verify_file(ctx,
558 rpctls_verify_cafile);
559 if (ret != 0 && rpctls_verify_capath != NULL)
560 ret = SSL_CTX_load_verify_dir(ctx,
561 rpctls_verify_capath);
563 ret = SSL_CTX_load_verify_locations(ctx,
564 rpctls_verify_cafile, rpctls_verify_capath);
567 rpctls_verbose_out("rpctls_setupcl_ssl: "
568 "Can't load verify locations\n");
573 * The man page says that the
574 * SSL_CTX_set0_CA_list() call is not normally
575 * needed, but I believe it is harmless.
577 if (rpctls_verify_cafile != NULL)
578 SSL_CTX_set0_CA_list(ctx,
579 SSL_load_client_CA_file(rpctls_verify_cafile));
583 * The RFC specifies that RPC-over-TLS must use TLS1.3.
584 * However, early FreeBSD versions (13.0, 13.1) did not
585 * support RX for KTLS1.3, so TLS1.2 needs to be used for
588 ret = SSL_CTX_set_min_proto_version(ctx, rpctls_tlsvers);
590 rpctls_verbose_out("rpctls_setupcl_ssl: "
591 "SSL_CTX_set_min_proto_version failed\n");
595 ret = SSL_CTX_set_max_proto_version(ctx, rpctls_tlsvers);
597 rpctls_verbose_out("rpctls_setupcl_ssl: "
598 "SSL_CTX_set_max_proto_version failed\n");
603 #ifdef SSL_OP_ENABLE_KTLS
604 SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS);
606 #ifdef SSL_MODE_NO_KTLS_TX
607 SSL_CTX_clear_mode(ctx, SSL_MODE_NO_KTLS_TX | SSL_MODE_NO_KTLS_RX);
613 rpctls_connect(SSL_CTX *ctx, int s, char *certname, u_int certlen, X509 **certp)
617 struct sockaddr_storage ad;
618 struct sockaddr *sad;
619 char hostnam[NI_MAXHOST], path[PATH_MAX];
626 sad = (struct sockaddr *)&ad;
629 rpctls_verbose_out("rpctls_connect: "
633 if (SSL_set_fd(ssl, s) != 1) {
634 rpctls_verbose_out("rpctls_connect: "
635 "SSL_set_fd failed\n");
641 * If rpctls_cert is true and certname is set, a alternate certificate
642 * and key exists in files named <certname>.pem and <certname>key.pem
643 * in rpctls_certdir that is to be used for mutual authentication.
645 if (rpctls_cert && certlen > 0) {
646 len = strlcpy(path, rpctls_certdir, sizeof(path));
647 rlen = sizeof(path) - len;
648 if (rlen <= certlen) {
652 memcpy(&path[len], certname, certlen);
656 if (strlcpy(&path[len], ".pem", rlen) != 4) {
660 ret = SSL_use_certificate_file(ssl, path, SSL_FILETYPE_PEM);
662 rpctls_verbose_out("rpctls_connect: can't use "
663 "certificate file path=%s ret=%d\n", path, ret);
667 if (strlcpy(&path[len], "key.pem", rlen) != 7) {
671 ret = SSL_use_PrivateKey_file(ssl, path, SSL_FILETYPE_PEM);
673 rpctls_verbose_out("rpctls_connect: Can't use "
674 "private key path=%s ret=%d\n", path, ret);
680 ret = SSL_connect(ssl);
682 rpctls_verbose_out("rpctls_connect: "
683 "SSL_connect failed %d: %s\n",
684 ret, ERR_error_string(ERR_get_error(), NULL));
689 cert = SSL_get_peer_certificate(ssl);
691 rpctls_verbose_out("rpctls_connect: get peer"
692 " certificate failed\n");
696 gethostret = rpctls_gethost(s, sad, hostnam, sizeof(hostnam));
699 verfret = SSL_get_verify_result(ssl);
700 if (verfret == X509_V_OK && (rpctls_verify_cafile != NULL ||
701 rpctls_verify_capath != NULL) && (gethostret == 0 ||
702 rpctls_checkhost(sad, cert, X509_CHECK_FLAG_NO_WILDCARDS) != 1))
703 verfret = X509_V_ERR_HOSTNAME_MISMATCH;
704 if (verfret != X509_V_OK && (rpctls_verify_cafile != NULL ||
705 rpctls_verify_capath != NULL)) {
706 if (verfret != X509_V_OK) {
707 cp = X509_NAME_oneline(X509_get_issuer_name(cert),
709 cp2 = X509_NAME_oneline(X509_get_subject_name(cert),
711 if (rpctls_debug_level == 0)
712 syslog(LOG_INFO | LOG_DAEMON,
713 "rpctls_connect: client IP %s "
714 "issuerName=%s subjectName=%s verify "
715 "failed %s\n", hostnam, cp, cp2,
716 X509_verify_cert_error_string(verfret));
719 "rpctls_connect: client IP %s "
720 "issuerName=%s subjectName=%s verify "
721 "failed %s\n", hostnam, cp, cp2,
722 X509_verify_cert_error_string(verfret));
729 /* Check to see if ktls is enabled on the connection. */
730 ret = BIO_get_ktls_send(SSL_get_wbio(ssl));
731 rpctls_verbose_out("rpctls_connect: BIO_get_ktls_send=%d\n", ret);
733 ret = BIO_get_ktls_recv(SSL_get_rbio(ssl));
734 rpctls_verbose_out("rpctls_connect: BIO_get_ktls_recv=%d\n",
738 if (rpctls_debug_level == 0)
739 syslog(LOG_ERR, "ktls not working\n");
741 fprintf(stderr, "ktls not working\n");
746 if (ret == X509_V_OK && (rpctls_verify_cafile != NULL ||
747 rpctls_verify_capath != NULL) && rpctls_crlfile != NULL)
756 rpctls_huphandler(int sig __unused)
759 rpctls_gothup = true;