2 * Copyright (c) 1982, 1986, 1989, 1990, 1993
3 * The Regents of the University of California. All rights reserved.
5 * sendfile(2) and related extensions:
6 * Copyright (c) 1998, David Greenman. All rights reserved.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 4. Neither the name of the University nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * @(#)uipc_syscalls.c 8.4 (Berkeley) 2/21/94
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
38 #include "opt_capsicum.h"
40 #include "opt_inet6.h"
42 #include "opt_compat.h"
43 #include "opt_ktrace.h"
45 #include <sys/param.h>
46 #include <sys/systm.h>
47 #include <sys/capsicum.h>
48 #include <sys/condvar.h>
49 #include <sys/kernel.h>
51 #include <sys/mutex.h>
52 #include <sys/sysproto.h>
53 #include <sys/malloc.h>
54 #include <sys/filedesc.h>
55 #include <sys/event.h>
57 #include <sys/fcntl.h>
59 #include <sys/filio.h>
62 #include <sys/mount.h>
64 #include <sys/protosw.h>
65 #include <sys/rwlock.h>
66 #include <sys/sf_buf.h>
67 #include <sys/sf_sync.h>
68 #include <sys/sf_base.h>
69 #include <sys/sysent.h>
70 #include <sys/socket.h>
71 #include <sys/socketvar.h>
72 #include <sys/signalvar.h>
73 #include <sys/syscallsubr.h>
74 #include <sys/sysctl.h>
76 #include <sys/vnode.h>
78 #include <sys/ktrace.h>
80 #ifdef COMPAT_FREEBSD32
81 #include <compat/freebsd32/freebsd32_util.h>
86 #include <security/audit/audit.h>
87 #include <security/mac/mac_framework.h>
90 #include <vm/vm_param.h>
91 #include <vm/vm_object.h>
92 #include <vm/vm_page.h>
93 #include <vm/vm_pager.h>
94 #include <vm/vm_kern.h>
95 #include <vm/vm_extern.h>
98 #if defined(INET) || defined(INET6)
100 #include <netinet/sctp.h>
101 #include <netinet/sctp_peeloff.h>
103 #endif /* INET || INET6 */
106 * Flags for accept1() and kern_accept4(), in addition to SOCK_CLOEXEC
109 #define ACCEPT4_INHERIT 0x1
110 #define ACCEPT4_COMPAT 0x2
112 static int sendit(struct thread *td, int s, struct msghdr *mp, int flags);
113 static int recvit(struct thread *td, int s, struct msghdr *mp, void *namelenp);
115 static int accept1(struct thread *td, int s, struct sockaddr *uname,
116 socklen_t *anamelen, int flags);
117 static int do_sendfile(struct thread *td, struct sendfile_args *uap,
119 static int getsockname1(struct thread *td, struct getsockname_args *uap,
121 static int getpeername1(struct thread *td, struct getpeername_args *uap,
124 counter_u64_t sfstat[sizeof(struct sfstat) / sizeof(uint64_t)];
126 static int filt_sfsync_attach(struct knote *kn);
127 static void filt_sfsync_detach(struct knote *kn);
128 static int filt_sfsync(struct knote *kn, long hint);
131 * sendfile(2)-related variables and associated sysctls
133 static SYSCTL_NODE(_kern_ipc, OID_AUTO, sendfile, CTLFLAG_RW, 0,
134 "sendfile(2) tunables");
135 static int sfreadahead = 1;
136 SYSCTL_INT(_kern_ipc_sendfile, OID_AUTO, readahead, CTLFLAG_RW,
137 &sfreadahead, 0, "Number of sendfile(2) read-ahead MAXBSIZE blocks");
140 static int sf_sync_debug = 0;
141 SYSCTL_INT(_debug, OID_AUTO, sf_sync_debug, CTLFLAG_RW,
142 &sf_sync_debug, 0, "Output debugging during sf_sync lifecycle");
143 #define SFSYNC_DPRINTF(s, ...) \
146 printf((s), ##__VA_ARGS__); \
149 #define SFSYNC_DPRINTF(c, ...)
152 static uma_zone_t zone_sfsync;
154 static struct filterops sendfile_filtops = {
156 .f_attach = filt_sfsync_attach,
157 .f_detach = filt_sfsync_detach,
158 .f_event = filt_sfsync,
162 sfstat_init(const void *unused)
165 COUNTER_ARRAY_ALLOC(sfstat, sizeof(struct sfstat) / sizeof(uint64_t),
168 SYSINIT(sfstat, SI_SUB_MBUF, SI_ORDER_FIRST, sfstat_init, NULL);
171 sf_sync_init(const void *unused)
174 zone_sfsync = uma_zcreate("sendfile_sync", sizeof(struct sendfile_sync),
179 kqueue_add_filteropts(EVFILT_SENDFILE, &sendfile_filtops);
181 SYSINIT(sf_sync, SI_SUB_MBUF, SI_ORDER_FIRST, sf_sync_init, NULL);
184 sfstat_sysctl(SYSCTL_HANDLER_ARGS)
188 COUNTER_ARRAY_COPY(sfstat, &s, sizeof(s) / sizeof(uint64_t));
190 COUNTER_ARRAY_ZERO(sfstat, sizeof(s) / sizeof(uint64_t));
191 return (SYSCTL_OUT(req, &s, sizeof(s)));
193 SYSCTL_PROC(_kern_ipc, OID_AUTO, sfstat, CTLTYPE_OPAQUE | CTLFLAG_RW,
194 NULL, 0, sfstat_sysctl, "I", "sendfile statistics");
197 * Convert a user file descriptor to a kernel file entry and check if required
198 * capability rights are present.
199 * A reference on the file entry is held upon returning.
202 getsock_cap(struct filedesc *fdp, int fd, cap_rights_t *rightsp,
203 struct file **fpp, u_int *fflagp)
208 error = fget_unlocked(fdp, fd, rightsp, 0, &fp, NULL);
211 if (fp->f_type != DTYPE_SOCKET) {
212 fdrop(fp, curthread);
216 *fflagp = fp->f_flag;
222 * System call interface to the socket abstraction.
224 #if defined(COMPAT_43)
225 #define COMPAT_OLDSOCK
231 struct socket_args /* {
239 int fd, error, type, oflag, fflag;
241 AUDIT_ARG_SOCKET(uap->domain, uap->type, uap->protocol);
246 if ((type & SOCK_CLOEXEC) != 0) {
247 type &= ~SOCK_CLOEXEC;
250 if ((type & SOCK_NONBLOCK) != 0) {
251 type &= ~SOCK_NONBLOCK;
256 error = mac_socket_check_create(td->td_ucred, uap->domain, type,
261 error = falloc(td, &fp, &fd, oflag);
264 /* An extra reference on `fp' has been held for us by falloc(). */
265 error = socreate(uap->domain, &so, type, uap->protocol,
268 fdclose(td->td_proc->p_fd, fp, fd, td);
270 finit(fp, FREAD | FWRITE | fflag, DTYPE_SOCKET, so, &socketops);
271 if ((fflag & FNONBLOCK) != 0)
272 (void) fo_ioctl(fp, FIONBIO, &fflag, td->td_ucred, td);
273 td->td_retval[0] = fd;
283 struct bind_args /* {
292 error = getsockaddr(&sa, uap->name, uap->namelen);
294 error = kern_bind(td, uap->s, sa);
301 kern_bindat(struct thread *td, int dirfd, int fd, struct sockaddr *sa)
309 AUDIT_ARG_SOCKADDR(td, dirfd, sa);
310 error = getsock_cap(td->td_proc->p_fd, fd,
311 cap_rights_init(&rights, CAP_BIND), &fp, NULL);
316 if (KTRPOINT(td, KTR_STRUCT))
320 error = mac_socket_check_bind(td->td_ucred, so, sa);
323 if (dirfd == AT_FDCWD)
324 error = sobind(so, sa, td);
326 error = sobindat(dirfd, so, sa, td);
335 kern_bind(struct thread *td, int fd, struct sockaddr *sa)
338 return (kern_bindat(td, AT_FDCWD, fd, sa));
345 struct bindat_args /* {
355 error = getsockaddr(&sa, uap->name, uap->namelen);
357 error = kern_bindat(td, uap->fd, uap->s, sa);
367 struct listen_args /* {
377 AUDIT_ARG_FD(uap->s);
378 error = getsock_cap(td->td_proc->p_fd, uap->s,
379 cap_rights_init(&rights, CAP_LISTEN), &fp, NULL);
383 error = mac_socket_check_listen(td->td_ucred, so);
386 error = solisten(so, uap->backlog, td);
396 accept1(td, s, uname, anamelen, flags)
399 struct sockaddr *uname;
403 struct sockaddr *name;
409 return (kern_accept4(td, s, NULL, NULL, flags, NULL));
411 error = copyin(anamelen, &namelen, sizeof (namelen));
415 error = kern_accept4(td, s, &name, &namelen, flags, &fp);
420 if (error == 0 && uname != NULL) {
421 #ifdef COMPAT_OLDSOCK
422 if (flags & ACCEPT4_COMPAT)
423 ((struct osockaddr *)name)->sa_family =
426 error = copyout(name, uname, namelen);
429 error = copyout(&namelen, anamelen,
432 fdclose(td->td_proc->p_fd, fp, td->td_retval[0], td);
434 free(name, M_SONAME);
439 kern_accept(struct thread *td, int s, struct sockaddr **name,
440 socklen_t *namelen, struct file **fp)
442 return (kern_accept4(td, s, name, namelen, ACCEPT4_INHERIT, fp));
446 kern_accept4(struct thread *td, int s, struct sockaddr **name,
447 socklen_t *namelen, int flags, struct file **fp)
449 struct filedesc *fdp;
450 struct file *headfp, *nfp = NULL;
451 struct sockaddr *sa = NULL;
452 struct socket *head, *so;
462 fdp = td->td_proc->p_fd;
463 error = getsock_cap(fdp, s, cap_rights_init(&rights, CAP_ACCEPT),
467 head = headfp->f_data;
468 if ((head->so_options & SO_ACCEPTCONN) == 0) {
473 error = mac_socket_check_accept(td->td_ucred, head);
477 error = falloc(td, &nfp, &fd, (flags & SOCK_CLOEXEC) ? O_CLOEXEC : 0);
481 if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->so_comp)) {
486 while (TAILQ_EMPTY(&head->so_comp) && head->so_error == 0) {
487 if (head->so_rcv.sb_state & SBS_CANTRCVMORE) {
488 head->so_error = ECONNABORTED;
491 error = msleep(&head->so_timeo, &accept_mtx, PSOCK | PCATCH,
498 if (head->so_error) {
499 error = head->so_error;
504 so = TAILQ_FIRST(&head->so_comp);
505 KASSERT(!(so->so_qstate & SQ_INCOMP), ("accept1: so SQ_INCOMP"));
506 KASSERT(so->so_qstate & SQ_COMP, ("accept1: so not SQ_COMP"));
509 * Before changing the flags on the socket, we have to bump the
510 * reference count. Otherwise, if the protocol calls sofree(),
511 * the socket will be released due to a zero refcount.
513 SOCK_LOCK(so); /* soref() and so_state update */
514 soref(so); /* file descriptor reference */
516 TAILQ_REMOVE(&head->so_comp, so, so_list);
518 if (flags & ACCEPT4_INHERIT)
519 so->so_state |= (head->so_state & SS_NBIO);
521 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0;
522 so->so_qstate &= ~SQ_COMP;
528 /* An extra reference on `nfp' has been held for us by falloc(). */
529 td->td_retval[0] = fd;
531 /* connection has been removed from the listen queue */
532 KNOTE_UNLOCKED(&head->so_rcv.sb_sel.si_note, 0);
534 if (flags & ACCEPT4_INHERIT) {
535 pgid = fgetown(&head->so_sigio);
537 fsetown(pgid, &so->so_sigio);
539 fflag &= ~(FNONBLOCK | FASYNC);
540 if (flags & SOCK_NONBLOCK)
544 finit(nfp, fflag, DTYPE_SOCKET, so, &socketops);
545 /* Sync socket nonblocking/async state with file flags */
546 tmp = fflag & FNONBLOCK;
547 (void) fo_ioctl(nfp, FIONBIO, &tmp, td->td_ucred, td);
548 tmp = fflag & FASYNC;
549 (void) fo_ioctl(nfp, FIOASYNC, &tmp, td->td_ucred, td);
551 error = soaccept(so, &sa);
559 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, sa);
561 /* check sa_len before it is destroyed */
562 if (*namelen > sa->sa_len)
563 *namelen = sa->sa_len;
565 if (KTRPOINT(td, KTR_STRUCT))
575 * close the new descriptor, assuming someone hasn't ripped it
579 fdclose(fdp, nfp, fd, td);
582 * Release explicitly held references before returning. We return
583 * a reference on nfp to the caller on success if they request it.
602 struct accept_args *uap;
605 return (accept1(td, uap->s, uap->name, uap->anamelen, ACCEPT4_INHERIT));
611 struct accept4_args *uap;
614 if (uap->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
617 return (accept1(td, uap->s, uap->name, uap->anamelen, uap->flags));
620 #ifdef COMPAT_OLDSOCK
624 struct accept_args *uap;
627 return (accept1(td, uap->s, uap->name, uap->anamelen,
628 ACCEPT4_INHERIT | ACCEPT4_COMPAT));
630 #endif /* COMPAT_OLDSOCK */
636 struct connect_args /* {
645 error = getsockaddr(&sa, uap->name, uap->namelen);
647 error = kern_connect(td, uap->s, sa);
654 kern_connectat(struct thread *td, int dirfd, int fd, struct sockaddr *sa)
659 int error, interrupted = 0;
662 AUDIT_ARG_SOCKADDR(td, dirfd, sa);
663 error = getsock_cap(td->td_proc->p_fd, fd,
664 cap_rights_init(&rights, CAP_CONNECT), &fp, NULL);
668 if (so->so_state & SS_ISCONNECTING) {
673 if (KTRPOINT(td, KTR_STRUCT))
677 error = mac_socket_check_connect(td->td_ucred, so, sa);
681 if (dirfd == AT_FDCWD)
682 error = soconnect(so, sa, td);
684 error = soconnectat(dirfd, so, sa, td);
687 if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) {
692 while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
693 error = msleep(&so->so_timeo, SOCK_MTX(so), PSOCK | PCATCH,
696 if (error == EINTR || error == ERESTART)
702 error = so->so_error;
708 so->so_state &= ~SS_ISCONNECTING;
709 if (error == ERESTART)
717 kern_connect(struct thread *td, int fd, struct sockaddr *sa)
720 return (kern_connectat(td, AT_FDCWD, fd, sa));
725 sys_connectat(td, uap)
727 struct connectat_args /* {
737 error = getsockaddr(&sa, uap->name, uap->namelen);
739 error = kern_connectat(td, uap->fd, uap->s, sa);
746 kern_socketpair(struct thread *td, int domain, int type, int protocol,
749 struct filedesc *fdp = td->td_proc->p_fd;
750 struct file *fp1, *fp2;
751 struct socket *so1, *so2;
752 int fd, error, oflag, fflag;
754 AUDIT_ARG_SOCKET(domain, type, protocol);
758 if ((type & SOCK_CLOEXEC) != 0) {
759 type &= ~SOCK_CLOEXEC;
762 if ((type & SOCK_NONBLOCK) != 0) {
763 type &= ~SOCK_NONBLOCK;
767 /* We might want to have a separate check for socket pairs. */
768 error = mac_socket_check_create(td->td_ucred, domain, type,
773 error = socreate(domain, &so1, type, protocol, td->td_ucred, td);
776 error = socreate(domain, &so2, type, protocol, td->td_ucred, td);
779 /* On success extra reference to `fp1' and 'fp2' is set by falloc. */
780 error = falloc(td, &fp1, &fd, oflag);
784 fp1->f_data = so1; /* so1 already has ref count */
785 error = falloc(td, &fp2, &fd, oflag);
788 fp2->f_data = so2; /* so2 already has ref count */
790 error = soconnect2(so1, so2);
793 if (type == SOCK_DGRAM) {
795 * Datagram socket connection is asymmetric.
797 error = soconnect2(so2, so1);
801 finit(fp1, FREAD | FWRITE | fflag, DTYPE_SOCKET, fp1->f_data,
803 finit(fp2, FREAD | FWRITE | fflag, DTYPE_SOCKET, fp2->f_data,
805 if ((fflag & FNONBLOCK) != 0) {
806 (void) fo_ioctl(fp1, FIONBIO, &fflag, td->td_ucred, td);
807 (void) fo_ioctl(fp2, FIONBIO, &fflag, td->td_ucred, td);
813 fdclose(fdp, fp2, rsv[1], td);
816 fdclose(fdp, fp1, rsv[0], td);
828 sys_socketpair(struct thread *td, struct socketpair_args *uap)
832 error = kern_socketpair(td, uap->domain, uap->type,
836 error = copyout(sv, uap->rsv, 2 * sizeof(int));
838 (void)kern_close(td, sv[0]);
839 (void)kern_close(td, sv[1]);
845 sendit(td, s, mp, flags)
851 struct mbuf *control;
855 #ifdef CAPABILITY_MODE
856 if (IN_CAPABILITY_MODE(td) && (mp->msg_name != NULL))
860 if (mp->msg_name != NULL) {
861 error = getsockaddr(&to, mp->msg_name, mp->msg_namelen);
871 if (mp->msg_control) {
872 if (mp->msg_controllen < sizeof(struct cmsghdr)
873 #ifdef COMPAT_OLDSOCK
874 && mp->msg_flags != MSG_COMPAT
880 error = sockargs(&control, mp->msg_control,
881 mp->msg_controllen, MT_CONTROL);
884 #ifdef COMPAT_OLDSOCK
885 if (mp->msg_flags == MSG_COMPAT) {
888 M_PREPEND(control, sizeof(*cm), M_WAITOK);
889 cm = mtod(control, struct cmsghdr *);
890 cm->cmsg_len = control->m_len;
891 cm->cmsg_level = SOL_SOCKET;
892 cm->cmsg_type = SCM_RIGHTS;
899 error = kern_sendit(td, s, mp, flags, control, UIO_USERSPACE);
907 kern_sendit(td, s, mp, flags, control, segflg)
912 struct mbuf *control;
921 struct uio *ktruio = NULL;
927 cap_rights_init(&rights, CAP_SEND);
928 if (mp->msg_name != NULL) {
929 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, mp->msg_name);
930 cap_rights_set(&rights, CAP_CONNECT);
932 error = getsock_cap(td->td_proc->p_fd, s, &rights, &fp, NULL);
935 so = (struct socket *)fp->f_data;
938 if (mp->msg_name != NULL && KTRPOINT(td, KTR_STRUCT))
939 ktrsockaddr(mp->msg_name);
942 if (mp->msg_name != NULL) {
943 error = mac_socket_check_connect(td->td_ucred, so,
948 error = mac_socket_check_send(td->td_ucred, so);
953 auio.uio_iov = mp->msg_iov;
954 auio.uio_iovcnt = mp->msg_iovlen;
955 auio.uio_segflg = segflg;
956 auio.uio_rw = UIO_WRITE;
958 auio.uio_offset = 0; /* XXX */
961 for (i = 0; i < mp->msg_iovlen; i++, iov++) {
962 if ((auio.uio_resid += iov->iov_len) < 0) {
968 if (KTRPOINT(td, KTR_GENIO))
969 ktruio = cloneuio(&auio);
971 len = auio.uio_resid;
972 error = sosend(so, mp->msg_name, &auio, 0, control, flags, td);
974 if (auio.uio_resid != len && (error == ERESTART ||
975 error == EINTR || error == EWOULDBLOCK))
977 /* Generation of SIGPIPE can be controlled per socket */
978 if (error == EPIPE && !(so->so_options & SO_NOSIGPIPE) &&
979 !(flags & MSG_NOSIGNAL)) {
980 PROC_LOCK(td->td_proc);
981 tdsignal(td, SIGPIPE);
982 PROC_UNLOCK(td->td_proc);
986 td->td_retval[0] = len - auio.uio_resid;
988 if (ktruio != NULL) {
989 ktruio->uio_resid = td->td_retval[0];
990 ktrgenio(s, UIO_WRITE, ktruio, error);
1001 struct sendto_args /* {
1013 msg.msg_name = uap->to;
1014 msg.msg_namelen = uap->tolen;
1015 msg.msg_iov = &aiov;
1017 msg.msg_control = 0;
1018 #ifdef COMPAT_OLDSOCK
1021 aiov.iov_base = uap->buf;
1022 aiov.iov_len = uap->len;
1023 return (sendit(td, uap->s, &msg, uap->flags));
1026 #ifdef COMPAT_OLDSOCK
1030 struct osend_args /* {
1041 msg.msg_namelen = 0;
1042 msg.msg_iov = &aiov;
1044 aiov.iov_base = uap->buf;
1045 aiov.iov_len = uap->len;
1046 msg.msg_control = 0;
1048 return (sendit(td, uap->s, &msg, uap->flags));
1054 struct osendmsg_args /* {
1064 error = copyin(uap->msg, &msg, sizeof (struct omsghdr));
1067 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE);
1071 msg.msg_flags = MSG_COMPAT;
1072 error = sendit(td, uap->s, &msg, uap->flags);
1079 sys_sendmsg(td, uap)
1081 struct sendmsg_args /* {
1091 error = copyin(uap->msg, &msg, sizeof (msg));
1094 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE);
1098 #ifdef COMPAT_OLDSOCK
1101 error = sendit(td, uap->s, &msg, uap->flags);
1107 kern_recvit(td, s, mp, fromseg, controlp)
1111 enum uio_seg fromseg;
1112 struct mbuf **controlp;
1116 struct mbuf *m, *control = NULL;
1120 struct sockaddr *fromsa = NULL;
1121 cap_rights_t rights;
1123 struct uio *ktruio = NULL;
1128 if (controlp != NULL)
1132 error = getsock_cap(td->td_proc->p_fd, s,
1133 cap_rights_init(&rights, CAP_RECV), &fp, NULL);
1139 error = mac_socket_check_receive(td->td_ucred, so);
1146 auio.uio_iov = mp->msg_iov;
1147 auio.uio_iovcnt = mp->msg_iovlen;
1148 auio.uio_segflg = UIO_USERSPACE;
1149 auio.uio_rw = UIO_READ;
1151 auio.uio_offset = 0; /* XXX */
1154 for (i = 0; i < mp->msg_iovlen; i++, iov++) {
1155 if ((auio.uio_resid += iov->iov_len) < 0) {
1161 if (KTRPOINT(td, KTR_GENIO))
1162 ktruio = cloneuio(&auio);
1164 len = auio.uio_resid;
1165 error = soreceive(so, &fromsa, &auio, NULL,
1166 (mp->msg_control || controlp) ? &control : NULL,
1169 if (auio.uio_resid != len && (error == ERESTART ||
1170 error == EINTR || error == EWOULDBLOCK))
1174 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, fromsa);
1176 if (ktruio != NULL) {
1177 ktruio->uio_resid = len - auio.uio_resid;
1178 ktrgenio(s, UIO_READ, ktruio, error);
1183 td->td_retval[0] = len - auio.uio_resid;
1185 len = mp->msg_namelen;
1186 if (len <= 0 || fromsa == NULL)
1189 /* save sa_len before it is destroyed by MSG_COMPAT */
1190 len = MIN(len, fromsa->sa_len);
1191 #ifdef COMPAT_OLDSOCK
1192 if (mp->msg_flags & MSG_COMPAT)
1193 ((struct osockaddr *)fromsa)->sa_family =
1196 if (fromseg == UIO_USERSPACE) {
1197 error = copyout(fromsa, mp->msg_name,
1202 bcopy(fromsa, mp->msg_name, len);
1204 mp->msg_namelen = len;
1206 if (mp->msg_control && controlp == NULL) {
1207 #ifdef COMPAT_OLDSOCK
1209 * We assume that old recvmsg calls won't receive access
1210 * rights and other control info, esp. as control info
1211 * is always optional and those options didn't exist in 4.3.
1212 * If we receive rights, trim the cmsghdr; anything else
1215 if (control && mp->msg_flags & MSG_COMPAT) {
1216 if (mtod(control, struct cmsghdr *)->cmsg_level !=
1218 mtod(control, struct cmsghdr *)->cmsg_type !=
1220 mp->msg_controllen = 0;
1223 control->m_len -= sizeof (struct cmsghdr);
1224 control->m_data += sizeof (struct cmsghdr);
1227 len = mp->msg_controllen;
1229 mp->msg_controllen = 0;
1230 ctlbuf = mp->msg_control;
1232 while (m && len > 0) {
1233 unsigned int tocopy;
1235 if (len >= m->m_len)
1238 mp->msg_flags |= MSG_CTRUNC;
1242 if ((error = copyout(mtod(m, caddr_t),
1243 ctlbuf, tocopy)) != 0)
1250 mp->msg_controllen = ctlbuf - (caddr_t)mp->msg_control;
1255 if (fromsa && KTRPOINT(td, KTR_STRUCT))
1256 ktrsockaddr(fromsa);
1258 free(fromsa, M_SONAME);
1260 if (error == 0 && controlp != NULL)
1261 *controlp = control;
1269 recvit(td, s, mp, namelenp)
1277 error = kern_recvit(td, s, mp, UIO_USERSPACE, NULL);
1280 if (namelenp != NULL) {
1281 error = copyout(&mp->msg_namelen, namelenp, sizeof (socklen_t));
1282 #ifdef COMPAT_OLDSOCK
1283 if (mp->msg_flags & MSG_COMPAT)
1284 error = 0; /* old recvfrom didn't check */
1291 sys_recvfrom(td, uap)
1293 struct recvfrom_args /* {
1298 struct sockaddr * __restrict from;
1299 socklen_t * __restrict fromlenaddr;
1306 if (uap->fromlenaddr) {
1307 error = copyin(uap->fromlenaddr,
1308 &msg.msg_namelen, sizeof (msg.msg_namelen));
1312 msg.msg_namelen = 0;
1314 msg.msg_name = uap->from;
1315 msg.msg_iov = &aiov;
1317 aiov.iov_base = uap->buf;
1318 aiov.iov_len = uap->len;
1319 msg.msg_control = 0;
1320 msg.msg_flags = uap->flags;
1321 error = recvit(td, uap->s, &msg, uap->fromlenaddr);
1326 #ifdef COMPAT_OLDSOCK
1330 struct recvfrom_args *uap;
1333 uap->flags |= MSG_COMPAT;
1334 return (sys_recvfrom(td, uap));
1338 #ifdef COMPAT_OLDSOCK
1342 struct orecv_args /* {
1353 msg.msg_namelen = 0;
1354 msg.msg_iov = &aiov;
1356 aiov.iov_base = uap->buf;
1357 aiov.iov_len = uap->len;
1358 msg.msg_control = 0;
1359 msg.msg_flags = uap->flags;
1360 return (recvit(td, uap->s, &msg, NULL));
1364 * Old recvmsg. This code takes advantage of the fact that the old msghdr
1365 * overlays the new one, missing only the flags, and with the (old) access
1366 * rights where the control fields are now.
1371 struct orecvmsg_args /* {
1373 struct omsghdr *msg;
1381 error = copyin(uap->msg, &msg, sizeof (struct omsghdr));
1384 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE);
1387 msg.msg_flags = uap->flags | MSG_COMPAT;
1389 error = recvit(td, uap->s, &msg, &uap->msg->msg_namelen);
1390 if (msg.msg_controllen && error == 0)
1391 error = copyout(&msg.msg_controllen,
1392 &uap->msg->msg_accrightslen, sizeof (int));
1399 sys_recvmsg(td, uap)
1401 struct recvmsg_args /* {
1408 struct iovec *uiov, *iov;
1411 error = copyin(uap->msg, &msg, sizeof (msg));
1414 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE);
1417 msg.msg_flags = uap->flags;
1418 #ifdef COMPAT_OLDSOCK
1419 msg.msg_flags &= ~MSG_COMPAT;
1423 error = recvit(td, uap->s, &msg, NULL);
1426 error = copyout(&msg, uap->msg, sizeof(msg));
1434 sys_shutdown(td, uap)
1436 struct shutdown_args /* {
1443 cap_rights_t rights;
1446 AUDIT_ARG_FD(uap->s);
1447 error = getsock_cap(td->td_proc->p_fd, uap->s,
1448 cap_rights_init(&rights, CAP_SHUTDOWN), &fp, NULL);
1451 error = soshutdown(so, uap->how);
1459 sys_setsockopt(td, uap)
1461 struct setsockopt_args /* {
1470 return (kern_setsockopt(td, uap->s, uap->level, uap->name,
1471 uap->val, UIO_USERSPACE, uap->valsize));
1475 kern_setsockopt(td, s, level, name, val, valseg, valsize)
1481 enum uio_seg valseg;
1486 struct sockopt sopt;
1487 cap_rights_t rights;
1490 if (val == NULL && valsize != 0)
1492 if ((int)valsize < 0)
1495 sopt.sopt_dir = SOPT_SET;
1496 sopt.sopt_level = level;
1497 sopt.sopt_name = name;
1498 sopt.sopt_val = val;
1499 sopt.sopt_valsize = valsize;
1505 sopt.sopt_td = NULL;
1508 panic("kern_setsockopt called with bad valseg");
1512 error = getsock_cap(td->td_proc->p_fd, s,
1513 cap_rights_init(&rights, CAP_SETSOCKOPT), &fp, NULL);
1516 error = sosetopt(so, &sopt);
1524 sys_getsockopt(td, uap)
1526 struct getsockopt_args /* {
1530 void * __restrict val;
1531 socklen_t * __restrict avalsize;
1538 error = copyin(uap->avalsize, &valsize, sizeof (valsize));
1543 error = kern_getsockopt(td, uap->s, uap->level, uap->name,
1544 uap->val, UIO_USERSPACE, &valsize);
1547 error = copyout(&valsize, uap->avalsize, sizeof (valsize));
1552 * Kernel version of getsockopt.
1553 * optval can be a userland or userspace. optlen is always a kernel pointer.
1556 kern_getsockopt(td, s, level, name, val, valseg, valsize)
1562 enum uio_seg valseg;
1567 struct sockopt sopt;
1568 cap_rights_t rights;
1573 if ((int)*valsize < 0)
1576 sopt.sopt_dir = SOPT_GET;
1577 sopt.sopt_level = level;
1578 sopt.sopt_name = name;
1579 sopt.sopt_val = val;
1580 sopt.sopt_valsize = (size_t)*valsize; /* checked non-negative above */
1586 sopt.sopt_td = NULL;
1589 panic("kern_getsockopt called with bad valseg");
1593 error = getsock_cap(td->td_proc->p_fd, s,
1594 cap_rights_init(&rights, CAP_GETSOCKOPT), &fp, NULL);
1597 error = sogetopt(so, &sopt);
1598 *valsize = sopt.sopt_valsize;
1605 * getsockname1() - Get socket name.
1609 getsockname1(td, uap, compat)
1611 struct getsockname_args /* {
1613 struct sockaddr * __restrict asa;
1614 socklen_t * __restrict alen;
1618 struct sockaddr *sa;
1622 error = copyin(uap->alen, &len, sizeof(len));
1626 error = kern_getsockname(td, uap->fdes, &sa, &len);
1631 #ifdef COMPAT_OLDSOCK
1633 ((struct osockaddr *)sa)->sa_family = sa->sa_family;
1635 error = copyout(sa, uap->asa, (u_int)len);
1639 error = copyout(&len, uap->alen, sizeof(len));
1644 kern_getsockname(struct thread *td, int fd, struct sockaddr **sa,
1649 cap_rights_t rights;
1654 error = getsock_cap(td->td_proc->p_fd, fd,
1655 cap_rights_init(&rights, CAP_GETSOCKNAME), &fp, NULL);
1660 CURVNET_SET(so->so_vnet);
1661 error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, sa);
1668 len = MIN(*alen, (*sa)->sa_len);
1671 if (KTRPOINT(td, KTR_STRUCT))
1676 if (error != 0 && *sa != NULL) {
1677 free(*sa, M_SONAME);
1684 sys_getsockname(td, uap)
1686 struct getsockname_args *uap;
1689 return (getsockname1(td, uap, 0));
1692 #ifdef COMPAT_OLDSOCK
1694 ogetsockname(td, uap)
1696 struct getsockname_args *uap;
1699 return (getsockname1(td, uap, 1));
1701 #endif /* COMPAT_OLDSOCK */
1704 * getpeername1() - Get name of peer for connected socket.
1708 getpeername1(td, uap, compat)
1710 struct getpeername_args /* {
1712 struct sockaddr * __restrict asa;
1713 socklen_t * __restrict alen;
1717 struct sockaddr *sa;
1721 error = copyin(uap->alen, &len, sizeof (len));
1725 error = kern_getpeername(td, uap->fdes, &sa, &len);
1730 #ifdef COMPAT_OLDSOCK
1732 ((struct osockaddr *)sa)->sa_family = sa->sa_family;
1734 error = copyout(sa, uap->asa, (u_int)len);
1738 error = copyout(&len, uap->alen, sizeof(len));
1743 kern_getpeername(struct thread *td, int fd, struct sockaddr **sa,
1748 cap_rights_t rights;
1753 error = getsock_cap(td->td_proc->p_fd, fd,
1754 cap_rights_init(&rights, CAP_GETPEERNAME), &fp, NULL);
1758 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
1763 CURVNET_SET(so->so_vnet);
1764 error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, sa);
1771 len = MIN(*alen, (*sa)->sa_len);
1774 if (KTRPOINT(td, KTR_STRUCT))
1778 if (error != 0 && *sa != NULL) {
1779 free(*sa, M_SONAME);
1788 sys_getpeername(td, uap)
1790 struct getpeername_args *uap;
1793 return (getpeername1(td, uap, 0));
1796 #ifdef COMPAT_OLDSOCK
1798 ogetpeername(td, uap)
1800 struct ogetpeername_args *uap;
1803 /* XXX uap should have type `getpeername_args *' to begin with. */
1804 return (getpeername1(td, (struct getpeername_args *)uap, 1));
1806 #endif /* COMPAT_OLDSOCK */
1809 sockargs(mp, buf, buflen, type)
1814 struct sockaddr *sa;
1818 if (buflen > MLEN) {
1819 #ifdef COMPAT_OLDSOCK
1820 if (type == MT_SONAME && buflen <= 112)
1821 buflen = MLEN; /* unix domain compat. hack */
1824 if (buflen > MCLBYTES)
1827 m = m_get2(buflen, M_WAITOK, type, 0);
1829 error = copyin(buf, mtod(m, caddr_t), (u_int)buflen);
1834 if (type == MT_SONAME) {
1835 sa = mtod(m, struct sockaddr *);
1837 #if defined(COMPAT_OLDSOCK) && BYTE_ORDER != BIG_ENDIAN
1838 if (sa->sa_family == 0 && sa->sa_len < AF_MAX)
1839 sa->sa_family = sa->sa_len;
1841 sa->sa_len = buflen;
1848 getsockaddr(namp, uaddr, len)
1849 struct sockaddr **namp;
1853 struct sockaddr *sa;
1856 if (len > SOCK_MAXADDRLEN)
1857 return (ENAMETOOLONG);
1858 if (len < offsetof(struct sockaddr, sa_data[0]))
1860 sa = malloc(len, M_SONAME, M_WAITOK);
1861 error = copyin(uaddr, sa, len);
1865 #if defined(COMPAT_OLDSOCK) && BYTE_ORDER != BIG_ENDIAN
1866 if (sa->sa_family == 0 && sa->sa_len < AF_MAX)
1867 sa->sa_family = sa->sa_len;
1876 filt_sfsync_attach(struct knote *kn)
1878 struct sendfile_sync *sfs = (struct sendfile_sync *) kn->kn_sdata;
1879 struct knlist *knl = &sfs->klist;
1881 SFSYNC_DPRINTF("%s: kn=%p, sfs=%p\n", __func__, kn, sfs);
1884 * Validate that we actually received this via the kernel API.
1886 if ((kn->kn_flags & EV_FLAG1) == 0)
1889 kn->kn_ptr.p_v = sfs;
1890 kn->kn_flags &= ~EV_FLAG1;
1892 knl->kl_lock(knl->kl_lockarg);
1894 * If we're in the "freeing" state,
1895 * don't allow the add. That way we don't
1896 * end up racing with some other thread that
1897 * is trying to finish some setup.
1899 if (sfs->state == SF_STATE_FREEING) {
1900 knl->kl_unlock(knl->kl_lockarg);
1903 knlist_add(&sfs->klist, kn, 1);
1904 knl->kl_unlock(knl->kl_lockarg);
1910 * Called when a knote is being detached.
1913 filt_sfsync_detach(struct knote *kn)
1916 struct sendfile_sync *sfs;
1919 sfs = kn->kn_ptr.p_v;
1922 SFSYNC_DPRINTF("%s: kn=%p, sfs=%p\n", __func__, kn, sfs);
1924 knl->kl_lock(knl->kl_lockarg);
1925 if (!knlist_empty(knl))
1926 knlist_remove(knl, kn, 1);
1929 * If the list is empty _AND_ the refcount is 0
1930 * _AND_ we've finished the setup phase and now
1931 * we're in the running phase, we can free the
1932 * underlying sendfile_sync.
1934 * But we shouldn't do it before finishing the
1935 * underlying divorce from the knote.
1937 * So, we have the sfsync lock held; transition
1938 * it to "freeing", then unlock, then free
1941 if (knlist_empty(knl)) {
1942 if (sfs->state == SF_STATE_COMPLETED && sfs->count == 0) {
1943 SFSYNC_DPRINTF("%s: (%llu) sfs=%p; completed, "
1944 "count==0, empty list: time to free!\n",
1946 (unsigned long long) curthread->td_tid,
1948 sf_sync_set_state(sfs, SF_STATE_FREEING, 1);
1952 knl->kl_unlock(knl->kl_lockarg);
1955 * Only call free if we're the one who has transitioned things
1956 * to free. Otherwise we could race with another thread that
1957 * is currently tearing things down.
1960 SFSYNC_DPRINTF("%s: (%llu) sfs=%p, %s:%d\n",
1962 (unsigned long long) curthread->td_tid,
1971 filt_sfsync(struct knote *kn, long hint)
1973 struct sendfile_sync *sfs = (struct sendfile_sync *) kn->kn_ptr.p_v;
1976 SFSYNC_DPRINTF("%s: kn=%p, sfs=%p\n", __func__, kn, sfs);
1979 * XXX add a lock assertion here!
1981 ret = (sfs->count == 0 && sfs->state == SF_STATE_COMPLETED);
1987 * Add more references to a vm_page + sf_buf + sendfile_sync.
1990 sf_ext_ref(void *arg1, void *arg2)
1992 struct sf_buf *sf = arg1;
1993 struct sendfile_sync *sfs = arg2;
1994 vm_page_t pg = sf_buf_page(sf);
2003 mtx_lock(&sfs->mtx);
2004 KASSERT(sfs->count > 0, ("Sendfile sync botchup count == 0"));
2006 mtx_unlock(&sfs->mtx);
2011 * Detach mapped page and release resources back to the system.
2014 sf_ext_free(void *arg1, void *arg2)
2016 struct sf_buf *sf = arg1;
2017 struct sendfile_sync *sfs = arg2;
2018 vm_page_t pg = sf_buf_page(sf);
2023 vm_page_unwire(pg, PQ_INACTIVE);
2025 * Check for the object going away on us. This can
2026 * happen since we don't hold a reference to it.
2027 * If so, we're responsible for freeing the page.
2029 if (pg->wire_count == 0 && pg->object == NULL)
2038 * Called to remove a reference to a sf_sync object.
2040 * This is generally done during the mbuf free path to signify
2041 * that one of the mbufs in the transaction has been completed.
2043 * If we're doing SF_SYNC and the refcount is zero then we'll wake
2046 * IF we're doing SF_KQUEUE and the refcount is zero then we'll
2047 * fire off the knote.
2050 sf_sync_deref(struct sendfile_sync *sfs)
2057 mtx_lock(&sfs->mtx);
2058 KASSERT(sfs->count> 0, ("Sendfile sync botchup count == 0"));
2062 * Only fire off the wakeup / kqueue notification if
2063 * we are in the running state.
2065 if (sfs->count == 0 && sfs->state == SF_STATE_COMPLETED) {
2066 if (sfs->flags & SF_SYNC)
2067 cv_signal(&sfs->cv);
2069 if (sfs->flags & SF_KQUEUE) {
2070 SFSYNC_DPRINTF("%s: (%llu) sfs=%p: knote!\n",
2072 (unsigned long long) curthread->td_tid,
2074 KNOTE_LOCKED(&sfs->klist, 1);
2078 * If we're not waiting around for a sync,
2079 * check if the knote list is empty.
2080 * If it is, we transition to free.
2082 * XXX I think it's about time I added some state
2083 * or flag that says whether we're supposed to be
2084 * waiting around until we've done a signal.
2086 * XXX Ie, the reason that I don't free it here
2087 * is because the caller will free the last reference,
2088 * not us. That should be codified in some flag
2089 * that indicates "self-free" rather than checking
2090 * for SF_SYNC all the time.
2092 if ((sfs->flags & SF_SYNC) == 0 && knlist_empty(&sfs->klist)) {
2093 SFSYNC_DPRINTF("%s: (%llu) sfs=%p; completed, "
2094 "count==0, empty list: time to free!\n",
2096 (unsigned long long) curthread->td_tid,
2098 sf_sync_set_state(sfs, SF_STATE_FREEING, 1);
2103 mtx_unlock(&sfs->mtx);
2106 * Attempt to do a free here.
2108 * We do this outside of the lock because it may destroy the
2109 * lock in question as it frees things. We can optimise this
2112 * XXX yes, we should make it a requirement to hold the
2113 * lock across sf_sync_free().
2116 SFSYNC_DPRINTF("%s: (%llu) sfs=%p\n",
2118 (unsigned long long) curthread->td_tid,
2125 * Allocate a sendfile_sync state structure.
2127 * For now this only knows about the "sleep" sync, but later it will
2128 * grow various other personalities.
2130 struct sendfile_sync *
2131 sf_sync_alloc(uint32_t flags)
2133 struct sendfile_sync *sfs;
2135 sfs = uma_zalloc(zone_sfsync, M_WAITOK | M_ZERO);
2136 mtx_init(&sfs->mtx, "sendfile", NULL, MTX_DEF);
2137 cv_init(&sfs->cv, "sendfile");
2139 sfs->state = SF_STATE_SETUP;
2140 knlist_init_mtx(&sfs->klist, &sfs->mtx);
2142 SFSYNC_DPRINTF("%s: sfs=%p, flags=0x%08x\n", __func__, sfs, sfs->flags);
2148 * Take a reference to a sfsync instance.
2150 * This has to map 1:1 to free calls coming in via sf_ext_free(),
2151 * so typically this will be referenced once for each mbuf allocated.
2154 sf_sync_ref(struct sendfile_sync *sfs)
2160 mtx_lock(&sfs->mtx);
2162 mtx_unlock(&sfs->mtx);
2166 sf_sync_syscall_wait(struct sendfile_sync *sfs)
2172 KASSERT(mtx_owned(&sfs->mtx), ("%s: sfs=%p: not locked but should be!",
2177 * If we're not requested to wait during the syscall,
2178 * don't bother waiting.
2180 if ((sfs->flags & SF_SYNC) == 0)
2184 * This is a bit suboptimal and confusing, so bear with me.
2186 * Ideally sf_sync_syscall_wait() will wait until
2187 * all pending mbuf transmit operations are done.
2188 * This means that when sendfile becomes async, it'll
2189 * run in the background and will transition from
2190 * RUNNING to COMPLETED when it's finished acquiring
2191 * new things to send. Then, when the mbufs finish
2192 * sending, COMPLETED + sfs->count == 0 is enough to
2193 * know that no further work is being done.
2195 * So, we will sleep on both RUNNING and COMPLETED.
2196 * It's up to the (in progress) async sendfile loop
2197 * to transition the sf_sync from RUNNING to
2198 * COMPLETED so the wakeup above will actually
2199 * do the cv_signal() call.
2201 if (sfs->state != SF_STATE_COMPLETED && sfs->state != SF_STATE_RUNNING)
2204 if (sfs->count != 0)
2205 cv_wait(&sfs->cv, &sfs->mtx);
2206 KASSERT(sfs->count == 0, ("sendfile sync still busy"));
2213 * Free an sf_sync if it's appropriate to.
2216 sf_sync_free(struct sendfile_sync *sfs)
2222 SFSYNC_DPRINTF("%s: (%lld) sfs=%p; called; state=%d, flags=0x%08x "
2225 (long long) curthread->td_tid,
2231 mtx_lock(&sfs->mtx);
2234 * We keep the sf_sync around if the state is active,
2235 * we are doing kqueue notification and we have active
2238 * If the caller wants to free us right this second it
2239 * should transition this to the freeing state.
2241 * So, complain loudly if they break this rule.
2243 if (sfs->state != SF_STATE_FREEING) {
2244 printf("%s: (%llu) sfs=%p; not freeing; let's wait!\n",
2246 (unsigned long long) curthread->td_tid,
2248 mtx_unlock(&sfs->mtx);
2252 KASSERT(sfs->count == 0, ("sendfile sync still busy"));
2253 cv_destroy(&sfs->cv);
2255 * This doesn't call knlist_detach() on each knote; it just frees
2258 knlist_delete(&sfs->klist, curthread, 1);
2259 mtx_destroy(&sfs->mtx);
2260 SFSYNC_DPRINTF("%s: (%llu) sfs=%p; freeing\n",
2262 (unsigned long long) curthread->td_tid,
2264 uma_zfree(zone_sfsync, sfs);
2268 * Setup a sf_sync to post a kqueue notification when things are complete.
2271 sf_sync_kqueue_setup(struct sendfile_sync *sfs, struct sf_hdtr_kq *sfkq)
2276 sfs->flags |= SF_KQUEUE;
2278 /* Check the flags are valid */
2279 if ((sfkq->kq_flags & ~(EV_CLEAR | EV_DISPATCH | EV_ONESHOT)) != 0)
2282 SFSYNC_DPRINTF("%s: sfs=%p: kqfd=%d, flags=0x%08x, ident=%p, udata=%p\n",
2287 (void *) sfkq->kq_ident,
2288 (void *) sfkq->kq_udata);
2290 /* Setup and register a knote on the given kqfd. */
2291 kev.ident = (uintptr_t) sfkq->kq_ident;
2292 kev.filter = EVFILT_SENDFILE;
2293 kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1 | sfkq->kq_flags;
2294 kev.data = (intptr_t) sfs;
2295 kev.udata = sfkq->kq_udata;
2297 error = kqfd_register(sfkq->kq_fd, &kev, curthread, 1);
2299 SFSYNC_DPRINTF("%s: returned %d\n", __func__, error);
2305 sf_sync_set_state(struct sendfile_sync *sfs, sendfile_sync_state_t state,
2308 sendfile_sync_state_t old_state;
2311 mtx_lock(&sfs->mtx);
2314 * Update our current state.
2316 old_state = sfs->state;
2318 SFSYNC_DPRINTF("%s: (%llu) sfs=%p; going from %d to %d\n",
2320 (unsigned long long) curthread->td_tid,
2326 * If we're transitioning from RUNNING to COMPLETED and the count is
2327 * zero, then post the knote. The caller may have completed the
2328 * send before we updated the state to COMPLETED and we need to make
2329 * sure this is communicated.
2331 if (old_state == SF_STATE_RUNNING
2332 && state == SF_STATE_COMPLETED
2334 && sfs->flags & SF_KQUEUE) {
2335 SFSYNC_DPRINTF("%s: (%llu) sfs=%p: triggering knote!\n",
2337 (unsigned long long) curthread->td_tid,
2339 KNOTE_LOCKED(&sfs->klist, 1);
2343 mtx_unlock(&sfs->mtx);
2347 * Set the retval/errno for the given transaction.
2349 * This will eventually/ideally be used when the KNOTE is fired off
2350 * to signify the completion of this transaction.
2352 * The sfsync lock should be held before entering this function.
2355 sf_sync_set_retval(struct sendfile_sync *sfs, off_t retval, int xerrno)
2358 KASSERT(mtx_owned(&sfs->mtx), ("%s: sfs=%p: not locked but should be!",
2362 SFSYNC_DPRINTF("%s: (%llu) sfs=%p: errno=%d, retval=%jd\n",
2364 (unsigned long long) curthread->td_tid,
2369 sfs->retval = retval;
2370 sfs->xerrno = xerrno;
2376 * int sendfile(int fd, int s, off_t offset, size_t nbytes,
2377 * struct sf_hdtr *hdtr, off_t *sbytes, int flags)
2379 * Send a file specified by 'fd' and starting at 'offset' to a socket
2380 * specified by 's'. Send only 'nbytes' of the file or until EOF if nbytes ==
2381 * 0. Optionally add a header and/or trailer to the socket output. If
2382 * specified, write the total number of bytes sent into *sbytes.
2385 sys_sendfile(struct thread *td, struct sendfile_args *uap)
2388 return (do_sendfile(td, uap, 0));
2392 _do_sendfile(struct thread *td, int src_fd, int sock_fd, int flags,
2393 int compat, off_t offset, size_t nbytes, off_t *sbytes,
2394 struct uio *hdr_uio,
2395 struct uio *trl_uio, struct sf_hdtr_kq *hdtr_kq)
2397 cap_rights_t rights;
2398 struct sendfile_sync *sfs = NULL;
2404 AUDIT_ARG_FD(src_fd);
2406 if (hdtr_kq != NULL)
2410 * sendfile(2) can start at any offset within a file so we require
2411 * CAP_READ+CAP_SEEK = CAP_PREAD.
2413 if ((error = fget_read(td, src_fd,
2414 cap_rights_init(&rights, CAP_PREAD), &fp)) != 0) {
2419 * IF SF_KQUEUE is set but we haven't copied in anything for
2420 * kqueue data, error out.
2422 if (flags & SF_KQUEUE && do_kqueue == 0) {
2423 SFSYNC_DPRINTF("%s: SF_KQUEUE but no KQUEUE data!\n", __func__);
2428 * If we need to wait for completion, initialise the sfsync
2431 if (flags & (SF_SYNC | SF_KQUEUE))
2432 sfs = sf_sync_alloc(flags & (SF_SYNC | SF_KQUEUE));
2434 if (flags & SF_KQUEUE) {
2435 error = sf_sync_kqueue_setup(sfs, hdtr_kq);
2437 SFSYNC_DPRINTF("%s: (%llu) error; sfs=%p\n",
2439 (unsigned long long) curthread->td_tid,
2441 sf_sync_set_state(sfs, SF_STATE_FREEING, 0);
2448 * Do the sendfile call.
2450 * If this fails, it'll free the mbuf chain which will free up the
2451 * sendfile_sync references.
2453 error = fo_sendfile(fp, sock_fd, hdr_uio, trl_uio, offset,
2454 nbytes, sbytes, flags, compat ? SFK_COMPAT : 0, sfs, td);
2457 * If the sendfile call succeeded, transition the sf_sync state
2458 * to RUNNING, then COMPLETED.
2460 * If the sendfile call failed, then the sendfile call may have
2461 * actually sent some data first - so we check to see whether
2462 * any data was sent. If some data was queued (ie, count > 0)
2463 * then we can't call free; we have to wait until the partial
2464 * transaction completes before we continue along.
2466 * This has the side effect of firing off the knote
2467 * if the refcount has hit zero by the time we get here.
2470 mtx_lock(&sfs->mtx);
2471 if (error == 0 || sfs->count > 0) {
2473 * When it's time to do async sendfile, the transition
2474 * to RUNNING signifies that we're actually actively
2475 * adding and completing mbufs. When the last disk
2476 * buffer is read (ie, when we're not doing any
2477 * further read IO and all subsequent stuff is mbuf
2478 * transmissions) we'll transition to COMPLETED
2479 * and when the final mbuf is freed, the completion
2482 sf_sync_set_state(sfs, SF_STATE_RUNNING, 1);
2485 * Set the retval before we signal completed.
2486 * If we do it the other way around then transitioning to
2487 * COMPLETED may post the knote before you set the return
2490 * XXX for now, errno is always 0, as we don't post
2491 * knotes if sendfile failed. Maybe that'll change later.
2493 sf_sync_set_retval(sfs, *sbytes, error);
2496 * And now transition to completed, which will kick off
2497 * the knote if required.
2499 sf_sync_set_state(sfs, SF_STATE_COMPLETED, 1);
2502 * Error isn't zero, sfs_count is zero, so we
2503 * won't have some other thing to wake things up.
2506 sf_sync_set_state(sfs, SF_STATE_FREEING, 1);
2511 * Next - wait if appropriate.
2513 sf_sync_syscall_wait(sfs);
2516 * If we're not doing kqueue notifications, we can
2517 * transition this immediately to the freeing state.
2519 if ((sfs->flags & SF_KQUEUE) == 0) {
2520 sf_sync_set_state(sfs, SF_STATE_FREEING, 1);
2524 mtx_unlock(&sfs->mtx);
2528 * If do_free is set, free here.
2530 * If we're doing no-kqueue notification and it's just sleep notification,
2531 * we also do free; it's the only chance we have.
2533 if (sfs != NULL && do_free == 1) {
2538 * XXX Should we wait until the send has completed before freeing the source
2539 * file handle? It's the previous behaviour, sure, but is it required?
2540 * We've wired down the page references after all.
2551 do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
2553 struct sf_hdtr hdtr;
2554 struct sf_hdtr_kq hdtr_kq;
2555 struct uio *hdr_uio, *trl_uio;
2561 * File offset must be positive. If it goes beyond EOF
2562 * we send only the header/trailer and no payload data.
2564 if (uap->offset < 0)
2567 hdr_uio = trl_uio = NULL;
2569 if (uap->hdtr != NULL) {
2570 error = copyin(uap->hdtr, &hdtr, sizeof(hdtr));
2573 if (hdtr.headers != NULL) {
2574 error = copyinuio(hdtr.headers, hdtr.hdr_cnt, &hdr_uio);
2578 if (hdtr.trailers != NULL) {
2579 error = copyinuio(hdtr.trailers, hdtr.trl_cnt, &trl_uio);
2585 * If SF_KQUEUE is set, then we need to also copy in
2586 * the kqueue data after the normal hdtr set and set
2589 if (uap->flags & SF_KQUEUE) {
2590 error = copyin(((char *) uap->hdtr) + sizeof(hdtr),
2600 error = _do_sendfile(td, uap->fd, uap->s, uap->flags, compat,
2601 uap->offset, uap->nbytes, &sbytes, hdr_uio, trl_uio, &hdtr_kq);
2603 if (uap->sbytes != NULL) {
2604 copyout(&sbytes, uap->sbytes, sizeof(off_t));
2607 free(hdr_uio, M_IOV);
2608 free(trl_uio, M_IOV);
2612 #ifdef COMPAT_FREEBSD4
2614 freebsd4_sendfile(struct thread *td, struct freebsd4_sendfile_args *uap)
2616 struct sendfile_args args;
2620 args.offset = uap->offset;
2621 args.nbytes = uap->nbytes;
2622 args.hdtr = uap->hdtr;
2623 args.sbytes = uap->sbytes;
2624 args.flags = uap->flags;
2626 return (do_sendfile(td, &args, 1));
2628 #endif /* COMPAT_FREEBSD4 */
2631 sendfile_readpage(vm_object_t obj, struct vnode *vp, int nd,
2632 off_t off, int xfsize, int bsize, struct thread *td, vm_page_t *res)
2637 int error, readahead, rv;
2639 pindex = OFF_TO_IDX(off);
2640 VM_OBJECT_WLOCK(obj);
2641 m = vm_page_grab(obj, pindex, (vp != NULL ? VM_ALLOC_NOBUSY |
2642 VM_ALLOC_IGN_SBUSY : 0) | VM_ALLOC_WIRED | VM_ALLOC_NORMAL);
2645 * Check if page is valid for what we need, otherwise initiate I/O.
2647 * The non-zero nd argument prevents disk I/O, instead we
2648 * return the caller what he specified in nd. In particular,
2649 * if we already turned some pages into mbufs, nd == EAGAIN
2650 * and the main function send them the pages before we come
2651 * here again and block.
2653 if (m->valid != 0 && vm_page_is_valid(m, off & PAGE_MASK, xfsize)) {
2656 VM_OBJECT_WUNLOCK(obj);
2659 } else if (nd != 0) {
2667 * Get the page from backing store.
2671 VM_OBJECT_WUNLOCK(obj);
2672 readahead = sfreadahead * MAXBSIZE;
2675 * Use vn_rdwr() instead of the pager interface for
2676 * the vnode, to allow the read-ahead.
2678 * XXXMAC: Because we don't have fp->f_cred here, we
2679 * pass in NOCRED. This is probably wrong, but is
2680 * consistent with our original implementation.
2682 error = vn_rdwr(UIO_READ, vp, NULL, readahead, trunc_page(off),
2683 UIO_NOCOPY, IO_NODELOCKED | IO_VMIO | ((readahead /
2684 bsize) << IO_SEQSHIFT), td->td_ucred, NOCRED, &resid, td);
2685 SFSTAT_INC(sf_iocnt);
2686 VM_OBJECT_WLOCK(obj);
2688 if (vm_pager_has_page(obj, pindex, NULL, NULL)) {
2689 rv = vm_pager_get_pages(obj, &m, 1, 0);
2690 SFSTAT_INC(sf_iocnt);
2691 m = vm_page_lookup(obj, pindex);
2694 else if (rv != VM_PAGER_OK) {
2703 m->valid = VM_PAGE_BITS_ALL;
2711 } else if (m != NULL) {
2714 vm_page_unwire(m, PQ_INACTIVE);
2717 * See if anyone else might know about this page. If
2718 * not and it is not valid, then free it.
2720 if (m->wire_count == 0 && m->valid == 0 && !vm_page_busied(m))
2724 KASSERT(error != 0 || (m->wire_count > 0 &&
2725 vm_page_is_valid(m, off & PAGE_MASK, xfsize)),
2726 ("wrong page state m %p off %#jx xfsize %d", m, (uintmax_t)off,
2728 VM_OBJECT_WUNLOCK(obj);
2733 sendfile_getobj(struct thread *td, struct file *fp, vm_object_t *obj_res,
2734 struct vnode **vp_res, struct shmfd **shmfd_res, off_t *obj_size,
2740 struct shmfd *shmfd;
2743 vp = *vp_res = NULL;
2745 shmfd = *shmfd_res = NULL;
2749 * The file descriptor must be a regular file and have a
2750 * backing VM object.
2752 if (fp->f_type == DTYPE_VNODE) {
2754 vn_lock(vp, LK_SHARED | LK_RETRY);
2755 if (vp->v_type != VREG) {
2759 *bsize = vp->v_mount->mnt_stat.f_iosize;
2760 error = VOP_GETATTR(vp, &va, td->td_ucred);
2763 *obj_size = va.va_size;
2769 } else if (fp->f_type == DTYPE_SHM) {
2771 obj = shmfd->shm_object;
2772 *obj_size = shmfd->shm_size;
2778 VM_OBJECT_WLOCK(obj);
2779 if ((obj->flags & OBJ_DEAD) != 0) {
2780 VM_OBJECT_WUNLOCK(obj);
2786 * Temporarily increase the backing VM object's reference
2787 * count so that a forced reclamation of its vnode does not
2788 * immediately destroy it.
2790 vm_object_reference_locked(obj);
2791 VM_OBJECT_WUNLOCK(obj);
2803 kern_sendfile_getsock(struct thread *td, int s, struct file **sock_fp,
2806 cap_rights_t rights;
2813 * The socket must be a stream socket and connected.
2815 error = getsock_cap(td->td_proc->p_fd, s, cap_rights_init(&rights,
2816 CAP_SEND), sock_fp, NULL);
2819 *so = (*sock_fp)->f_data;
2820 if ((*so)->so_type != SOCK_STREAM)
2822 if (((*so)->so_state & SS_ISCONNECTED) == 0)
2828 vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
2829 struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
2830 int kflags, struct sendfile_sync *sfs, struct thread *td)
2832 struct file *sock_fp;
2834 struct vm_object *obj;
2839 struct shmfd *shmfd;
2841 off_t off, xfsize, fsbytes, sbytes, rem, obj_size;
2842 int error, bsize, nd, hdrlen, mnw;
2848 fsbytes = sbytes = 0;
2853 error = sendfile_getobj(td, fp, &obj, &vp, &shmfd, &obj_size, &bsize);
2859 error = kern_sendfile_getsock(td, sockfd, &sock_fp, &so);
2864 * Do not wait on memory allocations but return ENOMEM for
2865 * caller to retry later.
2866 * XXX: Experimental.
2868 if (flags & SF_MNOWAIT)
2872 error = mac_socket_check_send(td->td_ucred, so);
2877 /* If headers are specified copy them into mbufs. */
2878 if (hdr_uio != NULL) {
2879 hdr_uio->uio_td = td;
2880 hdr_uio->uio_rw = UIO_WRITE;
2881 if (hdr_uio->uio_resid > 0) {
2883 * In FBSD < 5.0 the nbytes to send also included
2884 * the header. If compat is specified subtract the
2885 * header size from nbytes.
2887 if (kflags & SFK_COMPAT) {
2888 if (nbytes > hdr_uio->uio_resid)
2889 nbytes -= hdr_uio->uio_resid;
2893 m = m_uiotombuf(hdr_uio, (mnw ? M_NOWAIT : M_WAITOK),
2896 error = mnw ? EAGAIN : ENOBUFS;
2899 hdrlen = m_length(m, NULL);
2904 * Protect against multiple writers to the socket.
2906 * XXXRW: Historically this has assumed non-interruptibility, so now
2907 * we implement that, but possibly shouldn't.
2909 (void)sblock(&so->so_snd, SBL_WAIT | SBL_NOINTR);
2912 * Loop through the pages of the file, starting with the requested
2913 * offset. Get a file page (do I/O if necessary), map the file page
2914 * into an sf_buf, attach an mbuf header to the sf_buf, and queue
2916 * This is done in two loops. The inner loop turns as many pages
2917 * as it can, up to available socket buffer space, without blocking
2918 * into mbufs to have it bulk delivered into the socket send buffer.
2919 * The outer loop checks the state and available space of the socket
2920 * and takes care of the overall progress.
2922 for (off = offset; ; ) {
2928 if ((nbytes != 0 && nbytes == fsbytes) ||
2929 (nbytes == 0 && obj_size == fsbytes))
2938 * Check the socket state for ongoing connection,
2939 * no errors and space in socket buffer.
2940 * If space is low allow for the remainder of the
2941 * file to be processed if it fits the socket buffer.
2942 * Otherwise block in waiting for sufficient space
2943 * to proceed, or if the socket is nonblocking, return
2944 * to userland with EAGAIN while reporting how far
2946 * We wait until the socket buffer has significant free
2947 * space to do bulk sends. This makes good use of file
2948 * system read ahead and allows packet segmentation
2949 * offloading hardware to take over lots of work. If
2950 * we were not careful here we would send off only one
2953 SOCKBUF_LOCK(&so->so_snd);
2954 if (so->so_snd.sb_lowat < so->so_snd.sb_hiwat / 2)
2955 so->so_snd.sb_lowat = so->so_snd.sb_hiwat / 2;
2957 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
2959 SOCKBUF_UNLOCK(&so->so_snd);
2961 } else if (so->so_error) {
2962 error = so->so_error;
2964 SOCKBUF_UNLOCK(&so->so_snd);
2967 space = sbspace(&so->so_snd);
2970 space < so->so_snd.sb_lowat)) {
2971 if (so->so_state & SS_NBIO) {
2972 SOCKBUF_UNLOCK(&so->so_snd);
2977 * sbwait drops the lock while sleeping.
2978 * When we loop back to retry_space the
2979 * state may have changed and we retest
2982 error = sbwait(&so->so_snd);
2984 * An error from sbwait usually indicates that we've
2985 * been interrupted by a signal. If we've sent anything
2986 * then return bytes sent, otherwise return the error.
2989 SOCKBUF_UNLOCK(&so->so_snd);
2994 SOCKBUF_UNLOCK(&so->so_snd);
2997 * Reduce space in the socket buffer by the size of
2998 * the header mbuf chain.
2999 * hdrlen is set to 0 after the first loop.
3004 error = vn_lock(vp, LK_SHARED);
3007 error = VOP_GETATTR(vp, &va, td->td_ucred);
3008 if (error != 0 || off >= va.va_size) {
3012 obj_size = va.va_size;
3016 * Loop and construct maximum sized mbuf chain to be bulk
3017 * dumped into socket buffer.
3019 while (space > loopbytes) {
3024 * Calculate the amount to transfer.
3025 * Not to exceed a page, the EOF,
3026 * or the passed in nbytes.
3028 pgoff = (vm_offset_t)(off & PAGE_MASK);
3029 rem = obj_size - offset;
3031 rem = omin(rem, nbytes);
3032 rem -= fsbytes + loopbytes;
3033 xfsize = omin(PAGE_SIZE - pgoff, rem);
3034 xfsize = omin(space - loopbytes, xfsize);
3036 done = 1; /* all data sent */
3041 * Attempt to look up the page. Allocate
3042 * if not found or wait and loop if busy.
3045 nd = EAGAIN; /* send what we already got */
3046 else if ((flags & SF_NODISKIO) != 0)
3050 error = sendfile_readpage(obj, vp, nd, off,
3051 xfsize, bsize, td, &pg);
3053 if (error == EAGAIN)
3054 error = 0; /* not a real error */
3059 * Get a sendfile buf. When allocating the
3060 * first buffer for mbuf chain, we usually
3061 * wait as long as necessary, but this wait
3062 * can be interrupted. For consequent
3063 * buffers, do not sleep, since several
3064 * threads might exhaust the buffers and then
3067 sf = sf_buf_alloc(pg, (mnw || m != NULL) ? SFB_NOWAIT :
3070 SFSTAT_INC(sf_allocfail);
3072 vm_page_unwire(pg, PQ_INACTIVE);
3073 KASSERT(pg->object != NULL,
3074 ("%s: object disappeared", __func__));
3077 error = (mnw ? EAGAIN : EINTR);
3082 * Get an mbuf and set it up as having
3085 m0 = m_get((mnw ? M_NOWAIT : M_WAITOK), MT_DATA);
3087 error = (mnw ? EAGAIN : ENOBUFS);
3088 sf_ext_free(sf, NULL);
3092 * Attach EXT_SFBUF external storage.
3094 m0->m_ext.ext_buf = (caddr_t )sf_buf_kva(sf);
3095 m0->m_ext.ext_size = PAGE_SIZE;
3096 m0->m_ext.ext_arg1 = sf;
3097 m0->m_ext.ext_arg2 = sfs;
3098 m0->m_ext.ext_type = EXT_SFBUF;
3099 m0->m_ext.ext_flags = 0;
3100 m0->m_flags |= (M_EXT|M_RDONLY);
3101 m0->m_data = (char *)sf_buf_kva(sf) + pgoff;
3104 /* Append to mbuf chain. */
3108 m_last(m)->m_next = m0;
3113 /* Keep track of bits processed. */
3114 loopbytes += xfsize;
3118 * XXX eventually this should be a sfsync
3128 /* Add the buffer chain to the socket buffer. */
3132 mlen = m_length(m, NULL);
3133 SOCKBUF_LOCK(&so->so_snd);
3134 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
3136 SOCKBUF_UNLOCK(&so->so_snd);
3139 SOCKBUF_UNLOCK(&so->so_snd);
3140 CURVNET_SET(so->so_vnet);
3141 /* Avoid error aliasing. */
3142 err = (*so->so_proto->pr_usrreqs->pru_send)
3143 (so, 0, m, NULL, NULL, td);
3147 * We need two counters to get the
3148 * file offset and nbytes to send
3150 * - sbytes contains the total amount
3151 * of bytes sent, including headers.
3152 * - fsbytes contains the total amount
3153 * of bytes sent from the file.
3161 } else if (error == 0)
3163 m = NULL; /* pru_send always consumes */
3166 /* Quit outer loop on error or when we're done. */
3174 * Send trailers. Wimp out and use writev(2).
3176 if (trl_uio != NULL) {
3177 sbunlock(&so->so_snd);
3178 error = kern_writev(td, sockfd, trl_uio);
3180 sbytes += td->td_retval[0];
3185 sbunlock(&so->so_snd);
3188 * If there was no error we have to clear td->td_retval[0]
3189 * because it may have been set by writev.
3192 td->td_retval[0] = 0;
3198 vm_object_deallocate(obj);
3204 if (error == ERESTART)
3212 * Functionality only compiled in if SCTP is defined in the kernel Makefile,
3213 * otherwise all return EOPNOTSUPP.
3214 * XXX: We should make this loadable one day.
3217 sys_sctp_peeloff(td, uap)
3219 struct sctp_peeloff_args /* {
3224 #if (defined(INET) || defined(INET6)) && defined(SCTP)
3225 struct file *nfp = NULL;
3226 struct socket *head, *so;
3227 cap_rights_t rights;
3231 AUDIT_ARG_FD(uap->sd);
3232 error = fgetsock(td, uap->sd, cap_rights_init(&rights, CAP_PEELOFF),
3236 if (head->so_proto->pr_protocol != IPPROTO_SCTP) {
3240 error = sctp_can_peel_off(head, (sctp_assoc_t)uap->name);
3244 * At this point we know we do have a assoc to pull
3245 * we proceed to get the fd setup. This may block
3249 error = falloc(td, &nfp, &fd, 0);
3252 td->td_retval[0] = fd;
3254 CURVNET_SET(head->so_vnet);
3255 so = sonewconn(head, SS_ISCONNECTED);
3261 * Before changing the flags on the socket, we have to bump the
3262 * reference count. Otherwise, if the protocol calls sofree(),
3263 * the socket will be released due to a zero refcount.
3266 soref(so); /* file descriptor reference */
3271 TAILQ_REMOVE(&head->so_comp, so, so_list);
3273 so->so_state |= (head->so_state & SS_NBIO);
3274 so->so_state &= ~SS_NOFDREF;
3275 so->so_qstate &= ~SQ_COMP;
3278 finit(nfp, fflag, DTYPE_SOCKET, so, &socketops);
3279 error = sctp_do_peeloff(head, so, (sctp_assoc_t)uap->name);
3282 if (head->so_sigio != NULL)
3283 fsetown(fgetown(&head->so_sigio), &so->so_sigio);
3287 * close the new descriptor, assuming someone hasn't ripped it
3288 * out from under us.
3291 fdclose(td->td_proc->p_fd, nfp, fd, td);
3294 * Release explicitly held references before returning.
3304 return (EOPNOTSUPP);
3309 sys_sctp_generic_sendmsg (td, uap)
3311 struct sctp_generic_sendmsg_args /* {
3317 struct sctp_sndrcvinfo *sinfo,
3321 #if (defined(INET) || defined(INET6)) && defined(SCTP)
3322 struct sctp_sndrcvinfo sinfo, *u_sinfo = NULL;
3324 struct file *fp = NULL;
3325 struct sockaddr *to = NULL;
3327 struct uio *ktruio = NULL;
3330 struct iovec iov[1];
3331 cap_rights_t rights;
3334 if (uap->sinfo != NULL) {
3335 error = copyin(uap->sinfo, &sinfo, sizeof (sinfo));
3341 cap_rights_init(&rights, CAP_SEND);
3342 if (uap->tolen != 0) {
3343 error = getsockaddr(&to, uap->to, uap->tolen);
3348 cap_rights_set(&rights, CAP_CONNECT);
3351 AUDIT_ARG_FD(uap->sd);
3352 error = getsock_cap(td->td_proc->p_fd, uap->sd, &rights, &fp, NULL);
3356 if (to && (KTRPOINT(td, KTR_STRUCT)))
3360 iov[0].iov_base = uap->msg;
3361 iov[0].iov_len = uap->mlen;
3363 so = (struct socket *)fp->f_data;
3364 if (so->so_proto->pr_protocol != IPPROTO_SCTP) {
3369 error = mac_socket_check_send(td->td_ucred, so);
3375 auio.uio_iovcnt = 1;
3376 auio.uio_segflg = UIO_USERSPACE;
3377 auio.uio_rw = UIO_WRITE;
3379 auio.uio_offset = 0; /* XXX */
3381 len = auio.uio_resid = uap->mlen;
3382 CURVNET_SET(so->so_vnet);
3383 error = sctp_lower_sosend(so, to, &auio, (struct mbuf *)NULL,
3384 (struct mbuf *)NULL, uap->flags, u_sinfo, td);
3387 if (auio.uio_resid != len && (error == ERESTART ||
3388 error == EINTR || error == EWOULDBLOCK))
3390 /* Generation of SIGPIPE can be controlled per socket. */
3391 if (error == EPIPE && !(so->so_options & SO_NOSIGPIPE) &&
3392 !(uap->flags & MSG_NOSIGNAL)) {
3393 PROC_LOCK(td->td_proc);
3394 tdsignal(td, SIGPIPE);
3395 PROC_UNLOCK(td->td_proc);
3399 td->td_retval[0] = len - auio.uio_resid;
3401 if (ktruio != NULL) {
3402 ktruio->uio_resid = td->td_retval[0];
3403 ktrgenio(uap->sd, UIO_WRITE, ktruio, error);
3413 return (EOPNOTSUPP);
3418 sys_sctp_generic_sendmsg_iov(td, uap)
3420 struct sctp_generic_sendmsg_iov_args /* {
3426 struct sctp_sndrcvinfo *sinfo,
3430 #if (defined(INET) || defined(INET6)) && defined(SCTP)
3431 struct sctp_sndrcvinfo sinfo, *u_sinfo = NULL;
3433 struct file *fp = NULL;
3434 struct sockaddr *to = NULL;
3436 struct uio *ktruio = NULL;
3439 struct iovec *iov, *tiov;
3440 cap_rights_t rights;
3444 if (uap->sinfo != NULL) {
3445 error = copyin(uap->sinfo, &sinfo, sizeof (sinfo));
3450 cap_rights_init(&rights, CAP_SEND);
3451 if (uap->tolen != 0) {
3452 error = getsockaddr(&to, uap->to, uap->tolen);
3457 cap_rights_set(&rights, CAP_CONNECT);
3460 AUDIT_ARG_FD(uap->sd);
3461 error = getsock_cap(td->td_proc->p_fd, uap->sd, &rights, &fp, NULL);
3465 #ifdef COMPAT_FREEBSD32
3466 if (SV_CURPROC_FLAG(SV_ILP32))
3467 error = freebsd32_copyiniov((struct iovec32 *)uap->iov,
3468 uap->iovlen, &iov, EMSGSIZE);
3471 error = copyiniov(uap->iov, uap->iovlen, &iov, EMSGSIZE);
3475 if (to && (KTRPOINT(td, KTR_STRUCT)))
3479 so = (struct socket *)fp->f_data;
3480 if (so->so_proto->pr_protocol != IPPROTO_SCTP) {
3485 error = mac_socket_check_send(td->td_ucred, so);
3491 auio.uio_iovcnt = uap->iovlen;
3492 auio.uio_segflg = UIO_USERSPACE;
3493 auio.uio_rw = UIO_WRITE;
3495 auio.uio_offset = 0; /* XXX */
3498 for (i = 0; i <uap->iovlen; i++, tiov++) {
3499 if ((auio.uio_resid += tiov->iov_len) < 0) {
3504 len = auio.uio_resid;
3505 CURVNET_SET(so->so_vnet);
3506 error = sctp_lower_sosend(so, to, &auio,
3507 (struct mbuf *)NULL, (struct mbuf *)NULL,
3508 uap->flags, u_sinfo, td);
3511 if (auio.uio_resid != len && (error == ERESTART ||
3512 error == EINTR || error == EWOULDBLOCK))
3514 /* Generation of SIGPIPE can be controlled per socket */
3515 if (error == EPIPE && !(so->so_options & SO_NOSIGPIPE) &&
3516 !(uap->flags & MSG_NOSIGNAL)) {
3517 PROC_LOCK(td->td_proc);
3518 tdsignal(td, SIGPIPE);
3519 PROC_UNLOCK(td->td_proc);
3523 td->td_retval[0] = len - auio.uio_resid;
3525 if (ktruio != NULL) {
3526 ktruio->uio_resid = td->td_retval[0];
3527 ktrgenio(uap->sd, UIO_WRITE, ktruio, error);
3539 return (EOPNOTSUPP);
3544 sys_sctp_generic_recvmsg(td, uap)
3546 struct sctp_generic_recvmsg_args /* {
3550 struct sockaddr *from,
3551 __socklen_t *fromlenaddr,
3552 struct sctp_sndrcvinfo *sinfo,
3556 #if (defined(INET) || defined(INET6)) && defined(SCTP)
3557 uint8_t sockbufstore[256];
3559 struct iovec *iov, *tiov;
3560 struct sctp_sndrcvinfo sinfo;
3562 struct file *fp = NULL;
3563 struct sockaddr *fromsa;
3564 cap_rights_t rights;
3566 struct uio *ktruio = NULL;
3569 int error, fromlen, i, msg_flags;
3571 AUDIT_ARG_FD(uap->sd);
3572 error = getsock_cap(td->td_proc->p_fd, uap->sd,
3573 cap_rights_init(&rights, CAP_RECV), &fp, NULL);
3576 #ifdef COMPAT_FREEBSD32
3577 if (SV_CURPROC_FLAG(SV_ILP32))
3578 error = freebsd32_copyiniov((struct iovec32 *)uap->iov,
3579 uap->iovlen, &iov, EMSGSIZE);
3582 error = copyiniov(uap->iov, uap->iovlen, &iov, EMSGSIZE);
3587 if (so->so_proto->pr_protocol != IPPROTO_SCTP) {
3592 error = mac_socket_check_receive(td->td_ucred, so);
3597 if (uap->fromlenaddr != NULL) {
3598 error = copyin(uap->fromlenaddr, &fromlen, sizeof (fromlen));
3604 if (uap->msg_flags) {
3605 error = copyin(uap->msg_flags, &msg_flags, sizeof (int));
3612 auio.uio_iovcnt = uap->iovlen;
3613 auio.uio_segflg = UIO_USERSPACE;
3614 auio.uio_rw = UIO_READ;
3616 auio.uio_offset = 0; /* XXX */
3619 for (i = 0; i <uap->iovlen; i++, tiov++) {
3620 if ((auio.uio_resid += tiov->iov_len) < 0) {
3625 len = auio.uio_resid;
3626 fromsa = (struct sockaddr *)sockbufstore;
3629 if (KTRPOINT(td, KTR_GENIO))
3630 ktruio = cloneuio(&auio);
3632 memset(&sinfo, 0, sizeof(struct sctp_sndrcvinfo));
3633 CURVNET_SET(so->so_vnet);
3634 error = sctp_sorecvmsg(so, &auio, (struct mbuf **)NULL,
3635 fromsa, fromlen, &msg_flags,
3636 (struct sctp_sndrcvinfo *)&sinfo, 1);
3639 if (auio.uio_resid != len && (error == ERESTART ||
3640 error == EINTR || error == EWOULDBLOCK))
3644 error = copyout(&sinfo, uap->sinfo, sizeof (sinfo));
3647 if (ktruio != NULL) {
3648 ktruio->uio_resid = len - auio.uio_resid;
3649 ktrgenio(uap->sd, UIO_READ, ktruio, error);
3654 td->td_retval[0] = len - auio.uio_resid;
3656 if (fromlen && uap->from) {
3658 if (len <= 0 || fromsa == 0)
3661 len = MIN(len, fromsa->sa_len);
3662 error = copyout(fromsa, uap->from, (size_t)len);
3666 error = copyout(&len, uap->fromlenaddr, sizeof (socklen_t));
3671 if (KTRPOINT(td, KTR_STRUCT))
3672 ktrsockaddr(fromsa);
3674 if (uap->msg_flags) {
3675 error = copyout(&msg_flags, uap->msg_flags, sizeof (int));
3687 return (EOPNOTSUPP);