2 * Copyright (c) 1982, 1986, 1989, 1990, 1993
3 * The Regents of the University of California. All rights reserved.
5 * sendfile(2) and related extensions:
6 * Copyright (c) 1998, David Greenman. All rights reserved.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 4. Neither the name of the University nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * @(#)uipc_syscalls.c 8.4 (Berkeley) 2/21/94
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
38 #include "opt_capsicum.h"
40 #include "opt_inet6.h"
42 #include "opt_compat.h"
43 #include "opt_ktrace.h"
45 #include <sys/param.h>
46 #include <sys/systm.h>
47 #include <sys/capability.h>
48 #include <sys/condvar.h>
49 #include <sys/kernel.h>
51 #include <sys/mutex.h>
52 #include <sys/sysproto.h>
53 #include <sys/malloc.h>
54 #include <sys/filedesc.h>
55 #include <sys/event.h>
57 #include <sys/fcntl.h>
59 #include <sys/filio.h>
62 #include <sys/mount.h>
64 #include <sys/protosw.h>
65 #include <sys/rwlock.h>
66 #include <sys/sf_buf.h>
67 #include <sys/sf_sync.h>
68 #include <sys/sf_base.h>
69 #include <sys/sysent.h>
70 #include <sys/socket.h>
71 #include <sys/socketvar.h>
72 #include <sys/signalvar.h>
73 #include <sys/syscallsubr.h>
74 #include <sys/sysctl.h>
76 #include <sys/vnode.h>
78 #include <sys/ktrace.h>
80 #ifdef COMPAT_FREEBSD32
81 #include <compat/freebsd32/freebsd32_util.h>
86 #include <security/audit/audit.h>
87 #include <security/mac/mac_framework.h>
90 #include <vm/vm_param.h>
91 #include <vm/vm_object.h>
92 #include <vm/vm_page.h>
93 #include <vm/vm_pager.h>
94 #include <vm/vm_kern.h>
95 #include <vm/vm_extern.h>
98 #if defined(INET) || defined(INET6)
100 #include <netinet/sctp.h>
101 #include <netinet/sctp_peeloff.h>
103 #endif /* INET || INET6 */
106 * Flags for accept1() and kern_accept4(), in addition to SOCK_CLOEXEC
109 #define ACCEPT4_INHERIT 0x1
110 #define ACCEPT4_COMPAT 0x2
112 static int sendit(struct thread *td, int s, struct msghdr *mp, int flags);
113 static int recvit(struct thread *td, int s, struct msghdr *mp, void *namelenp);
115 static int accept1(struct thread *td, int s, struct sockaddr *uname,
116 socklen_t *anamelen, int flags);
117 static int do_sendfile(struct thread *td, struct sendfile_args *uap,
119 static int getsockname1(struct thread *td, struct getsockname_args *uap,
121 static int getpeername1(struct thread *td, struct getpeername_args *uap,
124 counter_u64_t sfstat[sizeof(struct sfstat) / sizeof(uint64_t)];
126 static int filt_sfsync_attach(struct knote *kn);
127 static void filt_sfsync_detach(struct knote *kn);
128 static int filt_sfsync(struct knote *kn, long hint);
131 * sendfile(2)-related variables and associated sysctls
133 static SYSCTL_NODE(_kern_ipc, OID_AUTO, sendfile, CTLFLAG_RW, 0,
134 "sendfile(2) tunables");
135 static int sfreadahead = 1;
136 SYSCTL_INT(_kern_ipc_sendfile, OID_AUTO, readahead, CTLFLAG_RW,
137 &sfreadahead, 0, "Number of sendfile(2) read-ahead MAXBSIZE blocks");
140 static int sf_sync_debug = 0;
141 SYSCTL_INT(_debug, OID_AUTO, sf_sync_debug, CTLFLAG_RW,
142 &sf_sync_debug, 0, "Output debugging during sf_sync lifecycle");
143 #define SFSYNC_DPRINTF(s, ...) \
146 printf((s), ##__VA_ARGS__); \
149 #define SFSYNC_DPRINTF(c, ...)
152 static uma_zone_t zone_sfsync;
154 static struct filterops sendfile_filtops = {
156 .f_attach = filt_sfsync_attach,
157 .f_detach = filt_sfsync_detach,
158 .f_event = filt_sfsync,
162 sfstat_init(const void *unused)
165 COUNTER_ARRAY_ALLOC(sfstat, sizeof(struct sfstat) / sizeof(uint64_t),
168 SYSINIT(sfstat, SI_SUB_MBUF, SI_ORDER_FIRST, sfstat_init, NULL);
171 sf_sync_init(const void *unused)
174 zone_sfsync = uma_zcreate("sendfile_sync", sizeof(struct sendfile_sync),
179 kqueue_add_filteropts(EVFILT_SENDFILE, &sendfile_filtops);
181 SYSINIT(sf_sync, SI_SUB_MBUF, SI_ORDER_FIRST, sf_sync_init, NULL);
184 sfstat_sysctl(SYSCTL_HANDLER_ARGS)
188 COUNTER_ARRAY_COPY(sfstat, &s, sizeof(s) / sizeof(uint64_t));
190 COUNTER_ARRAY_ZERO(sfstat, sizeof(s) / sizeof(uint64_t));
191 return (SYSCTL_OUT(req, &s, sizeof(s)));
193 SYSCTL_PROC(_kern_ipc, OID_AUTO, sfstat, CTLTYPE_OPAQUE | CTLFLAG_RW,
194 NULL, 0, sfstat_sysctl, "I", "sendfile statistics");
197 * Convert a user file descriptor to a kernel file entry and check if required
198 * capability rights are present.
199 * A reference on the file entry is held upon returning.
202 getsock_cap(struct filedesc *fdp, int fd, cap_rights_t *rightsp,
203 struct file **fpp, u_int *fflagp)
208 error = fget_unlocked(fdp, fd, rightsp, 0, &fp, NULL);
211 if (fp->f_type != DTYPE_SOCKET) {
212 fdrop(fp, curthread);
216 *fflagp = fp->f_flag;
222 * System call interface to the socket abstraction.
224 #if defined(COMPAT_43)
225 #define COMPAT_OLDSOCK
231 struct socket_args /* {
239 int fd, error, type, oflag, fflag;
241 AUDIT_ARG_SOCKET(uap->domain, uap->type, uap->protocol);
246 if ((type & SOCK_CLOEXEC) != 0) {
247 type &= ~SOCK_CLOEXEC;
250 if ((type & SOCK_NONBLOCK) != 0) {
251 type &= ~SOCK_NONBLOCK;
256 error = mac_socket_check_create(td->td_ucred, uap->domain, type,
261 error = falloc(td, &fp, &fd, oflag);
264 /* An extra reference on `fp' has been held for us by falloc(). */
265 error = socreate(uap->domain, &so, type, uap->protocol,
268 fdclose(td->td_proc->p_fd, fp, fd, td);
270 finit(fp, FREAD | FWRITE | fflag, DTYPE_SOCKET, so, &socketops);
271 if ((fflag & FNONBLOCK) != 0)
272 (void) fo_ioctl(fp, FIONBIO, &fflag, td->td_ucred, td);
273 td->td_retval[0] = fd;
283 struct bind_args /* {
292 error = getsockaddr(&sa, uap->name, uap->namelen);
294 error = kern_bind(td, uap->s, sa);
301 kern_bindat(struct thread *td, int dirfd, int fd, struct sockaddr *sa)
309 AUDIT_ARG_SOCKADDR(td, dirfd, sa);
310 error = getsock_cap(td->td_proc->p_fd, fd,
311 cap_rights_init(&rights, CAP_BIND), &fp, NULL);
316 if (KTRPOINT(td, KTR_STRUCT))
320 error = mac_socket_check_bind(td->td_ucred, so, sa);
323 if (dirfd == AT_FDCWD)
324 error = sobind(so, sa, td);
326 error = sobindat(dirfd, so, sa, td);
335 kern_bind(struct thread *td, int fd, struct sockaddr *sa)
338 return (kern_bindat(td, AT_FDCWD, fd, sa));
345 struct bindat_args /* {
355 error = getsockaddr(&sa, uap->name, uap->namelen);
357 error = kern_bindat(td, uap->fd, uap->s, sa);
367 struct listen_args /* {
377 AUDIT_ARG_FD(uap->s);
378 error = getsock_cap(td->td_proc->p_fd, uap->s,
379 cap_rights_init(&rights, CAP_LISTEN), &fp, NULL);
383 error = mac_socket_check_listen(td->td_ucred, so);
386 error = solisten(so, uap->backlog, td);
396 accept1(td, s, uname, anamelen, flags)
399 struct sockaddr *uname;
403 struct sockaddr *name;
409 return (kern_accept4(td, s, NULL, NULL, flags, NULL));
411 error = copyin(anamelen, &namelen, sizeof (namelen));
415 error = kern_accept4(td, s, &name, &namelen, flags, &fp);
418 * return a namelen of zero for older code which might
419 * ignore the return value from accept.
422 (void) copyout(&namelen, anamelen, sizeof(*anamelen));
426 if (error == 0 && uname != NULL) {
427 #ifdef COMPAT_OLDSOCK
428 if (flags & ACCEPT4_COMPAT)
429 ((struct osockaddr *)name)->sa_family =
432 error = copyout(name, uname, namelen);
435 error = copyout(&namelen, anamelen,
438 fdclose(td->td_proc->p_fd, fp, td->td_retval[0], td);
440 free(name, M_SONAME);
445 kern_accept(struct thread *td, int s, struct sockaddr **name,
446 socklen_t *namelen, struct file **fp)
448 return (kern_accept4(td, s, name, namelen, ACCEPT4_INHERIT, fp));
452 kern_accept4(struct thread *td, int s, struct sockaddr **name,
453 socklen_t *namelen, int flags, struct file **fp)
455 struct filedesc *fdp;
456 struct file *headfp, *nfp = NULL;
457 struct sockaddr *sa = NULL;
458 struct socket *head, *so;
468 fdp = td->td_proc->p_fd;
469 error = getsock_cap(fdp, s, cap_rights_init(&rights, CAP_ACCEPT),
473 head = headfp->f_data;
474 if ((head->so_options & SO_ACCEPTCONN) == 0) {
479 error = mac_socket_check_accept(td->td_ucred, head);
483 error = falloc(td, &nfp, &fd, (flags & SOCK_CLOEXEC) ? O_CLOEXEC : 0);
487 if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->so_comp)) {
492 while (TAILQ_EMPTY(&head->so_comp) && head->so_error == 0) {
493 if (head->so_rcv.sb_state & SBS_CANTRCVMORE) {
494 head->so_error = ECONNABORTED;
497 error = msleep(&head->so_timeo, &accept_mtx, PSOCK | PCATCH,
504 if (head->so_error) {
505 error = head->so_error;
510 so = TAILQ_FIRST(&head->so_comp);
511 KASSERT(!(so->so_qstate & SQ_INCOMP), ("accept1: so SQ_INCOMP"));
512 KASSERT(so->so_qstate & SQ_COMP, ("accept1: so not SQ_COMP"));
515 * Before changing the flags on the socket, we have to bump the
516 * reference count. Otherwise, if the protocol calls sofree(),
517 * the socket will be released due to a zero refcount.
519 SOCK_LOCK(so); /* soref() and so_state update */
520 soref(so); /* file descriptor reference */
522 TAILQ_REMOVE(&head->so_comp, so, so_list);
524 if (flags & ACCEPT4_INHERIT)
525 so->so_state |= (head->so_state & SS_NBIO);
527 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0;
528 so->so_qstate &= ~SQ_COMP;
534 /* An extra reference on `nfp' has been held for us by falloc(). */
535 td->td_retval[0] = fd;
537 /* connection has been removed from the listen queue */
538 KNOTE_UNLOCKED(&head->so_rcv.sb_sel.si_note, 0);
540 if (flags & ACCEPT4_INHERIT) {
541 pgid = fgetown(&head->so_sigio);
543 fsetown(pgid, &so->so_sigio);
545 fflag &= ~(FNONBLOCK | FASYNC);
546 if (flags & SOCK_NONBLOCK)
550 finit(nfp, fflag, DTYPE_SOCKET, so, &socketops);
551 /* Sync socket nonblocking/async state with file flags */
552 tmp = fflag & FNONBLOCK;
553 (void) fo_ioctl(nfp, FIONBIO, &tmp, td->td_ucred, td);
554 tmp = fflag & FASYNC;
555 (void) fo_ioctl(nfp, FIOASYNC, &tmp, td->td_ucred, td);
557 error = soaccept(so, &sa);
560 * return a namelen of zero for older code which might
561 * ignore the return value from accept.
572 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, sa);
574 /* check sa_len before it is destroyed */
575 if (*namelen > sa->sa_len)
576 *namelen = sa->sa_len;
578 if (KTRPOINT(td, KTR_STRUCT))
588 * close the new descriptor, assuming someone hasn't ripped it
592 fdclose(fdp, nfp, fd, td);
595 * Release explicitly held references before returning. We return
596 * a reference on nfp to the caller on success if they request it.
615 struct accept_args *uap;
618 return (accept1(td, uap->s, uap->name, uap->anamelen, ACCEPT4_INHERIT));
624 struct accept4_args *uap;
627 if (uap->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
630 return (accept1(td, uap->s, uap->name, uap->anamelen, uap->flags));
633 #ifdef COMPAT_OLDSOCK
637 struct accept_args *uap;
640 return (accept1(td, uap->s, uap->name, uap->anamelen,
641 ACCEPT4_INHERIT | ACCEPT4_COMPAT));
643 #endif /* COMPAT_OLDSOCK */
649 struct connect_args /* {
658 error = getsockaddr(&sa, uap->name, uap->namelen);
660 error = kern_connect(td, uap->s, sa);
667 kern_connectat(struct thread *td, int dirfd, int fd, struct sockaddr *sa)
672 int error, interrupted = 0;
675 AUDIT_ARG_SOCKADDR(td, dirfd, sa);
676 error = getsock_cap(td->td_proc->p_fd, fd,
677 cap_rights_init(&rights, CAP_CONNECT), &fp, NULL);
681 if (so->so_state & SS_ISCONNECTING) {
686 if (KTRPOINT(td, KTR_STRUCT))
690 error = mac_socket_check_connect(td->td_ucred, so, sa);
694 if (dirfd == AT_FDCWD)
695 error = soconnect(so, sa, td);
697 error = soconnectat(dirfd, so, sa, td);
700 if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) {
705 while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
706 error = msleep(&so->so_timeo, SOCK_MTX(so), PSOCK | PCATCH,
709 if (error == EINTR || error == ERESTART)
715 error = so->so_error;
721 so->so_state &= ~SS_ISCONNECTING;
722 if (error == ERESTART)
730 kern_connect(struct thread *td, int fd, struct sockaddr *sa)
733 return (kern_connectat(td, AT_FDCWD, fd, sa));
738 sys_connectat(td, uap)
740 struct connectat_args /* {
750 error = getsockaddr(&sa, uap->name, uap->namelen);
752 error = kern_connectat(td, uap->fd, uap->s, sa);
759 kern_socketpair(struct thread *td, int domain, int type, int protocol,
762 struct filedesc *fdp = td->td_proc->p_fd;
763 struct file *fp1, *fp2;
764 struct socket *so1, *so2;
765 int fd, error, oflag, fflag;
767 AUDIT_ARG_SOCKET(domain, type, protocol);
771 if ((type & SOCK_CLOEXEC) != 0) {
772 type &= ~SOCK_CLOEXEC;
775 if ((type & SOCK_NONBLOCK) != 0) {
776 type &= ~SOCK_NONBLOCK;
780 /* We might want to have a separate check for socket pairs. */
781 error = mac_socket_check_create(td->td_ucred, domain, type,
786 error = socreate(domain, &so1, type, protocol, td->td_ucred, td);
789 error = socreate(domain, &so2, type, protocol, td->td_ucred, td);
792 /* On success extra reference to `fp1' and 'fp2' is set by falloc. */
793 error = falloc(td, &fp1, &fd, oflag);
797 fp1->f_data = so1; /* so1 already has ref count */
798 error = falloc(td, &fp2, &fd, oflag);
801 fp2->f_data = so2; /* so2 already has ref count */
803 error = soconnect2(so1, so2);
806 if (type == SOCK_DGRAM) {
808 * Datagram socket connection is asymmetric.
810 error = soconnect2(so2, so1);
814 finit(fp1, FREAD | FWRITE | fflag, DTYPE_SOCKET, fp1->f_data,
816 finit(fp2, FREAD | FWRITE | fflag, DTYPE_SOCKET, fp2->f_data,
818 if ((fflag & FNONBLOCK) != 0) {
819 (void) fo_ioctl(fp1, FIONBIO, &fflag, td->td_ucred, td);
820 (void) fo_ioctl(fp2, FIONBIO, &fflag, td->td_ucred, td);
826 fdclose(fdp, fp2, rsv[1], td);
829 fdclose(fdp, fp1, rsv[0], td);
841 sys_socketpair(struct thread *td, struct socketpair_args *uap)
845 error = kern_socketpair(td, uap->domain, uap->type,
849 error = copyout(sv, uap->rsv, 2 * sizeof(int));
851 (void)kern_close(td, sv[0]);
852 (void)kern_close(td, sv[1]);
858 sendit(td, s, mp, flags)
864 struct mbuf *control;
868 #ifdef CAPABILITY_MODE
869 if (IN_CAPABILITY_MODE(td) && (mp->msg_name != NULL))
873 if (mp->msg_name != NULL) {
874 error = getsockaddr(&to, mp->msg_name, mp->msg_namelen);
884 if (mp->msg_control) {
885 if (mp->msg_controllen < sizeof(struct cmsghdr)
886 #ifdef COMPAT_OLDSOCK
887 && mp->msg_flags != MSG_COMPAT
893 error = sockargs(&control, mp->msg_control,
894 mp->msg_controllen, MT_CONTROL);
897 #ifdef COMPAT_OLDSOCK
898 if (mp->msg_flags == MSG_COMPAT) {
901 M_PREPEND(control, sizeof(*cm), M_WAITOK);
902 cm = mtod(control, struct cmsghdr *);
903 cm->cmsg_len = control->m_len;
904 cm->cmsg_level = SOL_SOCKET;
905 cm->cmsg_type = SCM_RIGHTS;
912 error = kern_sendit(td, s, mp, flags, control, UIO_USERSPACE);
920 kern_sendit(td, s, mp, flags, control, segflg)
925 struct mbuf *control;
934 struct uio *ktruio = NULL;
940 cap_rights_init(&rights, CAP_SEND);
941 if (mp->msg_name != NULL) {
942 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, mp->msg_name);
943 cap_rights_set(&rights, CAP_CONNECT);
945 error = getsock_cap(td->td_proc->p_fd, s, &rights, &fp, NULL);
948 so = (struct socket *)fp->f_data;
951 if (mp->msg_name != NULL && KTRPOINT(td, KTR_STRUCT))
952 ktrsockaddr(mp->msg_name);
955 if (mp->msg_name != NULL) {
956 error = mac_socket_check_connect(td->td_ucred, so,
961 error = mac_socket_check_send(td->td_ucred, so);
966 auio.uio_iov = mp->msg_iov;
967 auio.uio_iovcnt = mp->msg_iovlen;
968 auio.uio_segflg = segflg;
969 auio.uio_rw = UIO_WRITE;
971 auio.uio_offset = 0; /* XXX */
974 for (i = 0; i < mp->msg_iovlen; i++, iov++) {
975 if ((auio.uio_resid += iov->iov_len) < 0) {
981 if (KTRPOINT(td, KTR_GENIO))
982 ktruio = cloneuio(&auio);
984 len = auio.uio_resid;
985 error = sosend(so, mp->msg_name, &auio, 0, control, flags, td);
987 if (auio.uio_resid != len && (error == ERESTART ||
988 error == EINTR || error == EWOULDBLOCK))
990 /* Generation of SIGPIPE can be controlled per socket */
991 if (error == EPIPE && !(so->so_options & SO_NOSIGPIPE) &&
992 !(flags & MSG_NOSIGNAL)) {
993 PROC_LOCK(td->td_proc);
994 tdsignal(td, SIGPIPE);
995 PROC_UNLOCK(td->td_proc);
999 td->td_retval[0] = len - auio.uio_resid;
1001 if (ktruio != NULL) {
1002 ktruio->uio_resid = td->td_retval[0];
1003 ktrgenio(s, UIO_WRITE, ktruio, error);
1014 struct sendto_args /* {
1026 msg.msg_name = uap->to;
1027 msg.msg_namelen = uap->tolen;
1028 msg.msg_iov = &aiov;
1030 msg.msg_control = 0;
1031 #ifdef COMPAT_OLDSOCK
1034 aiov.iov_base = uap->buf;
1035 aiov.iov_len = uap->len;
1036 return (sendit(td, uap->s, &msg, uap->flags));
1039 #ifdef COMPAT_OLDSOCK
1043 struct osend_args /* {
1054 msg.msg_namelen = 0;
1055 msg.msg_iov = &aiov;
1057 aiov.iov_base = uap->buf;
1058 aiov.iov_len = uap->len;
1059 msg.msg_control = 0;
1061 return (sendit(td, uap->s, &msg, uap->flags));
1067 struct osendmsg_args /* {
1077 error = copyin(uap->msg, &msg, sizeof (struct omsghdr));
1080 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE);
1084 msg.msg_flags = MSG_COMPAT;
1085 error = sendit(td, uap->s, &msg, uap->flags);
1092 sys_sendmsg(td, uap)
1094 struct sendmsg_args /* {
1104 error = copyin(uap->msg, &msg, sizeof (msg));
1107 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE);
1111 #ifdef COMPAT_OLDSOCK
1114 error = sendit(td, uap->s, &msg, uap->flags);
1120 kern_recvit(td, s, mp, fromseg, controlp)
1124 enum uio_seg fromseg;
1125 struct mbuf **controlp;
1129 struct mbuf *m, *control = NULL;
1133 struct sockaddr *fromsa = NULL;
1134 cap_rights_t rights;
1136 struct uio *ktruio = NULL;
1141 if (controlp != NULL)
1145 error = getsock_cap(td->td_proc->p_fd, s,
1146 cap_rights_init(&rights, CAP_RECV), &fp, NULL);
1152 error = mac_socket_check_receive(td->td_ucred, so);
1159 auio.uio_iov = mp->msg_iov;
1160 auio.uio_iovcnt = mp->msg_iovlen;
1161 auio.uio_segflg = UIO_USERSPACE;
1162 auio.uio_rw = UIO_READ;
1164 auio.uio_offset = 0; /* XXX */
1167 for (i = 0; i < mp->msg_iovlen; i++, iov++) {
1168 if ((auio.uio_resid += iov->iov_len) < 0) {
1174 if (KTRPOINT(td, KTR_GENIO))
1175 ktruio = cloneuio(&auio);
1177 len = auio.uio_resid;
1178 error = soreceive(so, &fromsa, &auio, NULL,
1179 (mp->msg_control || controlp) ? &control : NULL,
1182 if (auio.uio_resid != len && (error == ERESTART ||
1183 error == EINTR || error == EWOULDBLOCK))
1187 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, fromsa);
1189 if (ktruio != NULL) {
1190 ktruio->uio_resid = len - auio.uio_resid;
1191 ktrgenio(s, UIO_READ, ktruio, error);
1196 td->td_retval[0] = len - auio.uio_resid;
1198 len = mp->msg_namelen;
1199 if (len <= 0 || fromsa == NULL)
1202 /* save sa_len before it is destroyed by MSG_COMPAT */
1203 len = MIN(len, fromsa->sa_len);
1204 #ifdef COMPAT_OLDSOCK
1205 if (mp->msg_flags & MSG_COMPAT)
1206 ((struct osockaddr *)fromsa)->sa_family =
1209 if (fromseg == UIO_USERSPACE) {
1210 error = copyout(fromsa, mp->msg_name,
1215 bcopy(fromsa, mp->msg_name, len);
1217 mp->msg_namelen = len;
1219 if (mp->msg_control && controlp == NULL) {
1220 #ifdef COMPAT_OLDSOCK
1222 * We assume that old recvmsg calls won't receive access
1223 * rights and other control info, esp. as control info
1224 * is always optional and those options didn't exist in 4.3.
1225 * If we receive rights, trim the cmsghdr; anything else
1228 if (control && mp->msg_flags & MSG_COMPAT) {
1229 if (mtod(control, struct cmsghdr *)->cmsg_level !=
1231 mtod(control, struct cmsghdr *)->cmsg_type !=
1233 mp->msg_controllen = 0;
1236 control->m_len -= sizeof (struct cmsghdr);
1237 control->m_data += sizeof (struct cmsghdr);
1240 len = mp->msg_controllen;
1242 mp->msg_controllen = 0;
1243 ctlbuf = mp->msg_control;
1245 while (m && len > 0) {
1246 unsigned int tocopy;
1248 if (len >= m->m_len)
1251 mp->msg_flags |= MSG_CTRUNC;
1255 if ((error = copyout(mtod(m, caddr_t),
1256 ctlbuf, tocopy)) != 0)
1263 mp->msg_controllen = ctlbuf - (caddr_t)mp->msg_control;
1268 if (fromsa && KTRPOINT(td, KTR_STRUCT))
1269 ktrsockaddr(fromsa);
1271 free(fromsa, M_SONAME);
1273 if (error == 0 && controlp != NULL)
1274 *controlp = control;
1282 recvit(td, s, mp, namelenp)
1290 error = kern_recvit(td, s, mp, UIO_USERSPACE, NULL);
1293 if (namelenp != NULL) {
1294 error = copyout(&mp->msg_namelen, namelenp, sizeof (socklen_t));
1295 #ifdef COMPAT_OLDSOCK
1296 if (mp->msg_flags & MSG_COMPAT)
1297 error = 0; /* old recvfrom didn't check */
1304 sys_recvfrom(td, uap)
1306 struct recvfrom_args /* {
1311 struct sockaddr * __restrict from;
1312 socklen_t * __restrict fromlenaddr;
1319 if (uap->fromlenaddr) {
1320 error = copyin(uap->fromlenaddr,
1321 &msg.msg_namelen, sizeof (msg.msg_namelen));
1325 msg.msg_namelen = 0;
1327 msg.msg_name = uap->from;
1328 msg.msg_iov = &aiov;
1330 aiov.iov_base = uap->buf;
1331 aiov.iov_len = uap->len;
1332 msg.msg_control = 0;
1333 msg.msg_flags = uap->flags;
1334 error = recvit(td, uap->s, &msg, uap->fromlenaddr);
1339 #ifdef COMPAT_OLDSOCK
1343 struct recvfrom_args *uap;
1346 uap->flags |= MSG_COMPAT;
1347 return (sys_recvfrom(td, uap));
1351 #ifdef COMPAT_OLDSOCK
1355 struct orecv_args /* {
1366 msg.msg_namelen = 0;
1367 msg.msg_iov = &aiov;
1369 aiov.iov_base = uap->buf;
1370 aiov.iov_len = uap->len;
1371 msg.msg_control = 0;
1372 msg.msg_flags = uap->flags;
1373 return (recvit(td, uap->s, &msg, NULL));
1377 * Old recvmsg. This code takes advantage of the fact that the old msghdr
1378 * overlays the new one, missing only the flags, and with the (old) access
1379 * rights where the control fields are now.
1384 struct orecvmsg_args /* {
1386 struct omsghdr *msg;
1394 error = copyin(uap->msg, &msg, sizeof (struct omsghdr));
1397 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE);
1400 msg.msg_flags = uap->flags | MSG_COMPAT;
1402 error = recvit(td, uap->s, &msg, &uap->msg->msg_namelen);
1403 if (msg.msg_controllen && error == 0)
1404 error = copyout(&msg.msg_controllen,
1405 &uap->msg->msg_accrightslen, sizeof (int));
1412 sys_recvmsg(td, uap)
1414 struct recvmsg_args /* {
1421 struct iovec *uiov, *iov;
1424 error = copyin(uap->msg, &msg, sizeof (msg));
1427 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE);
1430 msg.msg_flags = uap->flags;
1431 #ifdef COMPAT_OLDSOCK
1432 msg.msg_flags &= ~MSG_COMPAT;
1436 error = recvit(td, uap->s, &msg, NULL);
1439 error = copyout(&msg, uap->msg, sizeof(msg));
1447 sys_shutdown(td, uap)
1449 struct shutdown_args /* {
1456 cap_rights_t rights;
1459 AUDIT_ARG_FD(uap->s);
1460 error = getsock_cap(td->td_proc->p_fd, uap->s,
1461 cap_rights_init(&rights, CAP_SHUTDOWN), &fp, NULL);
1464 error = soshutdown(so, uap->how);
1472 sys_setsockopt(td, uap)
1474 struct setsockopt_args /* {
1483 return (kern_setsockopt(td, uap->s, uap->level, uap->name,
1484 uap->val, UIO_USERSPACE, uap->valsize));
1488 kern_setsockopt(td, s, level, name, val, valseg, valsize)
1494 enum uio_seg valseg;
1499 struct sockopt sopt;
1500 cap_rights_t rights;
1503 if (val == NULL && valsize != 0)
1505 if ((int)valsize < 0)
1508 sopt.sopt_dir = SOPT_SET;
1509 sopt.sopt_level = level;
1510 sopt.sopt_name = name;
1511 sopt.sopt_val = val;
1512 sopt.sopt_valsize = valsize;
1518 sopt.sopt_td = NULL;
1521 panic("kern_setsockopt called with bad valseg");
1525 error = getsock_cap(td->td_proc->p_fd, s,
1526 cap_rights_init(&rights, CAP_SETSOCKOPT), &fp, NULL);
1529 error = sosetopt(so, &sopt);
1537 sys_getsockopt(td, uap)
1539 struct getsockopt_args /* {
1543 void * __restrict val;
1544 socklen_t * __restrict avalsize;
1551 error = copyin(uap->avalsize, &valsize, sizeof (valsize));
1556 error = kern_getsockopt(td, uap->s, uap->level, uap->name,
1557 uap->val, UIO_USERSPACE, &valsize);
1560 error = copyout(&valsize, uap->avalsize, sizeof (valsize));
1565 * Kernel version of getsockopt.
1566 * optval can be a userland or userspace. optlen is always a kernel pointer.
1569 kern_getsockopt(td, s, level, name, val, valseg, valsize)
1575 enum uio_seg valseg;
1580 struct sockopt sopt;
1581 cap_rights_t rights;
1586 if ((int)*valsize < 0)
1589 sopt.sopt_dir = SOPT_GET;
1590 sopt.sopt_level = level;
1591 sopt.sopt_name = name;
1592 sopt.sopt_val = val;
1593 sopt.sopt_valsize = (size_t)*valsize; /* checked non-negative above */
1599 sopt.sopt_td = NULL;
1602 panic("kern_getsockopt called with bad valseg");
1606 error = getsock_cap(td->td_proc->p_fd, s,
1607 cap_rights_init(&rights, CAP_GETSOCKOPT), &fp, NULL);
1610 error = sogetopt(so, &sopt);
1611 *valsize = sopt.sopt_valsize;
1618 * getsockname1() - Get socket name.
1622 getsockname1(td, uap, compat)
1624 struct getsockname_args /* {
1626 struct sockaddr * __restrict asa;
1627 socklen_t * __restrict alen;
1631 struct sockaddr *sa;
1635 error = copyin(uap->alen, &len, sizeof(len));
1639 error = kern_getsockname(td, uap->fdes, &sa, &len);
1644 #ifdef COMPAT_OLDSOCK
1646 ((struct osockaddr *)sa)->sa_family = sa->sa_family;
1648 error = copyout(sa, uap->asa, (u_int)len);
1652 error = copyout(&len, uap->alen, sizeof(len));
1657 kern_getsockname(struct thread *td, int fd, struct sockaddr **sa,
1662 cap_rights_t rights;
1667 error = getsock_cap(td->td_proc->p_fd, fd,
1668 cap_rights_init(&rights, CAP_GETSOCKNAME), &fp, NULL);
1673 CURVNET_SET(so->so_vnet);
1674 error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, sa);
1681 len = MIN(*alen, (*sa)->sa_len);
1684 if (KTRPOINT(td, KTR_STRUCT))
1689 if (error != 0 && *sa != NULL) {
1690 free(*sa, M_SONAME);
1697 sys_getsockname(td, uap)
1699 struct getsockname_args *uap;
1702 return (getsockname1(td, uap, 0));
1705 #ifdef COMPAT_OLDSOCK
1707 ogetsockname(td, uap)
1709 struct getsockname_args *uap;
1712 return (getsockname1(td, uap, 1));
1714 #endif /* COMPAT_OLDSOCK */
1717 * getpeername1() - Get name of peer for connected socket.
1721 getpeername1(td, uap, compat)
1723 struct getpeername_args /* {
1725 struct sockaddr * __restrict asa;
1726 socklen_t * __restrict alen;
1730 struct sockaddr *sa;
1734 error = copyin(uap->alen, &len, sizeof (len));
1738 error = kern_getpeername(td, uap->fdes, &sa, &len);
1743 #ifdef COMPAT_OLDSOCK
1745 ((struct osockaddr *)sa)->sa_family = sa->sa_family;
1747 error = copyout(sa, uap->asa, (u_int)len);
1751 error = copyout(&len, uap->alen, sizeof(len));
1756 kern_getpeername(struct thread *td, int fd, struct sockaddr **sa,
1761 cap_rights_t rights;
1766 error = getsock_cap(td->td_proc->p_fd, fd,
1767 cap_rights_init(&rights, CAP_GETPEERNAME), &fp, NULL);
1771 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
1776 CURVNET_SET(so->so_vnet);
1777 error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, sa);
1784 len = MIN(*alen, (*sa)->sa_len);
1787 if (KTRPOINT(td, KTR_STRUCT))
1791 if (error != 0 && *sa != NULL) {
1792 free(*sa, M_SONAME);
1801 sys_getpeername(td, uap)
1803 struct getpeername_args *uap;
1806 return (getpeername1(td, uap, 0));
1809 #ifdef COMPAT_OLDSOCK
1811 ogetpeername(td, uap)
1813 struct ogetpeername_args *uap;
1816 /* XXX uap should have type `getpeername_args *' to begin with. */
1817 return (getpeername1(td, (struct getpeername_args *)uap, 1));
1819 #endif /* COMPAT_OLDSOCK */
1822 sockargs(mp, buf, buflen, type)
1827 struct sockaddr *sa;
1831 if (buflen > MLEN) {
1832 #ifdef COMPAT_OLDSOCK
1833 if (type == MT_SONAME && buflen <= 112)
1834 buflen = MLEN; /* unix domain compat. hack */
1837 if (buflen > MCLBYTES)
1840 m = m_get2(buflen, M_WAITOK, type, 0);
1842 error = copyin(buf, mtod(m, caddr_t), (u_int)buflen);
1847 if (type == MT_SONAME) {
1848 sa = mtod(m, struct sockaddr *);
1850 #if defined(COMPAT_OLDSOCK) && BYTE_ORDER != BIG_ENDIAN
1851 if (sa->sa_family == 0 && sa->sa_len < AF_MAX)
1852 sa->sa_family = sa->sa_len;
1854 sa->sa_len = buflen;
1861 getsockaddr(namp, uaddr, len)
1862 struct sockaddr **namp;
1866 struct sockaddr *sa;
1869 if (len > SOCK_MAXADDRLEN)
1870 return (ENAMETOOLONG);
1871 if (len < offsetof(struct sockaddr, sa_data[0]))
1873 sa = malloc(len, M_SONAME, M_WAITOK);
1874 error = copyin(uaddr, sa, len);
1878 #if defined(COMPAT_OLDSOCK) && BYTE_ORDER != BIG_ENDIAN
1879 if (sa->sa_family == 0 && sa->sa_len < AF_MAX)
1880 sa->sa_family = sa->sa_len;
1889 filt_sfsync_attach(struct knote *kn)
1891 struct sendfile_sync *sfs = (struct sendfile_sync *) kn->kn_sdata;
1892 struct knlist *knl = &sfs->klist;
1894 SFSYNC_DPRINTF("%s: kn=%p, sfs=%p\n", __func__, kn, sfs);
1897 * Validate that we actually received this via the kernel API.
1899 if ((kn->kn_flags & EV_FLAG1) == 0)
1902 kn->kn_ptr.p_v = sfs;
1903 kn->kn_flags &= ~EV_FLAG1;
1905 knl->kl_lock(knl->kl_lockarg);
1907 * If we're in the "freeing" state,
1908 * don't allow the add. That way we don't
1909 * end up racing with some other thread that
1910 * is trying to finish some setup.
1912 if (sfs->state == SF_STATE_FREEING) {
1913 knl->kl_unlock(knl->kl_lockarg);
1916 knlist_add(&sfs->klist, kn, 1);
1917 knl->kl_unlock(knl->kl_lockarg);
1923 * Called when a knote is being detached.
1926 filt_sfsync_detach(struct knote *kn)
1929 struct sendfile_sync *sfs;
1932 sfs = kn->kn_ptr.p_v;
1935 SFSYNC_DPRINTF("%s: kn=%p, sfs=%p\n", __func__, kn, sfs);
1937 knl->kl_lock(knl->kl_lockarg);
1938 if (!knlist_empty(knl))
1939 knlist_remove(knl, kn, 1);
1942 * If the list is empty _AND_ the refcount is 0
1943 * _AND_ we've finished the setup phase and now
1944 * we're in the running phase, we can free the
1945 * underlying sendfile_sync.
1947 * But we shouldn't do it before finishing the
1948 * underlying divorce from the knote.
1950 * So, we have the sfsync lock held; transition
1951 * it to "freeing", then unlock, then free
1954 if (knlist_empty(knl)) {
1955 if (sfs->state == SF_STATE_COMPLETED && sfs->count == 0) {
1956 SFSYNC_DPRINTF("%s: (%llu) sfs=%p; completed, "
1957 "count==0, empty list: time to free!\n",
1959 (unsigned long long) curthread->td_tid,
1961 sf_sync_set_state(sfs, SF_STATE_FREEING, 1);
1965 knl->kl_unlock(knl->kl_lockarg);
1968 * Only call free if we're the one who has transitioned things
1969 * to free. Otherwise we could race with another thread that
1970 * is currently tearing things down.
1973 SFSYNC_DPRINTF("%s: (%llu) sfs=%p, %s:%d\n",
1975 (unsigned long long) curthread->td_tid,
1984 filt_sfsync(struct knote *kn, long hint)
1986 struct sendfile_sync *sfs = (struct sendfile_sync *) kn->kn_ptr.p_v;
1989 SFSYNC_DPRINTF("%s: kn=%p, sfs=%p\n", __func__, kn, sfs);
1992 * XXX add a lock assertion here!
1994 ret = (sfs->count == 0 && sfs->state == SF_STATE_COMPLETED);
2001 * Detach mapped page and release resources back to the system.
2004 sf_buf_mext(struct mbuf *mb, void *addr, void *args)
2007 struct sendfile_sync *sfs;
2009 m = sf_buf_page(args);
2012 vm_page_unwire(m, 0);
2014 * Check for the object going away on us. This can
2015 * happen since we don't hold a reference to it.
2016 * If so, we're responsible for freeing the page.
2018 if (m->wire_count == 0 && m->object == NULL)
2026 * sfs may be invalid at this point, don't use it!
2028 return (EXT_FREE_OK);
2032 * Called to remove a reference to a sf_sync object.
2034 * This is generally done during the mbuf free path to signify
2035 * that one of the mbufs in the transaction has been completed.
2037 * If we're doing SF_SYNC and the refcount is zero then we'll wake
2040 * IF we're doing SF_KQUEUE and the refcount is zero then we'll
2041 * fire off the knote.
2044 sf_sync_deref(struct sendfile_sync *sfs)
2051 mtx_lock(&sfs->mtx);
2052 KASSERT(sfs->count> 0, ("Sendfile sync botchup count == 0"));
2056 * Only fire off the wakeup / kqueue notification if
2057 * we are in the running state.
2059 if (sfs->count == 0 && sfs->state == SF_STATE_COMPLETED) {
2060 if (sfs->flags & SF_SYNC)
2061 cv_signal(&sfs->cv);
2063 if (sfs->flags & SF_KQUEUE) {
2064 SFSYNC_DPRINTF("%s: (%llu) sfs=%p: knote!\n",
2066 (unsigned long long) curthread->td_tid,
2068 KNOTE_LOCKED(&sfs->klist, 1);
2072 * If we're not waiting around for a sync,
2073 * check if the knote list is empty.
2074 * If it is, we transition to free.
2076 * XXX I think it's about time I added some state
2077 * or flag that says whether we're supposed to be
2078 * waiting around until we've done a signal.
2080 * XXX Ie, the reason that I don't free it here
2081 * is because the caller will free the last reference,
2082 * not us. That should be codified in some flag
2083 * that indicates "self-free" rather than checking
2084 * for SF_SYNC all the time.
2086 if ((sfs->flags & SF_SYNC) == 0 && knlist_empty(&sfs->klist)) {
2087 SFSYNC_DPRINTF("%s: (%llu) sfs=%p; completed, "
2088 "count==0, empty list: time to free!\n",
2090 (unsigned long long) curthread->td_tid,
2092 sf_sync_set_state(sfs, SF_STATE_FREEING, 1);
2097 mtx_unlock(&sfs->mtx);
2100 * Attempt to do a free here.
2102 * We do this outside of the lock because it may destroy the
2103 * lock in question as it frees things. We can optimise this
2106 * XXX yes, we should make it a requirement to hold the
2107 * lock across sf_sync_free().
2110 SFSYNC_DPRINTF("%s: (%llu) sfs=%p\n",
2112 (unsigned long long) curthread->td_tid,
2119 * Allocate a sendfile_sync state structure.
2121 * For now this only knows about the "sleep" sync, but later it will
2122 * grow various other personalities.
2124 struct sendfile_sync *
2125 sf_sync_alloc(uint32_t flags)
2127 struct sendfile_sync *sfs;
2129 sfs = uma_zalloc(zone_sfsync, M_WAITOK | M_ZERO);
2130 mtx_init(&sfs->mtx, "sendfile", NULL, MTX_DEF);
2131 cv_init(&sfs->cv, "sendfile");
2133 sfs->state = SF_STATE_SETUP;
2134 knlist_init_mtx(&sfs->klist, &sfs->mtx);
2136 SFSYNC_DPRINTF("%s: sfs=%p, flags=0x%08x\n", __func__, sfs, sfs->flags);
2142 * Take a reference to a sfsync instance.
2144 * This has to map 1:1 to free calls coming in via sf_buf_mext(),
2145 * so typically this will be referenced once for each mbuf allocated.
2148 sf_sync_ref(struct sendfile_sync *sfs)
2154 mtx_lock(&sfs->mtx);
2156 mtx_unlock(&sfs->mtx);
2160 sf_sync_syscall_wait(struct sendfile_sync *sfs)
2166 KASSERT(mtx_owned(&sfs->mtx), ("%s: sfs=%p: not locked but should be!",
2171 * If we're not requested to wait during the syscall,
2172 * don't bother waiting.
2174 if ((sfs->flags & SF_SYNC) == 0)
2178 * This is a bit suboptimal and confusing, so bear with me.
2180 * Ideally sf_sync_syscall_wait() will wait until
2181 * all pending mbuf transmit operations are done.
2182 * This means that when sendfile becomes async, it'll
2183 * run in the background and will transition from
2184 * RUNNING to COMPLETED when it's finished acquiring
2185 * new things to send. Then, when the mbufs finish
2186 * sending, COMPLETED + sfs->count == 0 is enough to
2187 * know that no further work is being done.
2189 * So, we will sleep on both RUNNING and COMPLETED.
2190 * It's up to the (in progress) async sendfile loop
2191 * to transition the sf_sync from RUNNING to
2192 * COMPLETED so the wakeup above will actually
2193 * do the cv_signal() call.
2195 if (sfs->state != SF_STATE_COMPLETED && sfs->state != SF_STATE_RUNNING)
2198 if (sfs->count != 0)
2199 cv_wait(&sfs->cv, &sfs->mtx);
2200 KASSERT(sfs->count == 0, ("sendfile sync still busy"));
2207 * Free an sf_sync if it's appropriate to.
2210 sf_sync_free(struct sendfile_sync *sfs)
2216 SFSYNC_DPRINTF("%s: (%lld) sfs=%p; called; state=%d, flags=0x%08x "
2219 (long long) curthread->td_tid,
2225 mtx_lock(&sfs->mtx);
2228 * We keep the sf_sync around if the state is active,
2229 * we are doing kqueue notification and we have active
2232 * If the caller wants to free us right this second it
2233 * should transition this to the freeing state.
2235 * So, complain loudly if they break this rule.
2237 if (sfs->state != SF_STATE_FREEING) {
2238 printf("%s: (%llu) sfs=%p; not freeing; let's wait!\n",
2240 (unsigned long long) curthread->td_tid,
2242 mtx_unlock(&sfs->mtx);
2246 KASSERT(sfs->count == 0, ("sendfile sync still busy"));
2247 cv_destroy(&sfs->cv);
2249 * This doesn't call knlist_detach() on each knote; it just frees
2252 knlist_delete(&sfs->klist, curthread, 1);
2253 mtx_destroy(&sfs->mtx);
2254 SFSYNC_DPRINTF("%s: (%llu) sfs=%p; freeing\n",
2256 (unsigned long long) curthread->td_tid,
2258 uma_zfree(zone_sfsync, sfs);
2262 * Setup a sf_sync to post a kqueue notification when things are complete.
2265 sf_sync_kqueue_setup(struct sendfile_sync *sfs, struct sf_hdtr_kq *sfkq)
2270 sfs->flags |= SF_KQUEUE;
2272 /* Check the flags are valid */
2273 if ((sfkq->kq_flags & ~(EV_CLEAR | EV_DISPATCH | EV_ONESHOT)) != 0)
2276 SFSYNC_DPRINTF("%s: sfs=%p: kqfd=%d, flags=0x%08x, ident=%p, udata=%p\n",
2281 (void *) sfkq->kq_ident,
2282 (void *) sfkq->kq_udata);
2284 /* Setup and register a knote on the given kqfd. */
2285 kev.ident = (uintptr_t) sfkq->kq_ident;
2286 kev.filter = EVFILT_SENDFILE;
2287 kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1 | sfkq->kq_flags;
2288 kev.data = (intptr_t) sfs;
2289 kev.udata = sfkq->kq_udata;
2291 error = kqfd_register(sfkq->kq_fd, &kev, curthread, 1);
2293 SFSYNC_DPRINTF("%s: returned %d\n", __func__, error);
2299 sf_sync_set_state(struct sendfile_sync *sfs, sendfile_sync_state_t state,
2302 sendfile_sync_state_t old_state;
2305 mtx_lock(&sfs->mtx);
2308 * Update our current state.
2310 old_state = sfs->state;
2312 SFSYNC_DPRINTF("%s: (%llu) sfs=%p; going from %d to %d\n",
2314 (unsigned long long) curthread->td_tid,
2320 * If we're transitioning from RUNNING to COMPLETED and the count is
2321 * zero, then post the knote. The caller may have completed the
2322 * send before we updated the state to COMPLETED and we need to make
2323 * sure this is communicated.
2325 if (old_state == SF_STATE_RUNNING
2326 && state == SF_STATE_COMPLETED
2328 && sfs->flags & SF_KQUEUE) {
2329 SFSYNC_DPRINTF("%s: (%llu) sfs=%p: triggering knote!\n",
2331 (unsigned long long) curthread->td_tid,
2333 KNOTE_LOCKED(&sfs->klist, 1);
2337 mtx_unlock(&sfs->mtx);
2341 * Set the retval/errno for the given transaction.
2343 * This will eventually/ideally be used when the KNOTE is fired off
2344 * to signify the completion of this transaction.
2346 * The sfsync lock should be held before entering this function.
2349 sf_sync_set_retval(struct sendfile_sync *sfs, off_t retval, int xerrno)
2352 KASSERT(mtx_owned(&sfs->mtx), ("%s: sfs=%p: not locked but should be!",
2356 SFSYNC_DPRINTF("%s: (%llu) sfs=%p: errno=%d, retval=%jd\n",
2358 (unsigned long long) curthread->td_tid,
2363 sfs->retval = retval;
2364 sfs->xerrno = xerrno;
2370 * int sendfile(int fd, int s, off_t offset, size_t nbytes,
2371 * struct sf_hdtr *hdtr, off_t *sbytes, int flags)
2373 * Send a file specified by 'fd' and starting at 'offset' to a socket
2374 * specified by 's'. Send only 'nbytes' of the file or until EOF if nbytes ==
2375 * 0. Optionally add a header and/or trailer to the socket output. If
2376 * specified, write the total number of bytes sent into *sbytes.
2379 sys_sendfile(struct thread *td, struct sendfile_args *uap)
2382 return (do_sendfile(td, uap, 0));
2386 _do_sendfile(struct thread *td, int src_fd, int sock_fd, int flags,
2387 int compat, off_t offset, size_t nbytes, off_t *sbytes,
2388 struct uio *hdr_uio,
2389 struct uio *trl_uio, struct sf_hdtr_kq *hdtr_kq)
2391 cap_rights_t rights;
2392 struct sendfile_sync *sfs = NULL;
2398 AUDIT_ARG_FD(src_fd);
2400 if (hdtr_kq != NULL)
2404 * sendfile(2) can start at any offset within a file so we require
2405 * CAP_READ+CAP_SEEK = CAP_PREAD.
2407 if ((error = fget_read(td, src_fd,
2408 cap_rights_init(&rights, CAP_PREAD), &fp)) != 0) {
2413 * IF SF_KQUEUE is set but we haven't copied in anything for
2414 * kqueue data, error out.
2416 if (flags & SF_KQUEUE && do_kqueue == 0) {
2417 SFSYNC_DPRINTF("%s: SF_KQUEUE but no KQUEUE data!\n", __func__);
2422 * If we need to wait for completion, initialise the sfsync
2425 if (flags & (SF_SYNC | SF_KQUEUE))
2426 sfs = sf_sync_alloc(flags & (SF_SYNC | SF_KQUEUE));
2428 if (flags & SF_KQUEUE) {
2429 error = sf_sync_kqueue_setup(sfs, hdtr_kq);
2431 SFSYNC_DPRINTF("%s: (%llu) error; sfs=%p\n",
2433 (unsigned long long) curthread->td_tid,
2435 sf_sync_set_state(sfs, SF_STATE_FREEING, 0);
2442 * Do the sendfile call.
2444 * If this fails, it'll free the mbuf chain which will free up the
2445 * sendfile_sync references.
2447 error = fo_sendfile(fp, sock_fd, hdr_uio, trl_uio, offset,
2448 nbytes, sbytes, flags, compat ? SFK_COMPAT : 0, sfs, td);
2451 * If the sendfile call succeeded, transition the sf_sync state
2452 * to RUNNING, then COMPLETED.
2454 * If the sendfile call failed, then the sendfile call may have
2455 * actually sent some data first - so we check to see whether
2456 * any data was sent. If some data was queued (ie, count > 0)
2457 * then we can't call free; we have to wait until the partial
2458 * transaction completes before we continue along.
2460 * This has the side effect of firing off the knote
2461 * if the refcount has hit zero by the time we get here.
2464 mtx_lock(&sfs->mtx);
2465 if (error == 0 || sfs->count > 0) {
2467 * When it's time to do async sendfile, the transition
2468 * to RUNNING signifies that we're actually actively
2469 * adding and completing mbufs. When the last disk
2470 * buffer is read (ie, when we're not doing any
2471 * further read IO and all subsequent stuff is mbuf
2472 * transmissions) we'll transition to COMPLETED
2473 * and when the final mbuf is freed, the completion
2476 sf_sync_set_state(sfs, SF_STATE_RUNNING, 1);
2479 * Set the retval before we signal completed.
2480 * If we do it the other way around then transitioning to
2481 * COMPLETED may post the knote before you set the return
2484 * XXX for now, errno is always 0, as we don't post
2485 * knotes if sendfile failed. Maybe that'll change later.
2487 sf_sync_set_retval(sfs, *sbytes, error);
2490 * And now transition to completed, which will kick off
2491 * the knote if required.
2493 sf_sync_set_state(sfs, SF_STATE_COMPLETED, 1);
2496 * Error isn't zero, sfs_count is zero, so we
2497 * won't have some other thing to wake things up.
2500 sf_sync_set_state(sfs, SF_STATE_FREEING, 1);
2505 * Next - wait if appropriate.
2507 sf_sync_syscall_wait(sfs);
2510 * If we're not doing kqueue notifications, we can
2511 * transition this immediately to the freeing state.
2513 if ((sfs->flags & SF_KQUEUE) == 0) {
2514 sf_sync_set_state(sfs, SF_STATE_FREEING, 1);
2518 mtx_unlock(&sfs->mtx);
2522 * If do_free is set, free here.
2524 * If we're doing no-kqueue notification and it's just sleep notification,
2525 * we also do free; it's the only chance we have.
2527 if (sfs != NULL && do_free == 1) {
2532 * XXX Should we wait until the send has completed before freeing the source
2533 * file handle? It's the previous behaviour, sure, but is it required?
2534 * We've wired down the page references after all.
2545 do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
2547 struct sf_hdtr hdtr;
2548 struct sf_hdtr_kq hdtr_kq;
2549 struct uio *hdr_uio, *trl_uio;
2555 * File offset must be positive. If it goes beyond EOF
2556 * we send only the header/trailer and no payload data.
2558 if (uap->offset < 0)
2561 hdr_uio = trl_uio = NULL;
2563 if (uap->hdtr != NULL) {
2564 error = copyin(uap->hdtr, &hdtr, sizeof(hdtr));
2567 if (hdtr.headers != NULL) {
2568 error = copyinuio(hdtr.headers, hdtr.hdr_cnt, &hdr_uio);
2572 if (hdtr.trailers != NULL) {
2573 error = copyinuio(hdtr.trailers, hdtr.trl_cnt, &trl_uio);
2579 * If SF_KQUEUE is set, then we need to also copy in
2580 * the kqueue data after the normal hdtr set and set
2583 if (uap->flags & SF_KQUEUE) {
2584 error = copyin(((char *) uap->hdtr) + sizeof(hdtr),
2594 error = _do_sendfile(td, uap->fd, uap->s, uap->flags, compat,
2595 uap->offset, uap->nbytes, &sbytes, hdr_uio, trl_uio, &hdtr_kq);
2597 if (uap->sbytes != NULL) {
2598 copyout(&sbytes, uap->sbytes, sizeof(off_t));
2601 free(hdr_uio, M_IOV);
2602 free(trl_uio, M_IOV);
2606 #ifdef COMPAT_FREEBSD4
2608 freebsd4_sendfile(struct thread *td, struct freebsd4_sendfile_args *uap)
2610 struct sendfile_args args;
2614 args.offset = uap->offset;
2615 args.nbytes = uap->nbytes;
2616 args.hdtr = uap->hdtr;
2617 args.sbytes = uap->sbytes;
2618 args.flags = uap->flags;
2620 return (do_sendfile(td, &args, 1));
2622 #endif /* COMPAT_FREEBSD4 */
2625 sendfile_readpage(vm_object_t obj, struct vnode *vp, int nd,
2626 off_t off, int xfsize, int bsize, struct thread *td, vm_page_t *res)
2631 int error, readahead, rv;
2633 pindex = OFF_TO_IDX(off);
2634 VM_OBJECT_WLOCK(obj);
2635 m = vm_page_grab(obj, pindex, (vp != NULL ? VM_ALLOC_NOBUSY |
2636 VM_ALLOC_IGN_SBUSY : 0) | VM_ALLOC_WIRED | VM_ALLOC_NORMAL);
2639 * Check if page is valid for what we need, otherwise initiate I/O.
2641 * The non-zero nd argument prevents disk I/O, instead we
2642 * return the caller what he specified in nd. In particular,
2643 * if we already turned some pages into mbufs, nd == EAGAIN
2644 * and the main function send them the pages before we come
2645 * here again and block.
2647 if (m->valid != 0 && vm_page_is_valid(m, off & PAGE_MASK, xfsize)) {
2650 VM_OBJECT_WUNLOCK(obj);
2653 } else if (nd != 0) {
2661 * Get the page from backing store.
2665 VM_OBJECT_WUNLOCK(obj);
2666 readahead = sfreadahead * MAXBSIZE;
2669 * Use vn_rdwr() instead of the pager interface for
2670 * the vnode, to allow the read-ahead.
2672 * XXXMAC: Because we don't have fp->f_cred here, we
2673 * pass in NOCRED. This is probably wrong, but is
2674 * consistent with our original implementation.
2676 error = vn_rdwr(UIO_READ, vp, NULL, readahead, trunc_page(off),
2677 UIO_NOCOPY, IO_NODELOCKED | IO_VMIO | ((readahead /
2678 bsize) << IO_SEQSHIFT), td->td_ucred, NOCRED, &resid, td);
2679 SFSTAT_INC(sf_iocnt);
2680 VM_OBJECT_WLOCK(obj);
2682 if (vm_pager_has_page(obj, pindex, NULL, NULL)) {
2683 rv = vm_pager_get_pages(obj, &m, 1, 0);
2684 SFSTAT_INC(sf_iocnt);
2685 m = vm_page_lookup(obj, pindex);
2688 else if (rv != VM_PAGER_OK) {
2697 m->valid = VM_PAGE_BITS_ALL;
2705 } else if (m != NULL) {
2708 vm_page_unwire(m, 0);
2711 * See if anyone else might know about this page. If
2712 * not and it is not valid, then free it.
2714 if (m->wire_count == 0 && m->valid == 0 && !vm_page_busied(m))
2718 KASSERT(error != 0 || (m->wire_count > 0 &&
2719 vm_page_is_valid(m, off & PAGE_MASK, xfsize)),
2720 ("wrong page state m %p off %#jx xfsize %d", m, (uintmax_t)off,
2722 VM_OBJECT_WUNLOCK(obj);
2727 sendfile_getobj(struct thread *td, struct file *fp, vm_object_t *obj_res,
2728 struct vnode **vp_res, struct shmfd **shmfd_res, off_t *obj_size,
2734 struct shmfd *shmfd;
2737 vp = *vp_res = NULL;
2739 shmfd = *shmfd_res = NULL;
2743 * The file descriptor must be a regular file and have a
2744 * backing VM object.
2746 if (fp->f_type == DTYPE_VNODE) {
2748 vn_lock(vp, LK_SHARED | LK_RETRY);
2749 if (vp->v_type != VREG) {
2753 *bsize = vp->v_mount->mnt_stat.f_iosize;
2754 error = VOP_GETATTR(vp, &va, td->td_ucred);
2757 *obj_size = va.va_size;
2763 } else if (fp->f_type == DTYPE_SHM) {
2765 obj = shmfd->shm_object;
2766 *obj_size = shmfd->shm_size;
2772 VM_OBJECT_WLOCK(obj);
2773 if ((obj->flags & OBJ_DEAD) != 0) {
2774 VM_OBJECT_WUNLOCK(obj);
2780 * Temporarily increase the backing VM object's reference
2781 * count so that a forced reclamation of its vnode does not
2782 * immediately destroy it.
2784 vm_object_reference_locked(obj);
2785 VM_OBJECT_WUNLOCK(obj);
2797 kern_sendfile_getsock(struct thread *td, int s, struct file **sock_fp,
2800 cap_rights_t rights;
2807 * The socket must be a stream socket and connected.
2809 error = getsock_cap(td->td_proc->p_fd, s, cap_rights_init(&rights,
2810 CAP_SEND), sock_fp, NULL);
2813 *so = (*sock_fp)->f_data;
2814 if ((*so)->so_type != SOCK_STREAM)
2816 if (((*so)->so_state & SS_ISCONNECTED) == 0)
2822 vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
2823 struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
2824 int kflags, struct sendfile_sync *sfs, struct thread *td)
2826 struct file *sock_fp;
2828 struct vm_object *obj;
2833 struct shmfd *shmfd;
2835 off_t off, xfsize, fsbytes, sbytes, rem, obj_size;
2836 int error, bsize, nd, hdrlen, mnw;
2842 fsbytes = sbytes = 0;
2847 error = sendfile_getobj(td, fp, &obj, &vp, &shmfd, &obj_size, &bsize);
2853 error = kern_sendfile_getsock(td, sockfd, &sock_fp, &so);
2858 * Do not wait on memory allocations but return ENOMEM for
2859 * caller to retry later.
2860 * XXX: Experimental.
2862 if (flags & SF_MNOWAIT)
2866 error = mac_socket_check_send(td->td_ucred, so);
2871 /* If headers are specified copy them into mbufs. */
2872 if (hdr_uio != NULL) {
2873 hdr_uio->uio_td = td;
2874 hdr_uio->uio_rw = UIO_WRITE;
2875 if (hdr_uio->uio_resid > 0) {
2877 * In FBSD < 5.0 the nbytes to send also included
2878 * the header. If compat is specified subtract the
2879 * header size from nbytes.
2881 if (kflags & SFK_COMPAT) {
2882 if (nbytes > hdr_uio->uio_resid)
2883 nbytes -= hdr_uio->uio_resid;
2887 m = m_uiotombuf(hdr_uio, (mnw ? M_NOWAIT : M_WAITOK),
2890 error = mnw ? EAGAIN : ENOBUFS;
2893 hdrlen = m_length(m, NULL);
2898 * Protect against multiple writers to the socket.
2900 * XXXRW: Historically this has assumed non-interruptibility, so now
2901 * we implement that, but possibly shouldn't.
2903 (void)sblock(&so->so_snd, SBL_WAIT | SBL_NOINTR);
2906 * Loop through the pages of the file, starting with the requested
2907 * offset. Get a file page (do I/O if necessary), map the file page
2908 * into an sf_buf, attach an mbuf header to the sf_buf, and queue
2910 * This is done in two loops. The inner loop turns as many pages
2911 * as it can, up to available socket buffer space, without blocking
2912 * into mbufs to have it bulk delivered into the socket send buffer.
2913 * The outer loop checks the state and available space of the socket
2914 * and takes care of the overall progress.
2916 for (off = offset; ; ) {
2922 if ((nbytes != 0 && nbytes == fsbytes) ||
2923 (nbytes == 0 && obj_size == fsbytes))
2932 * Check the socket state for ongoing connection,
2933 * no errors and space in socket buffer.
2934 * If space is low allow for the remainder of the
2935 * file to be processed if it fits the socket buffer.
2936 * Otherwise block in waiting for sufficient space
2937 * to proceed, or if the socket is nonblocking, return
2938 * to userland with EAGAIN while reporting how far
2940 * We wait until the socket buffer has significant free
2941 * space to do bulk sends. This makes good use of file
2942 * system read ahead and allows packet segmentation
2943 * offloading hardware to take over lots of work. If
2944 * we were not careful here we would send off only one
2947 SOCKBUF_LOCK(&so->so_snd);
2948 if (so->so_snd.sb_lowat < so->so_snd.sb_hiwat / 2)
2949 so->so_snd.sb_lowat = so->so_snd.sb_hiwat / 2;
2951 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
2953 SOCKBUF_UNLOCK(&so->so_snd);
2955 } else if (so->so_error) {
2956 error = so->so_error;
2958 SOCKBUF_UNLOCK(&so->so_snd);
2961 space = sbspace(&so->so_snd);
2964 space < so->so_snd.sb_lowat)) {
2965 if (so->so_state & SS_NBIO) {
2966 SOCKBUF_UNLOCK(&so->so_snd);
2971 * sbwait drops the lock while sleeping.
2972 * When we loop back to retry_space the
2973 * state may have changed and we retest
2976 error = sbwait(&so->so_snd);
2978 * An error from sbwait usually indicates that we've
2979 * been interrupted by a signal. If we've sent anything
2980 * then return bytes sent, otherwise return the error.
2983 SOCKBUF_UNLOCK(&so->so_snd);
2988 SOCKBUF_UNLOCK(&so->so_snd);
2991 * Reduce space in the socket buffer by the size of
2992 * the header mbuf chain.
2993 * hdrlen is set to 0 after the first loop.
2998 error = vn_lock(vp, LK_SHARED);
3001 error = VOP_GETATTR(vp, &va, td->td_ucred);
3002 if (error != 0 || off >= va.va_size) {
3006 obj_size = va.va_size;
3010 * Loop and construct maximum sized mbuf chain to be bulk
3011 * dumped into socket buffer.
3013 while (space > loopbytes) {
3018 * Calculate the amount to transfer.
3019 * Not to exceed a page, the EOF,
3020 * or the passed in nbytes.
3022 pgoff = (vm_offset_t)(off & PAGE_MASK);
3023 rem = obj_size - offset;
3025 rem = omin(rem, nbytes);
3026 rem -= fsbytes + loopbytes;
3027 xfsize = omin(PAGE_SIZE - pgoff, rem);
3028 xfsize = omin(space - loopbytes, xfsize);
3030 done = 1; /* all data sent */
3035 * Attempt to look up the page. Allocate
3036 * if not found or wait and loop if busy.
3039 nd = EAGAIN; /* send what we already got */
3040 else if ((flags & SF_NODISKIO) != 0)
3044 error = sendfile_readpage(obj, vp, nd, off,
3045 xfsize, bsize, td, &pg);
3047 if (error == EAGAIN)
3048 error = 0; /* not a real error */
3053 * Get a sendfile buf. When allocating the
3054 * first buffer for mbuf chain, we usually
3055 * wait as long as necessary, but this wait
3056 * can be interrupted. For consequent
3057 * buffers, do not sleep, since several
3058 * threads might exhaust the buffers and then
3061 sf = sf_buf_alloc(pg, (mnw || m != NULL) ? SFB_NOWAIT :
3064 SFSTAT_INC(sf_allocfail);
3066 vm_page_unwire(pg, 0);
3067 KASSERT(pg->object != NULL,
3068 ("%s: object disappeared", __func__));
3071 error = (mnw ? EAGAIN : EINTR);
3076 * Get an mbuf and set it up as having
3079 m0 = m_get((mnw ? M_NOWAIT : M_WAITOK), MT_DATA);
3081 error = (mnw ? EAGAIN : ENOBUFS);
3082 (void)sf_buf_mext(NULL, NULL, sf);
3085 if (m_extadd(m0, (caddr_t )sf_buf_kva(sf), PAGE_SIZE,
3086 sf_buf_mext, sfs, sf, M_RDONLY, EXT_SFBUF,
3087 (mnw ? M_NOWAIT : M_WAITOK)) != 0) {
3088 error = (mnw ? EAGAIN : ENOBUFS);
3089 (void)sf_buf_mext(NULL, NULL, sf);
3093 m0->m_data = (char *)sf_buf_kva(sf) + pgoff;
3096 /* Append to mbuf chain. */
3100 m_last(m)->m_next = m0;
3105 /* Keep track of bits processed. */
3106 loopbytes += xfsize;
3110 * XXX eventually this should be a sfsync
3120 /* Add the buffer chain to the socket buffer. */
3124 mlen = m_length(m, NULL);
3125 SOCKBUF_LOCK(&so->so_snd);
3126 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
3128 SOCKBUF_UNLOCK(&so->so_snd);
3131 SOCKBUF_UNLOCK(&so->so_snd);
3132 CURVNET_SET(so->so_vnet);
3133 /* Avoid error aliasing. */
3134 err = (*so->so_proto->pr_usrreqs->pru_send)
3135 (so, 0, m, NULL, NULL, td);
3139 * We need two counters to get the
3140 * file offset and nbytes to send
3142 * - sbytes contains the total amount
3143 * of bytes sent, including headers.
3144 * - fsbytes contains the total amount
3145 * of bytes sent from the file.
3153 } else if (error == 0)
3155 m = NULL; /* pru_send always consumes */
3158 /* Quit outer loop on error or when we're done. */
3166 * Send trailers. Wimp out and use writev(2).
3168 if (trl_uio != NULL) {
3169 sbunlock(&so->so_snd);
3170 error = kern_writev(td, sockfd, trl_uio);
3172 sbytes += td->td_retval[0];
3177 sbunlock(&so->so_snd);
3180 * If there was no error we have to clear td->td_retval[0]
3181 * because it may have been set by writev.
3184 td->td_retval[0] = 0;
3190 vm_object_deallocate(obj);
3196 if (error == ERESTART)
3204 * Functionality only compiled in if SCTP is defined in the kernel Makefile,
3205 * otherwise all return EOPNOTSUPP.
3206 * XXX: We should make this loadable one day.
3209 sys_sctp_peeloff(td, uap)
3211 struct sctp_peeloff_args /* {
3216 #if (defined(INET) || defined(INET6)) && defined(SCTP)
3217 struct file *nfp = NULL;
3218 struct socket *head, *so;
3219 cap_rights_t rights;
3223 AUDIT_ARG_FD(uap->sd);
3224 error = fgetsock(td, uap->sd, cap_rights_init(&rights, CAP_PEELOFF),
3228 if (head->so_proto->pr_protocol != IPPROTO_SCTP) {
3232 error = sctp_can_peel_off(head, (sctp_assoc_t)uap->name);
3236 * At this point we know we do have a assoc to pull
3237 * we proceed to get the fd setup. This may block
3241 error = falloc(td, &nfp, &fd, 0);
3244 td->td_retval[0] = fd;
3246 CURVNET_SET(head->so_vnet);
3247 so = sonewconn(head, SS_ISCONNECTED);
3253 * Before changing the flags on the socket, we have to bump the
3254 * reference count. Otherwise, if the protocol calls sofree(),
3255 * the socket will be released due to a zero refcount.
3258 soref(so); /* file descriptor reference */
3263 TAILQ_REMOVE(&head->so_comp, so, so_list);
3265 so->so_state |= (head->so_state & SS_NBIO);
3266 so->so_state &= ~SS_NOFDREF;
3267 so->so_qstate &= ~SQ_COMP;
3270 finit(nfp, fflag, DTYPE_SOCKET, so, &socketops);
3271 error = sctp_do_peeloff(head, so, (sctp_assoc_t)uap->name);
3274 if (head->so_sigio != NULL)
3275 fsetown(fgetown(&head->so_sigio), &so->so_sigio);
3279 * close the new descriptor, assuming someone hasn't ripped it
3280 * out from under us.
3283 fdclose(td->td_proc->p_fd, nfp, fd, td);
3286 * Release explicitly held references before returning.
3296 return (EOPNOTSUPP);
3301 sys_sctp_generic_sendmsg (td, uap)
3303 struct sctp_generic_sendmsg_args /* {
3309 struct sctp_sndrcvinfo *sinfo,
3313 #if (defined(INET) || defined(INET6)) && defined(SCTP)
3314 struct sctp_sndrcvinfo sinfo, *u_sinfo = NULL;
3316 struct file *fp = NULL;
3317 struct sockaddr *to = NULL;
3319 struct uio *ktruio = NULL;
3322 struct iovec iov[1];
3323 cap_rights_t rights;
3326 if (uap->sinfo != NULL) {
3327 error = copyin(uap->sinfo, &sinfo, sizeof (sinfo));
3333 cap_rights_init(&rights, CAP_SEND);
3334 if (uap->tolen != 0) {
3335 error = getsockaddr(&to, uap->to, uap->tolen);
3340 cap_rights_set(&rights, CAP_CONNECT);
3343 AUDIT_ARG_FD(uap->sd);
3344 error = getsock_cap(td->td_proc->p_fd, uap->sd, &rights, &fp, NULL);
3348 if (to && (KTRPOINT(td, KTR_STRUCT)))
3352 iov[0].iov_base = uap->msg;
3353 iov[0].iov_len = uap->mlen;
3355 so = (struct socket *)fp->f_data;
3356 if (so->so_proto->pr_protocol != IPPROTO_SCTP) {
3361 error = mac_socket_check_send(td->td_ucred, so);
3367 auio.uio_iovcnt = 1;
3368 auio.uio_segflg = UIO_USERSPACE;
3369 auio.uio_rw = UIO_WRITE;
3371 auio.uio_offset = 0; /* XXX */
3373 len = auio.uio_resid = uap->mlen;
3374 CURVNET_SET(so->so_vnet);
3375 error = sctp_lower_sosend(so, to, &auio, (struct mbuf *)NULL,
3376 (struct mbuf *)NULL, uap->flags, u_sinfo, td);
3379 if (auio.uio_resid != len && (error == ERESTART ||
3380 error == EINTR || error == EWOULDBLOCK))
3382 /* Generation of SIGPIPE can be controlled per socket. */
3383 if (error == EPIPE && !(so->so_options & SO_NOSIGPIPE) &&
3384 !(uap->flags & MSG_NOSIGNAL)) {
3385 PROC_LOCK(td->td_proc);
3386 tdsignal(td, SIGPIPE);
3387 PROC_UNLOCK(td->td_proc);
3391 td->td_retval[0] = len - auio.uio_resid;
3393 if (ktruio != NULL) {
3394 ktruio->uio_resid = td->td_retval[0];
3395 ktrgenio(uap->sd, UIO_WRITE, ktruio, error);
3405 return (EOPNOTSUPP);
3410 sys_sctp_generic_sendmsg_iov(td, uap)
3412 struct sctp_generic_sendmsg_iov_args /* {
3418 struct sctp_sndrcvinfo *sinfo,
3422 #if (defined(INET) || defined(INET6)) && defined(SCTP)
3423 struct sctp_sndrcvinfo sinfo, *u_sinfo = NULL;
3425 struct file *fp = NULL;
3426 struct sockaddr *to = NULL;
3428 struct uio *ktruio = NULL;
3431 struct iovec *iov, *tiov;
3432 cap_rights_t rights;
3436 if (uap->sinfo != NULL) {
3437 error = copyin(uap->sinfo, &sinfo, sizeof (sinfo));
3442 cap_rights_init(&rights, CAP_SEND);
3443 if (uap->tolen != 0) {
3444 error = getsockaddr(&to, uap->to, uap->tolen);
3449 cap_rights_set(&rights, CAP_CONNECT);
3452 AUDIT_ARG_FD(uap->sd);
3453 error = getsock_cap(td->td_proc->p_fd, uap->sd, &rights, &fp, NULL);
3457 #ifdef COMPAT_FREEBSD32
3458 if (SV_CURPROC_FLAG(SV_ILP32))
3459 error = freebsd32_copyiniov((struct iovec32 *)uap->iov,
3460 uap->iovlen, &iov, EMSGSIZE);
3463 error = copyiniov(uap->iov, uap->iovlen, &iov, EMSGSIZE);
3467 if (to && (KTRPOINT(td, KTR_STRUCT)))
3471 so = (struct socket *)fp->f_data;
3472 if (so->so_proto->pr_protocol != IPPROTO_SCTP) {
3477 error = mac_socket_check_send(td->td_ucred, so);
3483 auio.uio_iovcnt = uap->iovlen;
3484 auio.uio_segflg = UIO_USERSPACE;
3485 auio.uio_rw = UIO_WRITE;
3487 auio.uio_offset = 0; /* XXX */
3490 for (i = 0; i <uap->iovlen; i++, tiov++) {
3491 if ((auio.uio_resid += tiov->iov_len) < 0) {
3496 len = auio.uio_resid;
3497 CURVNET_SET(so->so_vnet);
3498 error = sctp_lower_sosend(so, to, &auio,
3499 (struct mbuf *)NULL, (struct mbuf *)NULL,
3500 uap->flags, u_sinfo, td);
3503 if (auio.uio_resid != len && (error == ERESTART ||
3504 error == EINTR || error == EWOULDBLOCK))
3506 /* Generation of SIGPIPE can be controlled per socket */
3507 if (error == EPIPE && !(so->so_options & SO_NOSIGPIPE) &&
3508 !(uap->flags & MSG_NOSIGNAL)) {
3509 PROC_LOCK(td->td_proc);
3510 tdsignal(td, SIGPIPE);
3511 PROC_UNLOCK(td->td_proc);
3515 td->td_retval[0] = len - auio.uio_resid;
3517 if (ktruio != NULL) {
3518 ktruio->uio_resid = td->td_retval[0];
3519 ktrgenio(uap->sd, UIO_WRITE, ktruio, error);
3531 return (EOPNOTSUPP);
3536 sys_sctp_generic_recvmsg(td, uap)
3538 struct sctp_generic_recvmsg_args /* {
3542 struct sockaddr *from,
3543 __socklen_t *fromlenaddr,
3544 struct sctp_sndrcvinfo *sinfo,
3548 #if (defined(INET) || defined(INET6)) && defined(SCTP)
3549 uint8_t sockbufstore[256];
3551 struct iovec *iov, *tiov;
3552 struct sctp_sndrcvinfo sinfo;
3554 struct file *fp = NULL;
3555 struct sockaddr *fromsa;
3556 cap_rights_t rights;
3558 struct uio *ktruio = NULL;
3561 int error, fromlen, i, msg_flags;
3563 AUDIT_ARG_FD(uap->sd);
3564 error = getsock_cap(td->td_proc->p_fd, uap->sd,
3565 cap_rights_init(&rights, CAP_RECV), &fp, NULL);
3568 #ifdef COMPAT_FREEBSD32
3569 if (SV_CURPROC_FLAG(SV_ILP32))
3570 error = freebsd32_copyiniov((struct iovec32 *)uap->iov,
3571 uap->iovlen, &iov, EMSGSIZE);
3574 error = copyiniov(uap->iov, uap->iovlen, &iov, EMSGSIZE);
3579 if (so->so_proto->pr_protocol != IPPROTO_SCTP) {
3584 error = mac_socket_check_receive(td->td_ucred, so);
3589 if (uap->fromlenaddr != NULL) {
3590 error = copyin(uap->fromlenaddr, &fromlen, sizeof (fromlen));
3596 if (uap->msg_flags) {
3597 error = copyin(uap->msg_flags, &msg_flags, sizeof (int));
3604 auio.uio_iovcnt = uap->iovlen;
3605 auio.uio_segflg = UIO_USERSPACE;
3606 auio.uio_rw = UIO_READ;
3608 auio.uio_offset = 0; /* XXX */
3611 for (i = 0; i <uap->iovlen; i++, tiov++) {
3612 if ((auio.uio_resid += tiov->iov_len) < 0) {
3617 len = auio.uio_resid;
3618 fromsa = (struct sockaddr *)sockbufstore;
3621 if (KTRPOINT(td, KTR_GENIO))
3622 ktruio = cloneuio(&auio);
3624 memset(&sinfo, 0, sizeof(struct sctp_sndrcvinfo));
3625 CURVNET_SET(so->so_vnet);
3626 error = sctp_sorecvmsg(so, &auio, (struct mbuf **)NULL,
3627 fromsa, fromlen, &msg_flags,
3628 (struct sctp_sndrcvinfo *)&sinfo, 1);
3631 if (auio.uio_resid != len && (error == ERESTART ||
3632 error == EINTR || error == EWOULDBLOCK))
3636 error = copyout(&sinfo, uap->sinfo, sizeof (sinfo));
3639 if (ktruio != NULL) {
3640 ktruio->uio_resid = len - auio.uio_resid;
3641 ktrgenio(uap->sd, UIO_READ, ktruio, error);
3646 td->td_retval[0] = len - auio.uio_resid;
3648 if (fromlen && uap->from) {
3650 if (len <= 0 || fromsa == 0)
3653 len = MIN(len, fromsa->sa_len);
3654 error = copyout(fromsa, uap->from, (size_t)len);
3658 error = copyout(&len, uap->fromlenaddr, sizeof (socklen_t));
3663 if (KTRPOINT(td, KTR_STRUCT))
3664 ktrsockaddr(fromsa);
3666 if (uap->msg_flags) {
3667 error = copyout(&msg_flags, uap->msg_flags, sizeof (int));
3679 return (EOPNOTSUPP);