2 * Copyright (c) 2007 Roman Divacky
3 * Copyright (c) 2014 Dmitry Chagin
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
31 #include "opt_compat.h"
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/imgact.h>
36 #include <sys/kernel.h>
37 #include <sys/limits.h>
39 #include <sys/mutex.h>
40 #include <sys/capability.h>
41 #include <sys/types.h>
44 #include <sys/filedesc.h>
45 #include <sys/filio.h>
46 #include <sys/errno.h>
47 #include <sys/event.h>
50 #include <sys/selinfo.h>
52 #include <sys/syscallsubr.h>
53 #include <sys/timespec.h>
56 #include <machine/../linux32/linux.h>
57 #include <machine/../linux32/linux32_proto.h>
59 #include <machine/../linux/linux.h>
60 #include <machine/../linux/linux_proto.h>
63 #include <compat/linux/linux_emul.h>
64 #include <compat/linux/linux_event.h>
65 #include <compat/linux/linux_file.h>
66 #include <compat/linux/linux_util.h>
69 * epoll defines 'struct epoll_event' with the field 'data' as 64 bits
70 * on all architectures. But on 32 bit architectures BSD 'struct kevent' only
71 * has 32 bit opaque pointer as 'udata' field. So we can't pass epoll supplied
72 * data verbatuim. Therefore we allocate 64-bit memory block to pass
73 * user supplied data for every file descriptor.
76 typedef uint64_t epoll_udata_t;
78 struct epoll_emuldata {
79 uint32_t fdc; /* epoll udata max index */
80 epoll_udata_t udata[1]; /* epoll user data vector */
83 #define EPOLL_DEF_SZ 16
84 #define EPOLL_SIZE(fdn) \
85 (sizeof(struct epoll_emuldata)+(fdn) * sizeof(epoll_udata_t))
91 #if defined(__amd64__)
92 __attribute__((packed))
96 #define LINUX_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event))
98 static void epoll_fd_install(struct thread *td, int fd, epoll_udata_t udata);
99 static int epoll_to_kevent(struct thread *td, struct file *epfp,
100 int fd, struct epoll_event *l_event, int *kev_flags,
101 struct kevent *kevent, int *nkevents);
102 static void kevent_to_epoll(struct kevent *kevent, struct epoll_event *l_event);
103 static int epoll_kev_copyout(void *arg, struct kevent *kevp, int count);
104 static int epoll_kev_copyin(void *arg, struct kevent *kevp, int count);
105 static int epoll_delete_event(struct thread *td, struct file *epfp,
107 static int epoll_delete_all_events(struct thread *td, struct file *epfp,
110 struct epoll_copyin_args {
111 struct kevent *changelist;
114 struct epoll_copyout_args {
115 struct epoll_event *leventlist;
122 typedef uint64_t eventfd_t;
124 static fo_rdwr_t eventfd_read;
125 static fo_rdwr_t eventfd_write;
126 static fo_truncate_t eventfd_truncate;
127 static fo_ioctl_t eventfd_ioctl;
128 static fo_poll_t eventfd_poll;
129 static fo_kqfilter_t eventfd_kqfilter;
130 static fo_stat_t eventfd_stat;
131 static fo_close_t eventfd_close;
132 static fo_fill_kinfo_t eventfd_fill_kinfo;
134 static struct fileops eventfdops = {
135 .fo_read = eventfd_read,
136 .fo_write = eventfd_write,
137 .fo_truncate = eventfd_truncate,
138 .fo_ioctl = eventfd_ioctl,
139 .fo_poll = eventfd_poll,
140 .fo_kqfilter = eventfd_kqfilter,
141 .fo_stat = eventfd_stat,
142 .fo_close = eventfd_close,
143 .fo_chmod = invfo_chmod,
144 .fo_chown = invfo_chown,
145 .fo_sendfile = invfo_sendfile,
146 .fo_fill_kinfo = eventfd_fill_kinfo,
147 .fo_flags = DFLAG_PASSABLE
150 static void filt_eventfddetach(struct knote *kn);
151 static int filt_eventfdread(struct knote *kn, long hint);
152 static int filt_eventfdwrite(struct knote *kn, long hint);
154 static struct filterops eventfd_rfiltops = {
156 .f_detach = filt_eventfddetach,
157 .f_event = filt_eventfdread
159 static struct filterops eventfd_wfiltops = {
161 .f_detach = filt_eventfddetach,
162 .f_event = filt_eventfdwrite
168 struct selinfo efd_sel;
172 static int eventfd_create(struct thread *td, uint32_t initval, int flags);
176 epoll_fd_install(struct thread *td, int fd, epoll_udata_t udata)
178 struct linux_pemuldata *pem;
179 struct epoll_emuldata *emd;
185 KASSERT(pem != NULL, ("epoll proc emuldata not found.\n"));
187 LINUX_PEM_XLOCK(pem);
188 if (pem->epoll == NULL) {
189 emd = malloc(EPOLL_SIZE(fd), M_EPOLL, M_WAITOK);
195 emd = realloc(emd, EPOLL_SIZE(fd), M_EPOLL, M_WAITOK);
200 emd->udata[fd] = udata;
201 LINUX_PEM_XUNLOCK(pem);
205 epoll_create_common(struct thread *td, int flags)
209 error = kern_kqueue(td, flags, NULL);
213 epoll_fd_install(td, EPOLL_DEF_SZ, 0);
219 linux_epoll_create(struct thread *td, struct linux_epoll_create_args *args)
223 * args->size is unused. Linux just tests it
224 * and then forgets it as well.
229 return (epoll_create_common(td, 0));
233 linux_epoll_create1(struct thread *td, struct linux_epoll_create1_args *args)
237 if ((args->flags & ~(LINUX_O_CLOEXEC)) != 0)
241 if ((args->flags & LINUX_O_CLOEXEC) != 0)
244 return (epoll_create_common(td, flags));
247 /* Structure converting function from epoll to kevent. */
249 epoll_to_kevent(struct thread *td, struct file *epfp,
250 int fd, struct epoll_event *l_event, int *kev_flags,
251 struct kevent *kevent, int *nkevents)
253 uint32_t levents = l_event->events;
254 struct linux_pemuldata *pem;
257 /* flags related to how event is registered */
258 if ((levents & LINUX_EPOLLONESHOT) != 0)
259 *kev_flags |= EV_ONESHOT;
260 if ((levents & LINUX_EPOLLET) != 0)
261 *kev_flags |= EV_CLEAR;
262 if ((levents & LINUX_EPOLLERR) != 0)
263 *kev_flags |= EV_ERROR;
264 if ((levents & LINUX_EPOLLRDHUP) != 0)
265 *kev_flags |= EV_EOF;
267 /* flags related to what event is registered */
268 if ((levents & LINUX_EPOLL_EVRD) != 0) {
269 EV_SET(kevent++, fd, EVFILT_READ, *kev_flags, 0, 0, 0);
272 if ((levents & LINUX_EPOLL_EVWR) != 0) {
273 EV_SET(kevent++, fd, EVFILT_WRITE, *kev_flags, 0, 0, 0);
277 if ((levents & ~(LINUX_EPOLL_EVSUP)) != 0) {
281 KASSERT(pem != NULL, ("epoll proc emuldata not found.\n"));
282 KASSERT(pem->epoll != NULL, ("epoll proc epolldata not found.\n"));
284 LINUX_PEM_XLOCK(pem);
285 if ((pem->flags & LINUX_XUNSUP_EPOLL) == 0) {
286 pem->flags |= LINUX_XUNSUP_EPOLL;
287 LINUX_PEM_XUNLOCK(pem);
288 linux_msg(td, "epoll_ctl unsupported flags: 0x%x\n",
291 LINUX_PEM_XUNLOCK(pem);
299 * Structure converting function from kevent to epoll. In a case
300 * this is called on error in registration we store the error in
301 * event->data and pick it up later in linux_epoll_ctl().
304 kevent_to_epoll(struct kevent *kevent, struct epoll_event *l_event)
307 if ((kevent->flags & EV_ERROR) != 0) {
308 l_event->events = LINUX_EPOLLERR;
312 switch (kevent->filter) {
314 l_event->events = LINUX_EPOLLIN|LINUX_EPOLLRDNORM|LINUX_EPOLLPRI;
315 if ((kevent->flags & EV_EOF) != 0)
316 l_event->events |= LINUX_EPOLLRDHUP;
319 l_event->events = LINUX_EPOLLOUT|LINUX_EPOLLWRNORM;
325 * Copyout callback used by kevent. This converts kevent
326 * events to epoll events and copies them back to the
327 * userspace. This is also called on error on registering
331 epoll_kev_copyout(void *arg, struct kevent *kevp, int count)
333 struct epoll_copyout_args *args;
334 struct linux_pemuldata *pem;
335 struct epoll_emuldata *emd;
336 struct epoll_event *eep;
339 args = (struct epoll_copyout_args*) arg;
340 eep = malloc(sizeof(*eep) * count, M_EPOLL, M_WAITOK | M_ZERO);
342 pem = pem_find(args->p);
343 KASSERT(pem != NULL, ("epoll proc emuldata not found.\n"));
344 LINUX_PEM_SLOCK(pem);
346 KASSERT(emd != NULL, ("epoll proc epolldata not found.\n"));
348 for (i = 0; i < count; i++) {
349 kevent_to_epoll(&kevp[i], &eep[i]);
352 KASSERT(fd <= emd->fdc, ("epoll user data vector"
353 " is too small.\n"));
354 eep[i].data = emd->udata[fd];
356 LINUX_PEM_SUNLOCK(pem);
358 error = copyout(eep, args->leventlist, count * sizeof(*eep));
360 args->leventlist += count;
361 args->count += count;
362 } else if (args->error == 0)
370 * Copyin callback used by kevent. This copies already
371 * converted filters from kernel memory to the kevent
372 * internal kernel memory. Hence the memcpy instead of
376 epoll_kev_copyin(void *arg, struct kevent *kevp, int count)
378 struct epoll_copyin_args *args;
380 args = (struct epoll_copyin_args*) arg;
382 memcpy(kevp, args->changelist, count * sizeof(*kevp));
383 args->changelist += count;
389 * Load epoll filter, convert it to kevent filter
390 * and load it into kevent subsystem.
393 linux_epoll_ctl(struct thread *td, struct linux_epoll_ctl_args *args)
395 struct file *epfp, *fp;
396 struct epoll_copyin_args ciargs;
397 struct kevent kev[2];
398 struct kevent_copyops k_ops = { &ciargs,
401 struct epoll_event le;
407 if (args->op != LINUX_EPOLL_CTL_DEL) {
408 error = copyin(args->event, &le, sizeof(le));
413 error = fget(td, args->epfd,
414 cap_rights_init(&rights, CAP_KQUEUE_CHANGE), &epfp);
417 if (epfp->f_type != DTYPE_KQUEUE)
420 /* Protect user data vector from incorrectly supplied fd. */
421 error = fget(td, args->fd, cap_rights_init(&rights, CAP_POLL_EVENT), &fp);
425 /* Linux disallows spying on himself */
431 ciargs.changelist = kev;
434 case LINUX_EPOLL_CTL_MOD:
436 * We don't memorize which events were set for this FD
437 * on this level, so just delete all we could have set:
438 * EVFILT_READ and EVFILT_WRITE, ignoring any errors
440 error = epoll_delete_all_events(td, epfp, args->fd);
445 case LINUX_EPOLL_CTL_ADD:
446 kev_flags = EV_ADD | EV_ENABLE;
449 case LINUX_EPOLL_CTL_DEL:
450 /* CTL_DEL means unregister this fd with this epoll */
451 error = epoll_delete_all_events(td, epfp, args->fd);
459 error = epoll_to_kevent(td, epfp, args->fd, &le, &kev_flags,
464 epoll_fd_install(td, args->fd, le.data);
466 error = kern_kevent_fp(td, epfp, nchanges, 0, &k_ops, NULL);
477 * Wait for a filter to be triggered on the epoll file descriptor.
480 linux_epoll_wait_common(struct thread *td, int epfd, struct epoll_event *events,
481 int maxevents, int timeout, sigset_t *uset)
484 struct timespec ts, *tsp;
486 struct epoll_copyout_args coargs;
487 struct kevent_copyops k_ops = { &coargs,
492 if (maxevents <= 0 || maxevents > LINUX_MAX_EVENTS)
496 error = kern_sigprocmask(td, SIG_SETMASK, uset,
497 &td->td_oldsigmask, 0);
500 td->td_pflags |= TDP_OLDMASK;
502 * Make sure that ast() is called on return to
503 * usermode and TDP_OLDMASK is cleared, restoring old
507 td->td_flags |= TDF_ASTPENDING;
511 error = fget(td, epfd,
512 cap_rights_init(&rights, CAP_KQUEUE_EVENT), &epfp);
516 coargs.leventlist = events;
517 coargs.p = td->td_proc;
526 /* Convert from milliseconds to timespec. */
527 ts.tv_sec = timeout / 1000;
528 ts.tv_nsec = (timeout % 1000) * 1000000;
534 error = kern_kevent_fp(td, epfp, 0, maxevents, &k_ops, tsp);
535 if (error == 0 && coargs.error != 0)
536 error = coargs.error;
539 * kern_kevent might return ENOMEM which is not expected from epoll_wait.
540 * Maybe we should translate that but I don't think it matters at all.
543 td->td_retval[0] = coargs.count;
550 linux_epoll_wait(struct thread *td, struct linux_epoll_wait_args *args)
553 return (linux_epoll_wait_common(td, args->epfd, args->events,
554 args->maxevents, args->timeout, NULL));
558 linux_epoll_pwait(struct thread *td, struct linux_epoll_pwait_args *args)
560 sigset_t mask, *pmask;
564 if (args->mask != NULL) {
565 error = copyin(args->mask, &lmask, sizeof(l_sigset_t));
568 linux_to_bsd_sigset(&lmask, &mask);
572 return (linux_epoll_wait_common(td, args->epfd, args->events,
573 args->maxevents, args->timeout, pmask));
577 epoll_delete_event(struct thread *td, struct file *epfp, int fd, int filter)
579 struct epoll_copyin_args ciargs;
581 struct kevent_copyops k_ops = { &ciargs,
586 ciargs.changelist = &kev;
587 EV_SET(&kev, fd, filter, EV_DELETE | EV_DISABLE, 0, 0, 0);
589 error = kern_kevent_fp(td, epfp, 1, 0, &k_ops, NULL);
592 * here we ignore ENONT, because we don't keep track of events here
600 epoll_delete_all_events(struct thread *td, struct file *epfp, int fd)
604 error1 = epoll_delete_event(td, epfp, fd, EVFILT_READ);
605 error2 = epoll_delete_event(td, epfp, fd, EVFILT_WRITE);
607 /* report any errors we got */
608 return (error1 == 0 ? error2 : error1);
612 eventfd_create(struct thread *td, uint32_t initval, int flags)
614 struct filedesc *fdp;
617 int fflags, fd, error;
620 if ((flags & LINUX_O_CLOEXEC) != 0)
623 fdp = td->td_proc->p_fd;
624 error = falloc(td, &fp, &fd, fflags);
628 efd = malloc(sizeof(*efd), M_EPOLL, M_WAITOK | M_ZERO);
629 efd->efd_flags = flags;
630 efd->efd_count = initval;
631 mtx_init(&efd->efd_lock, "eventfd", NULL, MTX_DEF);
633 knlist_init_mtx(&efd->efd_sel.si_note, &efd->efd_lock);
635 fflags = FREAD | FWRITE;
636 if ((flags & LINUX_O_NONBLOCK) != 0)
639 finit(fp, fflags, DTYPE_LINUXEFD, efd, &eventfdops);
642 td->td_retval[0] = fd;
647 linux_eventfd(struct thread *td, struct linux_eventfd_args *args)
650 return (eventfd_create(td, args->initval, 0));
654 linux_eventfd2(struct thread *td, struct linux_eventfd2_args *args)
657 if ((args->flags & ~(LINUX_O_CLOEXEC|LINUX_O_NONBLOCK|LINUX_EFD_SEMAPHORE)) != 0)
660 return (eventfd_create(td, args->initval, args->flags));
664 eventfd_close(struct file *fp, struct thread *td)
669 if (fp->f_type != DTYPE_LINUXEFD || efd == NULL)
672 seldrain(&efd->efd_sel);
673 knlist_destroy(&efd->efd_sel.si_note);
675 fp->f_ops = &badfileops;
676 mtx_destroy(&efd->efd_lock);
683 eventfd_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
684 int flags, struct thread *td)
691 if (fp->f_type != DTYPE_LINUXEFD || efd == NULL)
694 if (uio->uio_resid < sizeof(eventfd_t))
698 mtx_lock(&efd->efd_lock);
700 if (efd->efd_count == 0) {
701 if ((efd->efd_flags & LINUX_O_NONBLOCK) != 0) {
702 mtx_unlock(&efd->efd_lock);
705 error = mtx_sleep(&efd->efd_count, &efd->efd_lock, PCATCH, "lefdrd", 0);
710 if ((efd->efd_flags & LINUX_EFD_SEMAPHORE) != 0) {
714 count = efd->efd_count;
717 KNOTE_LOCKED(&efd->efd_sel.si_note, 0);
718 selwakeup(&efd->efd_sel);
719 wakeup(&efd->efd_count);
720 mtx_unlock(&efd->efd_lock);
721 error = uiomove(&count, sizeof(eventfd_t), uio);
723 mtx_unlock(&efd->efd_lock);
729 eventfd_write(struct file *fp, struct uio *uio, struct ucred *active_cred,
730 int flags, struct thread *td)
737 if (fp->f_type != DTYPE_LINUXEFD || efd == NULL)
740 if (uio->uio_resid < sizeof(eventfd_t))
743 error = uiomove(&count, sizeof(eventfd_t), uio);
746 if (count == UINT64_MAX)
749 mtx_lock(&efd->efd_lock);
751 if (UINT64_MAX - efd->efd_count <= count) {
752 if ((efd->efd_flags & LINUX_O_NONBLOCK) != 0) {
753 mtx_unlock(&efd->efd_lock);
754 /* Do not not return the number of bytes written */
755 uio->uio_resid += sizeof(eventfd_t);
758 error = mtx_sleep(&efd->efd_count, &efd->efd_lock,
759 PCATCH, "lefdwr", 0);
764 efd->efd_count += count;
765 KNOTE_LOCKED(&efd->efd_sel.si_note, 0);
766 selwakeup(&efd->efd_sel);
767 wakeup(&efd->efd_count);
769 mtx_unlock(&efd->efd_lock);
775 eventfd_poll(struct file *fp, int events, struct ucred *active_cred,
782 if (fp->f_type != DTYPE_LINUXEFD || efd == NULL)
785 mtx_lock(&efd->efd_lock);
786 if ((events & (POLLIN|POLLRDNORM)) && efd->efd_count > 0)
787 revents |= events & (POLLIN|POLLRDNORM);
788 if ((events & (POLLOUT|POLLWRNORM)) && UINT64_MAX - 1 > efd->efd_count)
789 revents |= events & (POLLOUT|POLLWRNORM);
791 selrecord(td, &efd->efd_sel);
792 mtx_unlock(&efd->efd_lock);
799 eventfd_kqfilter(struct file *fp, struct knote *kn)
804 if (fp->f_type != DTYPE_LINUXEFD || efd == NULL)
807 mtx_lock(&efd->efd_lock);
808 switch (kn->kn_filter) {
810 kn->kn_fop = &eventfd_rfiltops;
813 kn->kn_fop = &eventfd_wfiltops;
816 mtx_unlock(&efd->efd_lock);
821 knlist_add(&efd->efd_sel.si_note, kn, 1);
822 mtx_unlock(&efd->efd_lock);
828 filt_eventfddetach(struct knote *kn)
830 struct eventfd *efd = kn->kn_hook;
832 mtx_lock(&efd->efd_lock);
833 knlist_remove(&efd->efd_sel.si_note, kn, 1);
834 mtx_unlock(&efd->efd_lock);
839 filt_eventfdread(struct knote *kn, long hint)
841 struct eventfd *efd = kn->kn_hook;
844 mtx_assert(&efd->efd_lock, MA_OWNED);
845 ret = (efd->efd_count > 0);
852 filt_eventfdwrite(struct knote *kn, long hint)
854 struct eventfd *efd = kn->kn_hook;
857 mtx_assert(&efd->efd_lock, MA_OWNED);
858 ret = (UINT64_MAX - 1 > efd->efd_count);
865 eventfd_truncate(struct file *fp, off_t length, struct ucred *active_cred,
874 eventfd_ioctl(struct file *fp, u_long cmd, void *data,
875 struct ucred *active_cred, struct thread *td)
880 if (fp->f_type != DTYPE_LINUXEFD || efd == NULL)
887 efd->efd_flags |= LINUX_O_NONBLOCK;
889 efd->efd_flags &= ~LINUX_O_NONBLOCK;
899 eventfd_stat(struct file *fp, struct stat *st, struct ucred *active_cred,
908 eventfd_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
911 kif->kf_type = KF_TYPE_UNKNOWN;