2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2007 Roman Divacky
5 * Copyright (c) 2014 Dmitry Chagin <dchagin@FreeBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <sys/param.h>
30 #include <sys/callout.h>
31 #include <sys/capsicum.h>
32 #include <sys/errno.h>
33 #include <sys/event.h>
34 #include <sys/eventfd.h>
36 #include <sys/filedesc.h>
37 #include <sys/filio.h>
38 #include <sys/limits.h>
40 #include <sys/mutex.h>
43 #include <sys/selinfo.h>
44 #include <sys/specialfd.h>
46 #include <sys/syscallsubr.h>
47 #include <sys/timerfd.h>
48 #include <sys/timespec.h>
52 #include <machine/../linux32/linux.h>
53 #include <machine/../linux32/linux32_proto.h>
55 #include <machine/../linux/linux.h>
56 #include <machine/../linux/linux_proto.h>
59 #include <compat/linux/linux_emul.h>
60 #include <compat/linux/linux_event.h>
61 #include <compat/linux/linux_file.h>
62 #include <compat/linux/linux_signal.h>
63 #include <compat/linux/linux_time.h>
64 #include <compat/linux/linux_util.h>
66 typedef uint64_t epoll_udata_t;
72 #if defined(__amd64__)
73 __attribute__((packed))
77 #define LINUX_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event))
79 static int epoll_to_kevent(struct thread *td, int fd,
80 struct epoll_event *l_event, struct kevent *kevent,
82 static void kevent_to_epoll(struct kevent *kevent, struct epoll_event *l_event);
83 static int epoll_kev_copyout(void *arg, struct kevent *kevp, int count);
84 static int epoll_kev_copyin(void *arg, struct kevent *kevp, int count);
85 static int epoll_register_kevent(struct thread *td, struct file *epfp,
86 int fd, int filter, unsigned int flags);
87 static int epoll_fd_registered(struct thread *td, struct file *epfp,
89 static int epoll_delete_all_events(struct thread *td, struct file *epfp,
92 struct epoll_copyin_args {
93 struct kevent *changelist;
96 struct epoll_copyout_args {
97 struct epoll_event *leventlist;
104 epoll_create_common(struct thread *td, int flags)
107 return (kern_kqueue(td, flags, NULL));
110 #ifdef LINUX_LEGACY_SYSCALLS
112 linux_epoll_create(struct thread *td, struct linux_epoll_create_args *args)
116 * args->size is unused. Linux just tests it
117 * and then forgets it as well.
122 return (epoll_create_common(td, 0));
127 linux_epoll_create1(struct thread *td, struct linux_epoll_create1_args *args)
131 if ((args->flags & ~(LINUX_O_CLOEXEC)) != 0)
135 if ((args->flags & LINUX_O_CLOEXEC) != 0)
138 return (epoll_create_common(td, flags));
141 /* Structure converting function from epoll to kevent. */
143 epoll_to_kevent(struct thread *td, int fd, struct epoll_event *l_event,
144 struct kevent *kevent, int *nkevents)
146 uint32_t levents = l_event->events;
147 struct linux_pemuldata *pem;
149 unsigned short kev_flags = EV_ADD | EV_ENABLE;
151 /* flags related to how event is registered */
152 if ((levents & LINUX_EPOLLONESHOT) != 0)
153 kev_flags |= EV_DISPATCH;
154 if ((levents & LINUX_EPOLLET) != 0)
155 kev_flags |= EV_CLEAR;
156 if ((levents & LINUX_EPOLLERR) != 0)
157 kev_flags |= EV_ERROR;
158 if ((levents & LINUX_EPOLLRDHUP) != 0)
161 /* flags related to what event is registered */
162 if ((levents & LINUX_EPOLL_EVRD) != 0) {
163 EV_SET(kevent, fd, EVFILT_READ, kev_flags, 0, 0, 0);
164 kevent->ext[0] = l_event->data;
168 if ((levents & LINUX_EPOLL_EVWR) != 0) {
169 EV_SET(kevent, fd, EVFILT_WRITE, kev_flags, 0, 0, 0);
170 kevent->ext[0] = l_event->data;
174 /* zero event mask is legal */
175 if ((levents & (LINUX_EPOLL_EVRD | LINUX_EPOLL_EVWR)) == 0) {
176 EV_SET(kevent++, fd, EVFILT_READ, EV_ADD|EV_DISABLE, 0, 0, 0);
180 if ((levents & ~(LINUX_EPOLL_EVSUP)) != 0) {
184 KASSERT(pem != NULL, ("epoll proc emuldata not found.\n"));
186 LINUX_PEM_XLOCK(pem);
187 if ((pem->flags & LINUX_XUNSUP_EPOLL) == 0) {
188 pem->flags |= LINUX_XUNSUP_EPOLL;
189 LINUX_PEM_XUNLOCK(pem);
190 linux_msg(td, "epoll_ctl unsupported flags: 0x%x",
193 LINUX_PEM_XUNLOCK(pem);
201 * Structure converting function from kevent to epoll. In a case
202 * this is called on error in registration we store the error in
203 * event->data and pick it up later in linux_epoll_ctl().
206 kevent_to_epoll(struct kevent *kevent, struct epoll_event *l_event)
209 l_event->data = kevent->ext[0];
211 if ((kevent->flags & EV_ERROR) != 0) {
212 l_event->events = LINUX_EPOLLERR;
216 /* XXX EPOLLPRI, EPOLLHUP */
217 switch (kevent->filter) {
219 l_event->events = LINUX_EPOLLIN;
220 if ((kevent->flags & EV_EOF) != 0)
221 l_event->events |= LINUX_EPOLLRDHUP;
224 l_event->events = LINUX_EPOLLOUT;
230 * Copyout callback used by kevent. This converts kevent
231 * events to epoll events and copies them back to the
232 * userspace. This is also called on error on registering
236 epoll_kev_copyout(void *arg, struct kevent *kevp, int count)
238 struct epoll_copyout_args *args;
239 struct epoll_event *eep;
242 args = (struct epoll_copyout_args*) arg;
243 eep = malloc(sizeof(*eep) * count, M_EPOLL, M_WAITOK | M_ZERO);
245 for (i = 0; i < count; i++)
246 kevent_to_epoll(&kevp[i], &eep[i]);
248 error = copyout(eep, args->leventlist, count * sizeof(*eep));
250 args->leventlist += count;
251 args->count += count;
252 } else if (args->error == 0)
260 * Copyin callback used by kevent. This copies already
261 * converted filters from kernel memory to the kevent
262 * internal kernel memory. Hence the memcpy instead of
266 epoll_kev_copyin(void *arg, struct kevent *kevp, int count)
268 struct epoll_copyin_args *args;
270 args = (struct epoll_copyin_args*) arg;
272 memcpy(kevp, args->changelist, count * sizeof(*kevp));
273 args->changelist += count;
279 * Load epoll filter, convert it to kevent filter
280 * and load it into kevent subsystem.
283 linux_epoll_ctl(struct thread *td, struct linux_epoll_ctl_args *args)
285 struct file *epfp, *fp;
286 struct epoll_copyin_args ciargs;
287 struct kevent kev[2];
288 struct kevent_copyops k_ops = { &ciargs,
291 struct epoll_event le;
296 if (args->op != LINUX_EPOLL_CTL_DEL) {
297 error = copyin(args->event, &le, sizeof(le));
302 error = fget(td, args->epfd,
303 cap_rights_init_one(&rights, CAP_KQUEUE_CHANGE), &epfp);
306 if (epfp->f_type != DTYPE_KQUEUE) {
311 /* Protect user data vector from incorrectly supplied fd. */
312 error = fget(td, args->fd,
313 cap_rights_init_one(&rights, CAP_POLL_EVENT), &fp);
317 /* Linux disallows spying on himself */
323 ciargs.changelist = kev;
325 if (args->op != LINUX_EPOLL_CTL_DEL) {
326 error = epoll_to_kevent(td, args->fd, &le, kev, &nchanges);
332 case LINUX_EPOLL_CTL_MOD:
333 error = epoll_delete_all_events(td, epfp, args->fd);
338 case LINUX_EPOLL_CTL_ADD:
339 if (epoll_fd_registered(td, epfp, args->fd)) {
345 case LINUX_EPOLL_CTL_DEL:
346 /* CTL_DEL means unregister this fd with this epoll */
347 error = epoll_delete_all_events(td, epfp, args->fd);
355 error = kern_kevent_fp(td, epfp, nchanges, 0, &k_ops, NULL);
366 * Wait for a filter to be triggered on the epoll file descriptor.
370 linux_epoll_wait_ts(struct thread *td, int epfd, struct epoll_event *events,
371 int maxevents, struct timespec *tsp, sigset_t *uset)
373 struct epoll_copyout_args coargs;
374 struct kevent_copyops k_ops = { &coargs,
382 if (maxevents <= 0 || maxevents > LINUX_MAX_EVENTS)
385 error = fget(td, epfd,
386 cap_rights_init_one(&rights, CAP_KQUEUE_EVENT), &epfp);
389 if (epfp->f_type != DTYPE_KQUEUE) {
394 error = kern_sigprocmask(td, SIG_SETMASK, uset,
398 td->td_pflags |= TDP_OLDMASK;
400 * Make sure that ast() is called on return to
401 * usermode and TDP_OLDMASK is cleared, restoring old
404 ast_sched(td, TDA_SIGSUSPEND);
407 coargs.leventlist = events;
408 coargs.p = td->td_proc;
412 error = kern_kevent_fp(td, epfp, 0, maxevents, &k_ops, tsp);
413 if (error == 0 && coargs.error != 0)
414 error = coargs.error;
417 * kern_kevent might return ENOMEM which is not expected from epoll_wait.
418 * Maybe we should translate that but I don't think it matters at all.
421 td->td_retval[0] = coargs.count;
424 error = kern_sigprocmask(td, SIG_SETMASK, &omask,
432 linux_epoll_wait_common(struct thread *td, int epfd, struct epoll_event *events,
433 int maxevents, int timeout, sigset_t *uset)
435 struct timespec ts, *tsp;
438 * Linux epoll_wait(2) man page states that timeout of -1 causes caller
439 * to block indefinitely. Real implementation does it if any negative
440 * timeout value is passed.
443 /* Convert from milliseconds to timespec. */
444 ts.tv_sec = timeout / 1000;
445 ts.tv_nsec = (timeout % 1000) * 1000000;
450 return (linux_epoll_wait_ts(td, epfd, events, maxevents, tsp, uset));
454 #ifdef LINUX_LEGACY_SYSCALLS
456 linux_epoll_wait(struct thread *td, struct linux_epoll_wait_args *args)
459 return (linux_epoll_wait_common(td, args->epfd, args->events,
460 args->maxevents, args->timeout, NULL));
465 linux_epoll_pwait(struct thread *td, struct linux_epoll_pwait_args *args)
467 sigset_t mask, *pmask;
470 error = linux_copyin_sigset(td, args->mask, sizeof(l_sigset_t),
475 return (linux_epoll_wait_common(td, args->epfd, args->events,
476 args->maxevents, args->timeout, pmask));
479 #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32))
481 linux_epoll_pwait2_64(struct thread *td, struct linux_epoll_pwait2_64_args *args)
483 struct timespec ts, *tsa;
484 sigset_t mask, *pmask;
487 error = linux_copyin_sigset(td, args->mask, sizeof(l_sigset_t),
493 error = linux_get_timespec64(&ts, args->timeout);
500 return (linux_epoll_wait_ts(td, args->epfd, args->events,
501 args->maxevents, tsa, pmask));
505 linux_epoll_pwait2(struct thread *td, struct linux_epoll_pwait2_args *args)
507 struct timespec ts, *tsa;
508 sigset_t mask, *pmask;
511 error = linux_copyin_sigset(td, args->mask, sizeof(l_sigset_t),
517 error = linux_get_timespec(&ts, args->timeout);
524 return (linux_epoll_wait_ts(td, args->epfd, args->events,
525 args->maxevents, tsa, pmask));
527 #endif /* __i386__ || (__amd64__ && COMPAT_LINUX32) */
530 epoll_register_kevent(struct thread *td, struct file *epfp, int fd, int filter,
533 struct epoll_copyin_args ciargs;
535 struct kevent_copyops k_ops = { &ciargs,
539 ciargs.changelist = &kev;
540 EV_SET(&kev, fd, filter, flags, 0, 0, 0);
542 return (kern_kevent_fp(td, epfp, 1, 0, &k_ops, NULL));
546 epoll_fd_registered(struct thread *td, struct file *epfp, int fd)
549 * Set empty filter flags to avoid accidental modification of already
550 * registered events. In the case of event re-registration:
551 * 1. If event does not exists kevent() does nothing and returns ENOENT
552 * 2. If event does exists, it's enabled/disabled state is preserved
553 * but fflags, data and udata fields are overwritten. So we can not
554 * set socket lowats and store user's context pointer in udata.
556 if (epoll_register_kevent(td, epfp, fd, EVFILT_READ, 0) != ENOENT ||
557 epoll_register_kevent(td, epfp, fd, EVFILT_WRITE, 0) != ENOENT)
564 epoll_delete_all_events(struct thread *td, struct file *epfp, int fd)
568 error1 = epoll_register_kevent(td, epfp, fd, EVFILT_READ, EV_DELETE);
569 error2 = epoll_register_kevent(td, epfp, fd, EVFILT_WRITE, EV_DELETE);
571 /* return 0 if at least one result positive */
572 return (error1 == 0 ? 0 : error2);
575 #ifdef LINUX_LEGACY_SYSCALLS
577 linux_eventfd(struct thread *td, struct linux_eventfd_args *args)
579 struct specialfd_eventfd ae;
581 bzero(&ae, sizeof(ae));
582 ae.initval = args->initval;
583 return (kern_specialfd(td, SPECIALFD_EVENTFD, &ae));
588 linux_eventfd2(struct thread *td, struct linux_eventfd2_args *args)
590 struct specialfd_eventfd ae;
593 if ((args->flags & ~(LINUX_O_CLOEXEC | LINUX_O_NONBLOCK |
594 LINUX_EFD_SEMAPHORE)) != 0)
597 if ((args->flags & LINUX_O_CLOEXEC) != 0)
598 flags |= EFD_CLOEXEC;
599 if ((args->flags & LINUX_O_NONBLOCK) != 0)
600 flags |= EFD_NONBLOCK;
601 if ((args->flags & LINUX_EFD_SEMAPHORE) != 0)
602 flags |= EFD_SEMAPHORE;
604 bzero(&ae, sizeof(ae));
606 ae.initval = args->initval;
607 return (kern_specialfd(td, SPECIALFD_EVENTFD, &ae));
611 linux_timerfd_create(struct thread *td, struct linux_timerfd_create_args *args)
616 error = linux_to_native_clockid(&clockid, args->clockid);
620 return (kern_timerfd_create(td, clockid, args->flags));
624 linux_timerfd_gettime(struct thread *td, struct linux_timerfd_gettime_args *args)
626 struct l_itimerspec lots;
627 struct itimerspec ots;
630 error = kern_timerfd_gettime(td, args->fd, &ots);
634 error = native_to_linux_itimerspec(&lots, &ots);
636 error = copyout(&lots, args->old_value, sizeof(lots));
642 linux_timerfd_settime(struct thread *td, struct linux_timerfd_settime_args *args)
644 struct l_itimerspec lots;
645 struct itimerspec nts, ots;
648 error = copyin(args->new_value, &lots, sizeof(lots));
651 error = linux_to_native_itimerspec(&nts, &lots);
654 if (args->old_value == NULL)
655 error = kern_timerfd_settime(td, args->fd, args->flags, &nts, NULL);
657 error = kern_timerfd_settime(td, args->fd, args->flags, &nts, &ots);
658 if (error == 0 && args->old_value != NULL) {
659 error = native_to_linux_itimerspec(&lots, &ots);
661 error = copyout(&lots, args->old_value, sizeof(lots));
667 #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32))
669 linux_timerfd_gettime64(struct thread *td, struct linux_timerfd_gettime64_args *args)
671 struct l_itimerspec64 lots;
672 struct itimerspec ots;
675 error = kern_timerfd_gettime(td, args->fd, &ots);
679 error = native_to_linux_itimerspec64(&lots, &ots);
681 error = copyout(&lots, args->old_value, sizeof(lots));
687 linux_timerfd_settime64(struct thread *td, struct linux_timerfd_settime64_args *args)
689 struct l_itimerspec64 lots;
690 struct itimerspec nts, ots;
693 error = copyin(args->new_value, &lots, sizeof(lots));
696 error = linux_to_native_itimerspec64(&nts, &lots);
699 if (args->old_value == NULL)
700 error = kern_timerfd_settime(td, args->fd, args->flags, &nts, NULL);
702 error = kern_timerfd_settime(td, args->fd, args->flags, &nts, &ots);
703 if (error == 0 && args->old_value != NULL) {
704 error = native_to_linux_itimerspec64(&lots, &ots);
706 error = copyout(&lots, args->old_value, sizeof(lots));