sys/kern/sys_pipe.c

   1 /*-
   2  * Copyright (c) 1996 John S. Dyson
   3  * Copyright (c) 2012 Giovanni Trematerra
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice immediately at the beginning of the file, without modification,
  11  *    this list of conditions, and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. Absolutely no warranty of function or purpose is made by the author
  16  *    John S. Dyson.
  17  * 4. Modifications may be freely made to this file if the above conditions
  18  *    are met.
  19  */
  20
  21 /*
  22  * This file contains a high-performance replacement for the socket-based
  23  * pipes scheme originally used in FreeBSD/4.4Lite.  It does not support
  24  * all features of sockets, but does do everything that pipes normally
  25  * do.
  26  */
  27
  28 /*
  29  * This code has two modes of operation, a small write mode and a large
  30  * write mode.  The small write mode acts like conventional pipes with
  31  * a kernel buffer.  If the buffer is less than PIPE_MINDIRECT, then the
  32  * "normal" pipe buffering is done.  If the buffer is between PIPE_MINDIRECT
  33  * and PIPE_SIZE in size, the sending process pins the underlying pages in
  34  * memory, and the receiving process copies directly from these pinned pages
  35  * in the sending process.
  36  *
  37  * If the sending process receives a signal, it is possible that it will
  38  * go away, and certainly its address space can change, because control
  39  * is returned back to the user-mode side.  In that case, the pipe code
  40  * arranges to copy the buffer supplied by the user process, to a pageable
  41  * kernel buffer, and the receiving process will grab the data from the
  42  * pageable kernel buffer.  Since signals don't happen all that often,
  43  * the copy operation is normally eliminated.
  44  *
  45  * The constant PIPE_MINDIRECT is chosen to make sure that buffering will
  46  * happen for small transfers so that the system will not spend all of
  47  * its time context switching.
  48  *
  49  * In order to limit the resource use of pipes, two sysctls exist:
  50  *
  51  * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable
  52  * address space available to us in pipe_map. This value is normally
  53  * autotuned, but may also be loader tuned.
  54  *
  55  * kern.ipc.pipekva - This read-only sysctl tracks the current amount of
  56  * memory in use by pipes.
  57  *
  58  * Based on how large pipekva is relative to maxpipekva, the following
  59  * will happen:
  60  *
  61  * 0% - 50%:
  62  *     New pipes are given 16K of memory backing, pipes may dynamically
  63  *     grow to as large as 64K where needed.
  64  * 50% - 75%:
  65  *     New pipes are given 4K (or PAGE_SIZE) of memory backing,
  66  *     existing pipes may NOT grow.
  67  * 75% - 100%:
  68  *     New pipes are given 4K (or PAGE_SIZE) of memory backing,
  69  *     existing pipes will be shrunk down to 4K whenever possible.
  70  *
  71  * Resizing may be disabled by setting kern.ipc.piperesizeallowed=0.  If
  72  * that is set,  the only resize that will occur is the 0 -> SMALL_PIPE_SIZE
  73  * resize which MUST occur for reverse-direction pipes when they are
  74  * first used.
  75  *
  76  * Additional information about the current state of pipes may be obtained
  77  * from kern.ipc.pipes, kern.ipc.pipefragretry, kern.ipc.pipeallocfail,
  78  * and kern.ipc.piperesizefail.
  79  *
  80  * Locking rules:  There are two locks present here:  A mutex, used via
  81  * PIPE_LOCK, and a flag, used via pipelock().  All locking is done via
  82  * the flag, as mutexes can not persist over uiomove.  The mutex
  83  * exists only to guard access to the flag, and is not in itself a
  84  * locking mechanism.  Also note that there is only a single mutex for
  85  * both directions of a pipe.
  86  *
  87  * As pipelock() may have to sleep before it can acquire the flag, it
  88  * is important to reread all data after a call to pipelock(); everything
  89  * in the structure may have changed.
  90  */
  91
  92 #include <sys/param.h>
  93 #include <sys/systm.h>
  94 #include <sys/conf.h>
  95 #include <sys/fcntl.h>
  96 #include <sys/file.h>
  97 #include <sys/filedesc.h>
  98 #include <sys/filio.h>
  99 #include <sys/kernel.h>
 100 #include <sys/lock.h>
 101 #include <sys/mutex.h>
 102 #include <sys/ttycom.h>
 103 #include <sys/stat.h>
 104 #include <sys/malloc.h>
 105 #include <sys/poll.h>
 106 #include <sys/selinfo.h>
 107 #include <sys/signalvar.h>
 108 #include <sys/syscallsubr.h>
 109 #include <sys/sysctl.h>
 110 #include <sys/sysproto.h>
 111 #include <sys/pipe.h>
 112 #include <sys/proc.h>
 113 #include <sys/vnode.h>
 114 #include <sys/uio.h>
 115 #include <sys/user.h>
 116 #include <sys/event.h>
 117
 118 #include <security/mac/mac_framework.h>
 119
 120 #include <vm/vm.h>
 121 #include <vm/vm_param.h>
 122 #include <vm/vm_object.h>
 123 #include <vm/vm_kern.h>
 124 #include <vm/vm_extern.h>
 125 #include <vm/pmap.h>
 126 #include <vm/vm_map.h>
 127 #include <vm/vm_page.h>
 128 #include <vm/uma.h>
 129
 130 /*
 131  * Use this define if you want to disable *fancy* VM things.  Expect an
 132  * approx 30% decrease in transfer rate.  This could be useful for
 133  * NetBSD or OpenBSD.
 134  */
 135 /* #define PIPE_NODIRECT */
 136
 137 #define PIPE_PEER(pipe) \
 138         (((pipe)->pipe_type & PIPE_TYPE_NAMED) ? (pipe) : ((pipe)->pipe_peer))
 139
 140 /*
 141  * interfaces to the outside world
 142  */
 143 static fo_rdwr_t        pipe_read;
 144 static fo_rdwr_t        pipe_write;
 145 static fo_truncate_t    pipe_truncate;
 146 static fo_ioctl_t       pipe_ioctl;
 147 static fo_poll_t        pipe_poll;
 148 static fo_kqfilter_t    pipe_kqfilter;
 149 static fo_stat_t        pipe_stat;
 150 static fo_close_t       pipe_close;
 151 static fo_chmod_t       pipe_chmod;
 152 static fo_chown_t       pipe_chown;
 153 static fo_fill_kinfo_t  pipe_fill_kinfo;
 154
 155 struct fileops pipeops = {
 156         .fo_read = pipe_read,
 157         .fo_write = pipe_write,
 158         .fo_truncate = pipe_truncate,
 159         .fo_ioctl = pipe_ioctl,
 160         .fo_poll = pipe_poll,
 161         .fo_kqfilter = pipe_kqfilter,
 162         .fo_stat = pipe_stat,
 163         .fo_close = pipe_close,
 164         .fo_chmod = pipe_chmod,
 165         .fo_chown = pipe_chown,
 166         .fo_sendfile = invfo_sendfile,
 167         .fo_fill_kinfo = pipe_fill_kinfo,
 168         .fo_cmp = file_kcmp_generic,
 169         .fo_flags = DFLAG_PASSABLE
 170 };
 171
 172 static void     filt_pipedetach(struct knote *kn);
 173 static void     filt_pipedetach_notsup(struct knote *kn);
 174 static int      filt_pipenotsup(struct knote *kn, long hint);
 175 static int      filt_piperead(struct knote *kn, long hint);
 176 static int      filt_pipewrite(struct knote *kn, long hint);
 177
 178 static struct filterops pipe_nfiltops = {
 179         .f_isfd = 1,
 180         .f_detach = filt_pipedetach_notsup,
 181         .f_event = filt_pipenotsup
 182 };
 183 static struct filterops pipe_rfiltops = {
 184         .f_isfd = 1,
 185         .f_detach = filt_pipedetach,
 186         .f_event = filt_piperead
 187 };
 188 static struct filterops pipe_wfiltops = {
 189         .f_isfd = 1,
 190         .f_detach = filt_pipedetach,
 191         .f_event = filt_pipewrite
 192 };
 193
 194 /*
 195  * Default pipe buffer size(s), this can be kind-of large now because pipe
 196  * space is pageable.  The pipe code will try to maintain locality of
 197  * reference for performance reasons, so small amounts of outstanding I/O
 198  * will not wipe the cache.
 199  */
 200 #define MINPIPESIZE (PIPE_SIZE/3)
 201 #define MAXPIPESIZE (2*PIPE_SIZE/3)
 202
 203 static long amountpipekva;
 204 static int pipefragretry;
 205 static int pipeallocfail;
 206 static int piperesizefail;
 207 static int piperesizeallowed = 1;
 208 static long pipe_mindirect = PIPE_MINDIRECT;
 209
 210 SYSCTL_LONG(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
 211            &maxpipekva, 0, "Pipe KVA limit");
 212 SYSCTL_LONG(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD,
 213            &amountpipekva, 0, "Pipe KVA usage");
 214 SYSCTL_INT(_kern_ipc, OID_AUTO, pipefragretry, CTLFLAG_RD,
 215           &pipefragretry, 0, "Pipe allocation retries due to fragmentation");
 216 SYSCTL_INT(_kern_ipc, OID_AUTO, pipeallocfail, CTLFLAG_RD,
 217           &pipeallocfail, 0, "Pipe allocation failures");
 218 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizefail, CTLFLAG_RD,
 219           &piperesizefail, 0, "Pipe resize failures");
 220 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizeallowed, CTLFLAG_RW,
 221           &piperesizeallowed, 0, "Pipe resizing allowed");
 222
 223 static void pipeinit(void *dummy __unused);
 224 static void pipeclose(struct pipe *cpipe);
 225 static void pipe_free_kmem(struct pipe *cpipe);
 226 static int pipe_create(struct pipe *pipe, bool backing);
 227 static int pipe_paircreate(struct thread *td, struct pipepair **p_pp);
 228 static __inline int pipelock(struct pipe *cpipe, int catch);
 229 static __inline void pipeunlock(struct pipe *cpipe);
 230 static void pipe_timestamp(struct timespec *tsp);
 231 #ifndef PIPE_NODIRECT
 232 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
 233 static void pipe_destroy_write_buffer(struct pipe *wpipe);
 234 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
 235 static void pipe_clone_write_buffer(struct pipe *wpipe);
 236 #endif
 237 static int pipespace(struct pipe *cpipe, int size);
 238 static int pipespace_new(struct pipe *cpipe, int size);
 239
 240 static int      pipe_zone_ctor(void *mem, int size, void *arg, int flags);
 241 static int      pipe_zone_init(void *mem, int size, int flags);
 242 static void     pipe_zone_fini(void *mem, int size);
 243
 244 static uma_zone_t pipe_zone;
 245 static struct unrhdr64 pipeino_unr;
 246 static dev_t pipedev_ino;
 247
 248 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL);
 249
 250 static void
 251 pipeinit(void *dummy __unused)
 252 {
 253
 254         pipe_zone = uma_zcreate("pipe", sizeof(struct pipepair),
 255             pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini,
 256             UMA_ALIGN_PTR, 0);
 257         KASSERT(pipe_zone != NULL, ("pipe_zone not initialized"));
 258         new_unrhdr64(&pipeino_unr, 1);
 259         pipedev_ino = devfs_alloc_cdp_inode();
 260         KASSERT(pipedev_ino > 0, ("pipe dev inode not initialized"));
 261 }
 262
 263 static int
 264 sysctl_handle_pipe_mindirect(SYSCTL_HANDLER_ARGS)
 265 {
 266         int error = 0;
 267         long tmp_pipe_mindirect = pipe_mindirect;
 268
 269         error = sysctl_handle_long(oidp, &tmp_pipe_mindirect, arg2, req);
 270         if (error != 0 || req->newptr == NULL)
 271                 return (error);
 272
 273         /*
 274          * Don't allow pipe_mindirect to be set so low that we violate
 275          * atomicity requirements.
 276          */
 277         if (tmp_pipe_mindirect <= PIPE_BUF)
 278                 return (EINVAL);
 279         pipe_mindirect = tmp_pipe_mindirect;
 280         return (0);
 281 }
 282 SYSCTL_OID(_kern_ipc, OID_AUTO, pipe_mindirect, CTLTYPE_LONG | CTLFLAG_RW,
 283     &pipe_mindirect, 0, sysctl_handle_pipe_mindirect, "L",
 284     "Minimum write size triggering VM optimization");
 285
 286 static int
 287 pipe_zone_ctor(void *mem, int size, void *arg, int flags)
 288 {
 289         struct pipepair *pp;
 290         struct pipe *rpipe, *wpipe;
 291
 292         KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size"));
 293
 294         pp = (struct pipepair *)mem;
 295
 296         /*
 297          * We zero both pipe endpoints to make sure all the kmem pointers
 298          * are NULL, flag fields are zero'd, etc.  We timestamp both
 299          * endpoints with the same time.
 300          */
 301         rpipe = &pp->pp_rpipe;
 302         bzero(rpipe, sizeof(*rpipe));
 303         pipe_timestamp(&rpipe->pipe_ctime);
 304         rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime;
 305
 306         wpipe = &pp->pp_wpipe;
 307         bzero(wpipe, sizeof(*wpipe));
 308         wpipe->pipe_ctime = rpipe->pipe_ctime;
 309         wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;
 310
 311         rpipe->pipe_peer = wpipe;
 312         rpipe->pipe_pair = pp;
 313         wpipe->pipe_peer = rpipe;
 314         wpipe->pipe_pair = pp;
 315
 316         /*
 317          * Mark both endpoints as present; they will later get free'd
 318          * one at a time.  When both are free'd, then the whole pair
 319          * is released.
 320          */
 321         rpipe->pipe_present = PIPE_ACTIVE;
 322         wpipe->pipe_present = PIPE_ACTIVE;
 323
 324         /*
 325          * Eventually, the MAC Framework may initialize the label
 326          * in ctor or init, but for now we do it elswhere to avoid
 327          * blocking in ctor or init.
 328          */
 329         pp->pp_label = NULL;
 330
 331         return (0);
 332 }
 333
 334 static int
 335 pipe_zone_init(void *mem, int size, int flags)
 336 {
 337         struct pipepair *pp;
 338
 339         KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size"));
 340
 341         pp = (struct pipepair *)mem;
 342
 343         mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_NEW);
 344         return (0);
 345 }
 346
 347 static void
 348 pipe_zone_fini(void *mem, int size)
 349 {
 350         struct pipepair *pp;
 351
 352         KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size"));
 353
 354         pp = (struct pipepair *)mem;
 355
 356         mtx_destroy(&pp->pp_mtx);
 357 }
 358
 359 static int
 360 pipe_paircreate(struct thread *td, struct pipepair **p_pp)
 361 {
 362         struct pipepair *pp;
 363         struct pipe *rpipe, *wpipe;
 364         int error;
 365
 366         *p_pp = pp = uma_zalloc(pipe_zone, M_WAITOK);
 367 #ifdef MAC
 368         /*
 369          * The MAC label is shared between the connected endpoints.  As a
 370          * result mac_pipe_init() and mac_pipe_create() are called once
 371          * for the pair, and not on the endpoints.
 372          */
 373         mac_pipe_init(pp);
 374         mac_pipe_create(td->td_ucred, pp);
 375 #endif
 376         rpipe = &pp->pp_rpipe;
 377         wpipe = &pp->pp_wpipe;
 378
 379         knlist_init_mtx(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe));
 380         knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe));
 381
 382         /*
 383          * Only the forward direction pipe is backed by big buffer by
 384          * default.
 385          */
 386         error = pipe_create(rpipe, true);
 387         if (error != 0)
 388                 goto fail;
 389         error = pipe_create(wpipe, false);
 390         if (error != 0) {
 391                 /*
 392                  * This cleanup leaves the pipe inode number for rpipe
 393                  * still allocated, but never used.  We do not free
 394                  * inode numbers for opened pipes, which is required
 395                  * for correctness because numbers must be unique.
 396                  * But also it avoids any memory use by the unr
 397                  * allocator, so stashing away the transient inode
 398                  * number is reasonable.
 399                  */
 400                 pipe_free_kmem(rpipe);
 401                 goto fail;
 402         }
 403
 404         rpipe->pipe_state |= PIPE_DIRECTOK;
 405         wpipe->pipe_state |= PIPE_DIRECTOK;
 406         return (0);
 407
 408 fail:
 409         knlist_destroy(&rpipe->pipe_sel.si_note);
 410         knlist_destroy(&wpipe->pipe_sel.si_note);
 411 #ifdef MAC
 412         mac_pipe_destroy(pp);
 413 #endif
 414         uma_zfree(pipe_zone, pp);
 415         return (error);
 416 }
 417
 418 int
 419 pipe_named_ctor(struct pipe **ppipe, struct thread *td)
 420 {
 421         struct pipepair *pp;
 422         int error;
 423
 424         error = pipe_paircreate(td, &pp);
 425         if (error != 0)
 426                 return (error);
 427         pp->pp_rpipe.pipe_type |= PIPE_TYPE_NAMED;
 428         *ppipe = &pp->pp_rpipe;
 429         return (0);
 430 }
 431
 432 void
 433 pipe_dtor(struct pipe *dpipe)
 434 {
 435         struct pipe *peer;
 436
 437         peer = (dpipe->pipe_type & PIPE_TYPE_NAMED) != 0 ? dpipe->pipe_peer : NULL;
 438         funsetown(&dpipe->pipe_sigio);
 439         pipeclose(dpipe);
 440         if (peer != NULL) {
 441                 funsetown(&peer->pipe_sigio);
 442                 pipeclose(peer);
 443         }
 444 }
 445
 446 /*
 447  * Get a timestamp.
 448  *
 449  * This used to be vfs_timestamp but the higher precision is unnecessary and
 450  * can very negatively affect performance in virtualized environments (e.g., on
 451  * vms running on amd64 when using the rdtscp instruction).
 452  */
 453 static void
 454 pipe_timestamp(struct timespec *tsp)
 455 {
 456
 457         getnanotime(tsp);
 458 }
 459
 460 /*
 461  * The pipe system call for the DTYPE_PIPE type of pipes.  If we fail, let
 462  * the zone pick up the pieces via pipeclose().
 463  */
 464 int
 465 kern_pipe(struct thread *td, int fildes[2], int flags, struct filecaps *fcaps1,
 466     struct filecaps *fcaps2)
 467 {
 468         struct file *rf, *wf;
 469         struct pipe *rpipe, *wpipe;
 470         struct pipepair *pp;
 471         int fd, fflags, error;
 472
 473         error = pipe_paircreate(td, &pp);
 474         if (error != 0)
 475                 return (error);
 476         rpipe = &pp->pp_rpipe;
 477         wpipe = &pp->pp_wpipe;
 478         error = falloc_caps(td, &rf, &fd, flags, fcaps1);
 479         if (error) {
 480                 pipeclose(rpipe);
 481                 pipeclose(wpipe);
 482                 return (error);
 483         }
 484         /* An extra reference on `rf' has been held for us by falloc_caps(). */
 485         fildes[0] = fd;
 486
 487         fflags = FREAD | FWRITE;
 488         if ((flags & O_NONBLOCK) != 0)
 489                 fflags |= FNONBLOCK;
 490
 491         /*
 492          * Warning: once we've gotten past allocation of the fd for the
 493          * read-side, we can only drop the read side via fdrop() in order
 494          * to avoid races against processes which manage to dup() the read
 495          * side while we are blocked trying to allocate the write side.
 496          */
 497         finit(rf, fflags, DTYPE_PIPE, rpipe, &pipeops);
 498         error = falloc_caps(td, &wf, &fd, flags, fcaps2);
 499         if (error) {
 500                 fdclose(td, rf, fildes[0]);
 501                 fdrop(rf, td);
 502                 /* rpipe has been closed by fdrop(). */
 503                 pipeclose(wpipe);
 504                 return (error);
 505         }
 506         /* An extra reference on `wf' has been held for us by falloc_caps(). */
 507         finit(wf, fflags, DTYPE_PIPE, wpipe, &pipeops);
 508         fdrop(wf, td);
 509         fildes[1] = fd;
 510         fdrop(rf, td);
 511
 512         return (0);
 513 }
 514
 515 #ifdef COMPAT_FREEBSD10
 516 /* ARGSUSED */
 517 int
 518 freebsd10_pipe(struct thread *td, struct freebsd10_pipe_args *uap __unused)
 519 {
 520         int error;
 521         int fildes[2];
 522
 523         error = kern_pipe(td, fildes, 0, NULL, NULL);
 524         if (error)
 525                 return (error);
 526
 527         td->td_retval[0] = fildes[0];
 528         td->td_retval[1] = fildes[1];
 529
 530         return (0);
 531 }
 532 #endif
 533
 534 int
 535 sys_pipe2(struct thread *td, struct pipe2_args *uap)
 536 {
 537         int error, fildes[2];
 538
 539         if (uap->flags & ~(O_CLOEXEC | O_NONBLOCK))
 540                 return (EINVAL);
 541         error = kern_pipe(td, fildes, uap->flags, NULL, NULL);
 542         if (error)
 543                 return (error);
 544         error = copyout(fildes, uap->fildes, 2 * sizeof(int));
 545         if (error) {
 546                 (void)kern_close(td, fildes[0]);
 547                 (void)kern_close(td, fildes[1]);
 548         }
 549         return (error);
 550 }
 551
 552 /*
 553  * Allocate kva for pipe circular buffer, the space is pageable
 554  * This routine will 'realloc' the size of a pipe safely, if it fails
 555  * it will retain the old buffer.
 556  * If it fails it will return ENOMEM.
 557  */
 558 static int
 559 pipespace_new(struct pipe *cpipe, int size)
 560 {
 561         caddr_t buffer;
 562         int error, cnt, firstseg;
 563         static int curfail = 0;
 564         static struct timeval lastfail;
 565
 566         KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked"));
 567         KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW),
 568                 ("pipespace: resize of direct writes not allowed"));
 569 retry:
 570         cnt = cpipe->pipe_buffer.cnt;
 571         if (cnt > size)
 572                 size = cnt;
 573
 574         size = round_page(size);
 575         buffer = (caddr_t) vm_map_min(pipe_map);
 576
 577         error = vm_map_find(pipe_map, NULL, 0, (vm_offset_t *)&buffer, size, 0,
 578             VMFS_ANY_SPACE, VM_PROT_RW, VM_PROT_RW, 0);
 579         if (error != KERN_SUCCESS) {
 580                 if (cpipe->pipe_buffer.buffer == NULL &&
 581                     size > SMALL_PIPE_SIZE) {
 582                         size = SMALL_PIPE_SIZE;
 583                         pipefragretry++;
 584                         goto retry;
 585                 }
 586                 if (cpipe->pipe_buffer.buffer == NULL) {
 587                         pipeallocfail++;
 588                         if (ppsratecheck(&lastfail, &curfail, 1))
 589                                 printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n");
 590                 } else {
 591                         piperesizefail++;
 592                 }
 593                 return (ENOMEM);
 594         }
 595
 596         /* copy data, then free old resources if we're resizing */
 597         if (cnt > 0) {
 598                 if (cpipe->pipe_buffer.in <= cpipe->pipe_buffer.out) {
 599                         firstseg = cpipe->pipe_buffer.size - cpipe->pipe_buffer.out;
 600                         bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out],
 601                                 buffer, firstseg);
 602                         if ((cnt - firstseg) > 0)
 603                                 bcopy(cpipe->pipe_buffer.buffer, &buffer[firstseg],
 604                                         cpipe->pipe_buffer.in);
 605                 } else {
 606                         bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out],
 607                                 buffer, cnt);
 608                 }
 609         }
 610         pipe_free_kmem(cpipe);
 611         cpipe->pipe_buffer.buffer = buffer;
 612         cpipe->pipe_buffer.size = size;
 613         cpipe->pipe_buffer.in = cnt;
 614         cpipe->pipe_buffer.out = 0;
 615         cpipe->pipe_buffer.cnt = cnt;
 616         atomic_add_long(&amountpipekva, cpipe->pipe_buffer.size);
 617         return (0);
 618 }
 619
 620 /*
 621  * Wrapper for pipespace_new() that performs locking assertions.
 622  */
 623 static int
 624 pipespace(struct pipe *cpipe, int size)
 625 {
 626
 627         KASSERT(cpipe->pipe_state & PIPE_LOCKFL,
 628             ("Unlocked pipe passed to pipespace"));
 629         return (pipespace_new(cpipe, size));
 630 }
 631
 632 /*
 633  * lock a pipe for I/O, blocking other access
 634  */
 635 static __inline int
 636 pipelock(struct pipe *cpipe, int catch)
 637 {
 638         int error, prio;
 639
 640         PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
 641
 642         prio = PRIBIO;
 643         if (catch)
 644                 prio |= PCATCH;
 645         while (cpipe->pipe_state & PIPE_LOCKFL) {
 646                 KASSERT(cpipe->pipe_waiters >= 0,
 647                     ("%s: bad waiter count %d", __func__,
 648                     cpipe->pipe_waiters));
 649                 cpipe->pipe_waiters++;
 650                 error = msleep(&cpipe->pipe_waiters, PIPE_MTX(cpipe), prio,
 651                     "pipelk", 0);
 652                 cpipe->pipe_waiters--;
 653                 if (error != 0)
 654                         return (error);
 655         }
 656         cpipe->pipe_state |= PIPE_LOCKFL;
 657         return (0);
 658 }
 659
 660 /*
 661  * unlock a pipe I/O lock
 662  */
 663 static __inline void
 664 pipeunlock(struct pipe *cpipe)
 665 {
 666
 667         PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
 668         KASSERT(cpipe->pipe_state & PIPE_LOCKFL,
 669                 ("Unlocked pipe passed to pipeunlock"));
 670         KASSERT(cpipe->pipe_waiters >= 0,
 671             ("%s: bad waiter count %d", __func__,
 672             cpipe->pipe_waiters));
 673         cpipe->pipe_state &= ~PIPE_LOCKFL;
 674         if (cpipe->pipe_waiters > 0)
 675                 wakeup_one(&cpipe->pipe_waiters);
 676 }
 677
 678 void
 679 pipeselwakeup(struct pipe *cpipe)
 680 {
 681
 682         PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
 683         if (cpipe->pipe_state & PIPE_SEL) {
 684                 selwakeuppri(&cpipe->pipe_sel, PSOCK);
 685                 if (!SEL_WAITING(&cpipe->pipe_sel))
 686                         cpipe->pipe_state &= ~PIPE_SEL;
 687         }
 688         if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio)
 689                 pgsigio(&cpipe->pipe_sigio, SIGIO, 0);
 690         KNOTE_LOCKED(&cpipe->pipe_sel.si_note, 0);
 691 }
 692
 693 /*
 694  * Initialize and allocate VM and memory for pipe.  The structure
 695  * will start out zero'd from the ctor, so we just manage the kmem.
 696  */
 697 static int
 698 pipe_create(struct pipe *pipe, bool large_backing)
 699 {
 700         int error;
 701
 702         error = pipespace_new(pipe, !large_backing || amountpipekva >
 703             maxpipekva / 2 ? SMALL_PIPE_SIZE : PIPE_SIZE);
 704         if (error == 0)
 705                 pipe->pipe_ino = alloc_unr64(&pipeino_unr);
 706         return (error);
 707 }
 708
 709 /* ARGSUSED */
 710 static int
 711 pipe_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
 712     int flags, struct thread *td)
 713 {
 714         struct pipe *rpipe;
 715         int error;
 716         int nread = 0;
 717         int size;
 718
 719         rpipe = fp->f_data;
 720
 721         /*
 722          * Try to avoid locking the pipe if we have nothing to do.
 723          *
 724          * There are programs which share one pipe amongst multiple processes
 725          * and perform non-blocking reads in parallel, even if the pipe is
 726          * empty.  This in particular is the case with BSD make, which when
 727          * spawned with a high -j number can find itself with over half of the
 728          * calls failing to find anything.
 729          */
 730         if ((fp->f_flag & FNONBLOCK) != 0 && !mac_pipe_check_read_enabled()) {
 731                 if (__predict_false(uio->uio_resid == 0))
 732                         return (0);
 733                 if ((atomic_load_short(&rpipe->pipe_state) & PIPE_EOF) == 0 &&
 734                     atomic_load_int(&rpipe->pipe_buffer.cnt) == 0 &&
 735                     atomic_load_int(&rpipe->pipe_pages.cnt) == 0)
 736                         return (EAGAIN);
 737         }
 738
 739         PIPE_LOCK(rpipe);
 740         ++rpipe->pipe_busy;
 741         error = pipelock(rpipe, 1);
 742         if (error)
 743                 goto unlocked_error;
 744
 745 #ifdef MAC
 746         error = mac_pipe_check_read(active_cred, rpipe->pipe_pair);
 747         if (error)
 748                 goto locked_error;
 749 #endif
 750         if (amountpipekva > (3 * maxpipekva) / 4) {
 751                 if ((rpipe->pipe_state & PIPE_DIRECTW) == 0 &&
 752                     rpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&
 753                     rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&
 754                     piperesizeallowed == 1) {
 755                         PIPE_UNLOCK(rpipe);
 756                         pipespace(rpipe, SMALL_PIPE_SIZE);
 757                         PIPE_LOCK(rpipe);
 758                 }
 759         }
 760
 761         while (uio->uio_resid) {
 762                 /*
 763                  * normal pipe buffer receive
 764                  */
 765                 if (rpipe->pipe_buffer.cnt > 0) {
 766                         size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out;
 767                         if (size > rpipe->pipe_buffer.cnt)
 768                                 size = rpipe->pipe_buffer.cnt;
 769                         if (size > uio->uio_resid)
 770                                 size = uio->uio_resid;
 771
 772                         PIPE_UNLOCK(rpipe);
 773                         error = uiomove(
 774                             &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out],
 775                             size, uio);
 776                         PIPE_LOCK(rpipe);
 777                         if (error)
 778                                 break;
 779
 780                         rpipe->pipe_buffer.out += size;
 781                         if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size)
 782                                 rpipe->pipe_buffer.out = 0;
 783
 784                         rpipe->pipe_buffer.cnt -= size;
 785
 786                         /*
 787                          * If there is no more to read in the pipe, reset
 788                          * its pointers to the beginning.  This improves
 789                          * cache hit stats.
 790                          */
 791                         if (rpipe->pipe_buffer.cnt == 0) {
 792                                 rpipe->pipe_buffer.in = 0;
 793                                 rpipe->pipe_buffer.out = 0;
 794                         }
 795                         nread += size;
 796 #ifndef PIPE_NODIRECT
 797                 /*
 798                  * Direct copy, bypassing a kernel buffer.
 799                  */
 800                 } else if ((size = rpipe->pipe_pages.cnt) != 0) {
 801                         if (size > uio->uio_resid)
 802                                 size = (u_int) uio->uio_resid;
 803                         PIPE_UNLOCK(rpipe);
 804                         error = uiomove_fromphys(rpipe->pipe_pages.ms,
 805                             rpipe->pipe_pages.pos, size, uio);
 806                         PIPE_LOCK(rpipe);
 807                         if (error)
 808                                 break;
 809                         nread += size;
 810                         rpipe->pipe_pages.pos += size;
 811                         rpipe->pipe_pages.cnt -= size;
 812                         if (rpipe->pipe_pages.cnt == 0) {
 813                                 rpipe->pipe_state &= ~PIPE_WANTW;
 814                                 wakeup(rpipe);
 815                         }
 816 #endif
 817                 } else {
 818                         /*
 819                          * detect EOF condition
 820                          * read returns 0 on EOF, no need to set error
 821                          */
 822                         if (rpipe->pipe_state & PIPE_EOF)
 823                                 break;
 824
 825                         /*
 826                          * If the "write-side" has been blocked, wake it up now.
 827                          */
 828                         if (rpipe->pipe_state & PIPE_WANTW) {
 829                                 rpipe->pipe_state &= ~PIPE_WANTW;
 830                                 wakeup(rpipe);
 831                         }
 832
 833                         /*
 834                          * Break if some data was read.
 835                          */
 836                         if (nread > 0)
 837                                 break;
 838
 839                         /*
 840                          * Unlock the pipe buffer for our remaining processing.
 841                          * We will either break out with an error or we will
 842                          * sleep and relock to loop.
 843                          */
 844                         pipeunlock(rpipe);
 845
 846                         /*
 847                          * Handle non-blocking mode operation or
 848                          * wait for more data.
 849                          */
 850                         if (fp->f_flag & FNONBLOCK) {
 851                                 error = EAGAIN;
 852                         } else {
 853                                 rpipe->pipe_state |= PIPE_WANTR;
 854                                 if ((error = msleep(rpipe, PIPE_MTX(rpipe),
 855                                     PRIBIO | PCATCH,
 856                                     "piperd", 0)) == 0)
 857                                         error = pipelock(rpipe, 1);
 858                         }
 859                         if (error)
 860                                 goto unlocked_error;
 861                 }
 862         }
 863 #ifdef MAC
 864 locked_error:
 865 #endif
 866         pipeunlock(rpipe);
 867
 868         /* XXX: should probably do this before getting any locks. */
 869         if (error == 0)
 870                 pipe_timestamp(&rpipe->pipe_atime);
 871 unlocked_error:
 872         --rpipe->pipe_busy;
 873
 874         /*
 875          * PIPE_WANT processing only makes sense if pipe_busy is 0.
 876          */
 877         if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) {
 878                 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW);
 879                 wakeup(rpipe);
 880         } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) {
 881                 /*
 882                  * Handle write blocking hysteresis.
 883                  */
 884                 if (rpipe->pipe_state & PIPE_WANTW) {
 885                         rpipe->pipe_state &= ~PIPE_WANTW;
 886                         wakeup(rpipe);
 887                 }
 888         }
 889
 890         /*
 891          * Only wake up writers if there was actually something read.
 892          * Otherwise, when calling read(2) at EOF, a spurious wakeup occurs.
 893          */
 894         if (nread > 0 &&
 895             rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt >= PIPE_BUF)
 896                 pipeselwakeup(rpipe);
 897
 898         PIPE_UNLOCK(rpipe);
 899         if (nread > 0)
 900                 td->td_ru.ru_msgrcv++;
 901         return (error);
 902 }
 903
 904 #ifndef PIPE_NODIRECT
 905 /*
 906  * Map the sending processes' buffer into kernel space and wire it.
 907  * This is similar to a physical write operation.
 908  */
 909 static int
 910 pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio)
 911 {
 912         u_int size;
 913         int i;
 914
 915         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
 916         KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,
 917             ("%s: PIPE_DIRECTW set on %p", __func__, wpipe));
 918         KASSERT(wpipe->pipe_pages.cnt == 0,
 919             ("%s: pipe map for %p contains residual data", __func__, wpipe));
 920
 921         if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size)
 922                 size = wpipe->pipe_buffer.size;
 923         else
 924                 size = uio->uio_iov->iov_len;
 925
 926         wpipe->pipe_state |= PIPE_DIRECTW;
 927         PIPE_UNLOCK(wpipe);
 928         i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
 929             (vm_offset_t)uio->uio_iov->iov_base, size, VM_PROT_READ,
 930             wpipe->pipe_pages.ms, PIPENPAGES);
 931         PIPE_LOCK(wpipe);
 932         if (i < 0) {
 933                 wpipe->pipe_state &= ~PIPE_DIRECTW;
 934                 return (EFAULT);
 935         }
 936
 937         wpipe->pipe_pages.npages = i;
 938         wpipe->pipe_pages.pos =
 939             ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK;
 940         wpipe->pipe_pages.cnt = size;
 941
 942         uio->uio_iov->iov_len -= size;
 943         uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size;
 944         if (uio->uio_iov->iov_len == 0)
 945                 uio->uio_iov++;
 946         uio->uio_resid -= size;
 947         uio->uio_offset += size;
 948         return (0);
 949 }
 950
 951 /*
 952  * Unwire the process buffer.
 953  */
 954 static void
 955 pipe_destroy_write_buffer(struct pipe *wpipe)
 956 {
 957
 958         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
 959         KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,
 960             ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));
 961         KASSERT(wpipe->pipe_pages.cnt == 0,
 962             ("%s: pipe map for %p contains residual data", __func__, wpipe));
 963
 964         wpipe->pipe_state &= ~PIPE_DIRECTW;
 965         vm_page_unhold_pages(wpipe->pipe_pages.ms, wpipe->pipe_pages.npages);
 966         wpipe->pipe_pages.npages = 0;
 967 }
 968
 969 /*
 970  * In the case of a signal, the writing process might go away.  This
 971  * code copies the data into the circular buffer so that the source
 972  * pages can be freed without loss of data.
 973  */
 974 static void
 975 pipe_clone_write_buffer(struct pipe *wpipe)
 976 {
 977         struct uio uio;
 978         struct iovec iov;
 979         int size;
 980         int pos;
 981
 982         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
 983         KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,
 984             ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));
 985
 986         size = wpipe->pipe_pages.cnt;
 987         pos = wpipe->pipe_pages.pos;
 988         wpipe->pipe_pages.cnt = 0;
 989
 990         wpipe->pipe_buffer.in = size;
 991         wpipe->pipe_buffer.out = 0;
 992         wpipe->pipe_buffer.cnt = size;
 993
 994         PIPE_UNLOCK(wpipe);
 995         iov.iov_base = wpipe->pipe_buffer.buffer;
 996         iov.iov_len = size;
 997         uio.uio_iov = &iov;
 998         uio.uio_iovcnt = 1;
 999         uio.uio_offset = 0;
1000         uio.uio_resid = size;
1001         uio.uio_segflg = UIO_SYSSPACE;
1002         uio.uio_rw = UIO_READ;
1003         uio.uio_td = curthread;
1004         uiomove_fromphys(wpipe->pipe_pages.ms, pos, size, &uio);
1005         PIPE_LOCK(wpipe);
1006         pipe_destroy_write_buffer(wpipe);
1007 }
1008
1009 /*
1010  * This implements the pipe buffer write mechanism.  Note that only
1011  * a direct write OR a normal pipe write can be pending at any given time.
1012  * If there are any characters in the pipe buffer, the direct write will
1013  * be deferred until the receiving process grabs all of the bytes from
1014  * the pipe buffer.  Then the direct mapping write is set-up.
1015  */
1016 static int
1017 pipe_direct_write(struct pipe *wpipe, struct uio *uio)
1018 {
1019         int error;
1020
1021 retry:
1022         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
1023         if ((wpipe->pipe_state & PIPE_EOF) != 0) {
1024                 error = EPIPE;
1025                 goto error1;
1026         }
1027         if (wpipe->pipe_state & PIPE_DIRECTW) {
1028                 if (wpipe->pipe_state & PIPE_WANTR) {
1029                         wpipe->pipe_state &= ~PIPE_WANTR;
1030                         wakeup(wpipe);
1031                 }
1032                 pipeselwakeup(wpipe);
1033                 wpipe->pipe_state |= PIPE_WANTW;
1034                 pipeunlock(wpipe);
1035                 error = msleep(wpipe, PIPE_MTX(wpipe),
1036                     PRIBIO | PCATCH, "pipdww", 0);
1037                 pipelock(wpipe, 0);
1038                 if (error != 0)
1039                         goto error1;
1040                 goto retry;
1041         }
1042         if (wpipe->pipe_buffer.cnt > 0) {
1043                 if (wpipe->pipe_state & PIPE_WANTR) {
1044                         wpipe->pipe_state &= ~PIPE_WANTR;
1045                         wakeup(wpipe);
1046                 }
1047                 pipeselwakeup(wpipe);
1048                 wpipe->pipe_state |= PIPE_WANTW;
1049                 pipeunlock(wpipe);
1050                 error = msleep(wpipe, PIPE_MTX(wpipe),
1051                     PRIBIO | PCATCH, "pipdwc", 0);
1052                 pipelock(wpipe, 0);
1053                 if (error != 0)
1054                         goto error1;
1055                 goto retry;
1056         }
1057
1058         error = pipe_build_write_buffer(wpipe, uio);
1059         if (error) {
1060                 goto error1;
1061         }
1062
1063         while (wpipe->pipe_pages.cnt != 0 &&
1064             (wpipe->pipe_state & PIPE_EOF) == 0) {
1065                 if (wpipe->pipe_state & PIPE_WANTR) {
1066                         wpipe->pipe_state &= ~PIPE_WANTR;
1067                         wakeup(wpipe);
1068                 }
1069                 pipeselwakeup(wpipe);
1070                 wpipe->pipe_state |= PIPE_WANTW;
1071                 pipeunlock(wpipe);
1072                 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,
1073                     "pipdwt", 0);
1074                 pipelock(wpipe, 0);
1075                 if (error != 0)
1076                         break;
1077         }
1078
1079         if ((wpipe->pipe_state & PIPE_EOF) != 0) {
1080                 wpipe->pipe_pages.cnt = 0;
1081                 pipe_destroy_write_buffer(wpipe);
1082                 pipeselwakeup(wpipe);
1083                 error = EPIPE;
1084         } else if (error == EINTR || error == ERESTART) {
1085                 pipe_clone_write_buffer(wpipe);
1086         } else {
1087                 pipe_destroy_write_buffer(wpipe);
1088         }
1089         KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,
1090             ("pipe %p leaked PIPE_DIRECTW", wpipe));
1091         return (error);
1092
1093 error1:
1094         wakeup(wpipe);
1095         return (error);
1096 }
1097 #endif
1098
1099 static int
1100 pipe_write(struct file *fp, struct uio *uio, struct ucred *active_cred,
1101     int flags, struct thread *td)
1102 {
1103         struct pipe *wpipe, *rpipe;
1104         ssize_t orig_resid;
1105         int desiredsize, error;
1106
1107         rpipe = fp->f_data;
1108         wpipe = PIPE_PEER(rpipe);
1109         PIPE_LOCK(rpipe);
1110         error = pipelock(wpipe, 1);
1111         if (error) {
1112                 PIPE_UNLOCK(rpipe);
1113                 return (error);
1114         }
1115         /*
1116          * detect loss of pipe read side, issue SIGPIPE if lost.
1117          */
1118         if (wpipe->pipe_present != PIPE_ACTIVE ||
1119             (wpipe->pipe_state & PIPE_EOF)) {
1120                 pipeunlock(wpipe);
1121                 PIPE_UNLOCK(rpipe);
1122                 return (EPIPE);
1123         }
1124 #ifdef MAC
1125         error = mac_pipe_check_write(active_cred, wpipe->pipe_pair);
1126         if (error) {
1127                 pipeunlock(wpipe);
1128                 PIPE_UNLOCK(rpipe);
1129                 return (error);
1130         }
1131 #endif
1132         ++wpipe->pipe_busy;
1133
1134         /* Choose a larger size if it's advantageous */
1135         desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size);
1136         while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) {
1137                 if (piperesizeallowed != 1)
1138                         break;
1139                 if (amountpipekva > maxpipekva / 2)
1140                         break;
1141                 if (desiredsize == BIG_PIPE_SIZE)
1142                         break;
1143                 desiredsize = desiredsize * 2;
1144         }
1145
1146         /* Choose a smaller size if we're in a OOM situation */
1147         if (amountpipekva > (3 * maxpipekva) / 4 &&
1148             wpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&
1149             wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&
1150             piperesizeallowed == 1)
1151                 desiredsize = SMALL_PIPE_SIZE;
1152
1153         /* Resize if the above determined that a new size was necessary */
1154         if (desiredsize != wpipe->pipe_buffer.size &&
1155             (wpipe->pipe_state & PIPE_DIRECTW) == 0) {
1156                 PIPE_UNLOCK(wpipe);
1157                 pipespace(wpipe, desiredsize);
1158                 PIPE_LOCK(wpipe);
1159         }
1160         MPASS(wpipe->pipe_buffer.size != 0);
1161
1162         orig_resid = uio->uio_resid;
1163
1164         while (uio->uio_resid) {
1165                 int space;
1166
1167                 if (wpipe->pipe_state & PIPE_EOF) {
1168                         error = EPIPE;
1169                         break;
1170                 }
1171 #ifndef PIPE_NODIRECT
1172                 /*
1173                  * If the transfer is large, we can gain performance if
1174                  * we do process-to-process copies directly.
1175                  * If the write is non-blocking, we don't use the
1176                  * direct write mechanism.
1177                  *
1178                  * The direct write mechanism will detect the reader going
1179                  * away on us.
1180                  */
1181                 if (uio->uio_segflg == UIO_USERSPACE &&
1182                     uio->uio_iov->iov_len >= pipe_mindirect &&
1183                     wpipe->pipe_buffer.size >= pipe_mindirect &&
1184                     (fp->f_flag & FNONBLOCK) == 0) {
1185                         error = pipe_direct_write(wpipe, uio);
1186                         if (error != 0)
1187                                 break;
1188                         continue;
1189                 }
1190 #endif
1191
1192                 /*
1193                  * Pipe buffered writes cannot be coincidental with
1194                  * direct writes.  We wait until the currently executing
1195                  * direct write is completed before we start filling the
1196                  * pipe buffer.  We break out if a signal occurs or the
1197                  * reader goes away.
1198                  */
1199                 if (wpipe->pipe_pages.cnt != 0) {
1200                         if (wpipe->pipe_state & PIPE_WANTR) {
1201                                 wpipe->pipe_state &= ~PIPE_WANTR;
1202                                 wakeup(wpipe);
1203                         }
1204                         pipeselwakeup(wpipe);
1205                         wpipe->pipe_state |= PIPE_WANTW;
1206                         pipeunlock(wpipe);
1207                         error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,
1208                             "pipbww", 0);
1209                         pipelock(wpipe, 0);
1210                         if (error != 0)
1211                                 break;
1212                         continue;
1213                 }
1214
1215                 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
1216
1217                 /* Writes of size <= PIPE_BUF must be atomic. */
1218                 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF))
1219                         space = 0;
1220
1221                 if (space > 0) {
1222                         int size;       /* Transfer size */
1223                         int segsize;    /* first segment to transfer */
1224
1225                         /*
1226                          * Transfer size is minimum of uio transfer
1227                          * and free space in pipe buffer.
1228                          */
1229                         if (space > uio->uio_resid)
1230                                 size = uio->uio_resid;
1231                         else
1232                                 size = space;
1233                         /*
1234                          * First segment to transfer is minimum of
1235                          * transfer size and contiguous space in
1236                          * pipe buffer.  If first segment to transfer
1237                          * is less than the transfer size, we've got
1238                          * a wraparound in the buffer.
1239                          */
1240                         segsize = wpipe->pipe_buffer.size -
1241                                 wpipe->pipe_buffer.in;
1242                         if (segsize > size)
1243                                 segsize = size;
1244
1245                         /* Transfer first segment */
1246
1247                         PIPE_UNLOCK(rpipe);
1248                         error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],
1249                                         segsize, uio);
1250                         PIPE_LOCK(rpipe);
1251
1252                         if (error == 0 && segsize < size) {
1253                                 KASSERT(wpipe->pipe_buffer.in + segsize ==
1254                                         wpipe->pipe_buffer.size,
1255                                         ("Pipe buffer wraparound disappeared"));
1256                                 /*
1257                                  * Transfer remaining part now, to
1258                                  * support atomic writes.  Wraparound
1259                                  * happened.
1260                                  */
1261
1262                                 PIPE_UNLOCK(rpipe);
1263                                 error = uiomove(
1264                                     &wpipe->pipe_buffer.buffer[0],
1265                                     size - segsize, uio);
1266                                 PIPE_LOCK(rpipe);
1267                         }
1268                         if (error == 0) {
1269                                 wpipe->pipe_buffer.in += size;
1270                                 if (wpipe->pipe_buffer.in >=
1271                                     wpipe->pipe_buffer.size) {
1272                                         KASSERT(wpipe->pipe_buffer.in ==
1273                                                 size - segsize +
1274                                                 wpipe->pipe_buffer.size,
1275                                                 ("Expected wraparound bad"));
1276                                         wpipe->pipe_buffer.in = size - segsize;
1277                                 }
1278
1279                                 wpipe->pipe_buffer.cnt += size;
1280                                 KASSERT(wpipe->pipe_buffer.cnt <=
1281                                         wpipe->pipe_buffer.size,
1282                                         ("Pipe buffer overflow"));
1283                         }
1284                         if (error != 0)
1285                                 break;
1286                         continue;
1287                 } else {
1288                         /*
1289                          * If the "read-side" has been blocked, wake it up now.
1290                          */
1291                         if (wpipe->pipe_state & PIPE_WANTR) {
1292                                 wpipe->pipe_state &= ~PIPE_WANTR;
1293                                 wakeup(wpipe);
1294                         }
1295
1296                         /*
1297                          * don't block on non-blocking I/O
1298                          */
1299                         if (fp->f_flag & FNONBLOCK) {
1300                                 error = EAGAIN;
1301                                 break;
1302                         }
1303
1304                         /*
1305                          * We have no more space and have something to offer,
1306                          * wake up select/poll.
1307                          */
1308                         pipeselwakeup(wpipe);
1309
1310                         wpipe->pipe_state |= PIPE_WANTW;
1311                         pipeunlock(wpipe);
1312                         error = msleep(wpipe, PIPE_MTX(rpipe),
1313                             PRIBIO | PCATCH, "pipewr", 0);
1314                         pipelock(wpipe, 0);
1315                         if (error != 0)
1316                                 break;
1317                         continue;
1318                 }
1319         }
1320
1321         --wpipe->pipe_busy;
1322
1323         if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {
1324                 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
1325                 wakeup(wpipe);
1326         } else if (wpipe->pipe_buffer.cnt > 0) {
1327                 /*
1328                  * If we have put any characters in the buffer, we wake up
1329                  * the reader.
1330                  */
1331                 if (wpipe->pipe_state & PIPE_WANTR) {
1332                         wpipe->pipe_state &= ~PIPE_WANTR;
1333                         wakeup(wpipe);
1334                 }
1335         }
1336
1337         /*
1338          * Don't return EPIPE if any byte was written.
1339          * EINTR and other interrupts are handled by generic I/O layer.
1340          * Do not pretend that I/O succeeded for obvious user error
1341          * like EFAULT.
1342          */
1343         if (uio->uio_resid != orig_resid && error == EPIPE)
1344                 error = 0;
1345
1346         if (error == 0)
1347                 pipe_timestamp(&wpipe->pipe_mtime);
1348
1349         /*
1350          * We have something to offer,
1351          * wake up select/poll.
1352          */
1353         if (wpipe->pipe_buffer.cnt)
1354                 pipeselwakeup(wpipe);
1355
1356         pipeunlock(wpipe);
1357         PIPE_UNLOCK(rpipe);
1358         if (uio->uio_resid != orig_resid)
1359                 td->td_ru.ru_msgsnd++;
1360         return (error);
1361 }
1362
1363 /* ARGSUSED */
1364 static int
1365 pipe_truncate(struct file *fp, off_t length, struct ucred *active_cred,
1366     struct thread *td)
1367 {
1368         struct pipe *cpipe;
1369         int error;
1370
1371         cpipe = fp->f_data;
1372         if (cpipe->pipe_type & PIPE_TYPE_NAMED)
1373                 error = vnops.fo_truncate(fp, length, active_cred, td);
1374         else
1375                 error = invfo_truncate(fp, length, active_cred, td);
1376         return (error);
1377 }
1378
1379 /*
1380  * we implement a very minimal set of ioctls for compatibility with sockets.
1381  */
1382 static int
1383 pipe_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *active_cred,
1384     struct thread *td)
1385 {
1386         struct pipe *mpipe = fp->f_data;
1387         int error;
1388
1389         PIPE_LOCK(mpipe);
1390
1391 #ifdef MAC
1392         error = mac_pipe_check_ioctl(active_cred, mpipe->pipe_pair, cmd, data);
1393         if (error) {
1394                 PIPE_UNLOCK(mpipe);
1395                 return (error);
1396         }
1397 #endif
1398
1399         error = 0;
1400         switch (cmd) {
1401         case FIONBIO:
1402                 break;
1403
1404         case FIOASYNC:
1405                 if (*(int *)data) {
1406                         mpipe->pipe_state |= PIPE_ASYNC;
1407                 } else {
1408                         mpipe->pipe_state &= ~PIPE_ASYNC;
1409                 }
1410                 break;
1411
1412         case FIONREAD:
1413                 if (!(fp->f_flag & FREAD)) {
1414                         *(int *)data = 0;
1415                         PIPE_UNLOCK(mpipe);
1416                         return (0);
1417                 }
1418                 if (mpipe->pipe_pages.cnt != 0)
1419                         *(int *)data = mpipe->pipe_pages.cnt;
1420                 else
1421                         *(int *)data = mpipe->pipe_buffer.cnt;
1422                 break;
1423
1424         case FIOSETOWN:
1425                 PIPE_UNLOCK(mpipe);
1426                 error = fsetown(*(int *)data, &mpipe->pipe_sigio);
1427                 goto out_unlocked;
1428
1429         case FIOGETOWN:
1430                 *(int *)data = fgetown(&mpipe->pipe_sigio);
1431                 break;
1432
1433         /* This is deprecated, FIOSETOWN should be used instead. */
1434         case TIOCSPGRP:
1435                 PIPE_UNLOCK(mpipe);
1436                 error = fsetown(-(*(int *)data), &mpipe->pipe_sigio);
1437                 goto out_unlocked;
1438
1439         /* This is deprecated, FIOGETOWN should be used instead. */
1440         case TIOCGPGRP:
1441                 *(int *)data = -fgetown(&mpipe->pipe_sigio);
1442                 break;
1443
1444         default:
1445                 error = ENOTTY;
1446                 break;
1447         }
1448         PIPE_UNLOCK(mpipe);
1449 out_unlocked:
1450         return (error);
1451 }
1452
1453 static int
1454 pipe_poll(struct file *fp, int events, struct ucred *active_cred,
1455     struct thread *td)
1456 {
1457         struct pipe *rpipe;
1458         struct pipe *wpipe;
1459         int levents, revents;
1460 #ifdef MAC
1461         int error;
1462 #endif
1463
1464         revents = 0;
1465         rpipe = fp->f_data;
1466         wpipe = PIPE_PEER(rpipe);
1467         PIPE_LOCK(rpipe);
1468 #ifdef MAC
1469         error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair);
1470         if (error)
1471                 goto locked_error;
1472 #endif
1473         if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM))
1474                 if (rpipe->pipe_pages.cnt > 0 || rpipe->pipe_buffer.cnt > 0)
1475                         revents |= events & (POLLIN | POLLRDNORM);
1476
1477         if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM))
1478                 if (wpipe->pipe_present != PIPE_ACTIVE ||
1479                     (wpipe->pipe_state & PIPE_EOF) ||
1480                     ((wpipe->pipe_state & PIPE_DIRECTW) == 0 &&
1481                      ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF ||
1482                          wpipe->pipe_buffer.size == 0)))
1483                         revents |= events & (POLLOUT | POLLWRNORM);
1484
1485         levents = events &
1486             (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | POLLRDBAND);
1487         if (rpipe->pipe_type & PIPE_TYPE_NAMED && fp->f_flag & FREAD && levents &&
1488             fp->f_pipegen == rpipe->pipe_wgen)
1489                 events |= POLLINIGNEOF;
1490
1491         if ((events & POLLINIGNEOF) == 0) {
1492                 if (rpipe->pipe_state & PIPE_EOF) {
1493                         if (fp->f_flag & FREAD)
1494                                 revents |= (events & (POLLIN | POLLRDNORM));
1495                         if (wpipe->pipe_present != PIPE_ACTIVE ||
1496                             (wpipe->pipe_state & PIPE_EOF))
1497                                 revents |= POLLHUP;
1498                 }
1499         }
1500
1501         if (revents == 0) {
1502                 /*
1503                  * Add ourselves regardless of eventmask as we have to return
1504                  * POLLHUP even if it was not asked for.
1505                  */
1506                 if ((fp->f_flag & FREAD) != 0) {
1507                         selrecord(td, &rpipe->pipe_sel);
1508                         if (SEL_WAITING(&rpipe->pipe_sel))
1509                                 rpipe->pipe_state |= PIPE_SEL;
1510                 }
1511
1512                 if ((fp->f_flag & FWRITE) != 0 &&
1513                     wpipe->pipe_present == PIPE_ACTIVE) {
1514                         selrecord(td, &wpipe->pipe_sel);
1515                         if (SEL_WAITING(&wpipe->pipe_sel))
1516                                 wpipe->pipe_state |= PIPE_SEL;
1517                 }
1518         }
1519 #ifdef MAC
1520 locked_error:
1521 #endif
1522         PIPE_UNLOCK(rpipe);
1523
1524         return (revents);
1525 }
1526
1527 /*
1528  * We shouldn't need locks here as we're doing a read and this should
1529  * be a natural race.
1530  */
1531 static int
1532 pipe_stat(struct file *fp, struct stat *ub, struct ucred *active_cred)
1533 {
1534         struct pipe *pipe;
1535 #ifdef MAC
1536         int error;
1537 #endif
1538
1539         pipe = fp->f_data;
1540 #ifdef MAC
1541         if (mac_pipe_check_stat_enabled()) {
1542                 PIPE_LOCK(pipe);
1543                 error = mac_pipe_check_stat(active_cred, pipe->pipe_pair);
1544                 PIPE_UNLOCK(pipe);
1545                 if (error) {
1546                         return (error);
1547                 }
1548         }
1549 #endif
1550
1551         /* For named pipes ask the underlying filesystem. */
1552         if (pipe->pipe_type & PIPE_TYPE_NAMED) {
1553                 return (vnops.fo_stat(fp, ub, active_cred));
1554         }
1555
1556         bzero(ub, sizeof(*ub));
1557         ub->st_mode = S_IFIFO;
1558         ub->st_blksize = PAGE_SIZE;
1559         if (pipe->pipe_pages.cnt != 0)
1560                 ub->st_size = pipe->pipe_pages.cnt;
1561         else
1562                 ub->st_size = pipe->pipe_buffer.cnt;
1563         ub->st_blocks = howmany(ub->st_size, ub->st_blksize);
1564         ub->st_atim = pipe->pipe_atime;
1565         ub->st_mtim = pipe->pipe_mtime;
1566         ub->st_ctim = pipe->pipe_ctime;
1567         ub->st_uid = fp->f_cred->cr_uid;
1568         ub->st_gid = fp->f_cred->cr_gid;
1569         ub->st_dev = pipedev_ino;
1570         ub->st_ino = pipe->pipe_ino;
1571         /*
1572          * Left as 0: st_nlink, st_rdev, st_flags, st_gen.
1573          */
1574         return (0);
1575 }
1576
1577 /* ARGSUSED */
1578 static int
1579 pipe_close(struct file *fp, struct thread *td)
1580 {
1581
1582         if (fp->f_vnode != NULL)
1583                 return vnops.fo_close(fp, td);
1584         fp->f_ops = &badfileops;
1585         pipe_dtor(fp->f_data);
1586         fp->f_data = NULL;
1587         return (0);
1588 }
1589
1590 static int
1591 pipe_chmod(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td)
1592 {
1593         struct pipe *cpipe;
1594         int error;
1595
1596         cpipe = fp->f_data;
1597         if (cpipe->pipe_type & PIPE_TYPE_NAMED)
1598                 error = vn_chmod(fp, mode, active_cred, td);
1599         else
1600                 error = invfo_chmod(fp, mode, active_cred, td);
1601         return (error);
1602 }
1603
1604 static int
1605 pipe_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
1606     struct thread *td)
1607 {
1608         struct pipe *cpipe;
1609         int error;
1610
1611         cpipe = fp->f_data;
1612         if (cpipe->pipe_type & PIPE_TYPE_NAMED)
1613                 error = vn_chown(fp, uid, gid, active_cred, td);
1614         else
1615                 error = invfo_chown(fp, uid, gid, active_cred, td);
1616         return (error);
1617 }
1618
1619 static int
1620 pipe_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
1621 {
1622         struct pipe *pi;
1623
1624         if (fp->f_type == DTYPE_FIFO)
1625                 return (vn_fill_kinfo(fp, kif, fdp));
1626         kif->kf_type = KF_TYPE_PIPE;
1627         pi = fp->f_data;
1628         kif->kf_un.kf_pipe.kf_pipe_addr = (uintptr_t)pi;
1629         kif->kf_un.kf_pipe.kf_pipe_peer = (uintptr_t)pi->pipe_peer;
1630         kif->kf_un.kf_pipe.kf_pipe_buffer_cnt = pi->pipe_buffer.cnt;
1631         kif->kf_un.kf_pipe.kf_pipe_buffer_in = pi->pipe_buffer.in;
1632         kif->kf_un.kf_pipe.kf_pipe_buffer_out = pi->pipe_buffer.out;
1633         kif->kf_un.kf_pipe.kf_pipe_buffer_size = pi->pipe_buffer.size;
1634         return (0);
1635 }
1636
1637 static void
1638 pipe_free_kmem(struct pipe *cpipe)
1639 {
1640
1641         KASSERT(!mtx_owned(PIPE_MTX(cpipe)),
1642             ("pipe_free_kmem: pipe mutex locked"));
1643
1644         if (cpipe->pipe_buffer.buffer != NULL) {
1645                 atomic_subtract_long(&amountpipekva, cpipe->pipe_buffer.size);
1646                 vm_map_remove(pipe_map,
1647                     (vm_offset_t)cpipe->pipe_buffer.buffer,
1648                     (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size);
1649                 cpipe->pipe_buffer.buffer = NULL;
1650         }
1651 #ifndef PIPE_NODIRECT
1652         {
1653                 cpipe->pipe_pages.cnt = 0;
1654                 cpipe->pipe_pages.pos = 0;
1655                 cpipe->pipe_pages.npages = 0;
1656         }
1657 #endif
1658 }
1659
1660 /*
1661  * shutdown the pipe
1662  */
1663 static void
1664 pipeclose(struct pipe *cpipe)
1665 {
1666 #ifdef MAC
1667         struct pipepair *pp;
1668 #endif
1669         struct pipe *ppipe;
1670
1671         KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL"));
1672
1673         PIPE_LOCK(cpipe);
1674         pipelock(cpipe, 0);
1675 #ifdef MAC
1676         pp = cpipe->pipe_pair;
1677 #endif
1678
1679         /*
1680          * If the other side is blocked, wake it up saying that
1681          * we want to close it down.
1682          */
1683         cpipe->pipe_state |= PIPE_EOF;
1684         while (cpipe->pipe_busy) {
1685                 wakeup(cpipe);
1686                 cpipe->pipe_state |= PIPE_WANT;
1687                 pipeunlock(cpipe);
1688                 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0);
1689                 pipelock(cpipe, 0);
1690         }
1691
1692         pipeselwakeup(cpipe);
1693
1694         /*
1695          * Disconnect from peer, if any.
1696          */
1697         ppipe = cpipe->pipe_peer;
1698         if (ppipe->pipe_present == PIPE_ACTIVE) {
1699                 ppipe->pipe_state |= PIPE_EOF;
1700                 wakeup(ppipe);
1701                 pipeselwakeup(ppipe);
1702         }
1703
1704         /*
1705          * Mark this endpoint as free.  Release kmem resources.  We
1706          * don't mark this endpoint as unused until we've finished
1707          * doing that, or the pipe might disappear out from under
1708          * us.
1709          */
1710         PIPE_UNLOCK(cpipe);
1711         pipe_free_kmem(cpipe);
1712         PIPE_LOCK(cpipe);
1713         cpipe->pipe_present = PIPE_CLOSING;
1714         pipeunlock(cpipe);
1715
1716         /*
1717          * knlist_clear() may sleep dropping the PIPE_MTX. Set the
1718          * PIPE_FINALIZED, that allows other end to free the
1719          * pipe_pair, only after the knotes are completely dismantled.
1720          */
1721         knlist_clear(&cpipe->pipe_sel.si_note, 1);
1722         cpipe->pipe_present = PIPE_FINALIZED;
1723         seldrain(&cpipe->pipe_sel);
1724         knlist_destroy(&cpipe->pipe_sel.si_note);
1725
1726         /*
1727          * If both endpoints are now closed, release the memory for the
1728          * pipe pair.  If not, unlock.
1729          */
1730         if (ppipe->pipe_present == PIPE_FINALIZED) {
1731                 PIPE_UNLOCK(cpipe);
1732 #ifdef MAC
1733                 mac_pipe_destroy(pp);
1734 #endif
1735                 uma_zfree(pipe_zone, cpipe->pipe_pair);
1736         } else
1737                 PIPE_UNLOCK(cpipe);
1738 }
1739
1740 /*ARGSUSED*/
1741 static int
1742 pipe_kqfilter(struct file *fp, struct knote *kn)
1743 {
1744         struct pipe *cpipe;
1745
1746         /*
1747          * If a filter is requested that is not supported by this file
1748          * descriptor, don't return an error, but also don't ever generate an
1749          * event.
1750          */
1751         if ((kn->kn_filter == EVFILT_READ) && !(fp->f_flag & FREAD)) {
1752                 kn->kn_fop = &pipe_nfiltops;
1753                 return (0);
1754         }
1755         if ((kn->kn_filter == EVFILT_WRITE) && !(fp->f_flag & FWRITE)) {
1756                 kn->kn_fop = &pipe_nfiltops;
1757                 return (0);
1758         }
1759         cpipe = fp->f_data;
1760         PIPE_LOCK(cpipe);
1761         switch (kn->kn_filter) {
1762         case EVFILT_READ:
1763                 kn->kn_fop = &pipe_rfiltops;
1764                 break;
1765         case EVFILT_WRITE:
1766                 kn->kn_fop = &pipe_wfiltops;
1767                 if (cpipe->pipe_peer->pipe_present != PIPE_ACTIVE) {
1768                         /* other end of pipe has been closed */
1769                         PIPE_UNLOCK(cpipe);
1770                         return (EPIPE);
1771                 }
1772                 cpipe = PIPE_PEER(cpipe);
1773                 break;
1774         default:
1775                 if ((cpipe->pipe_type & PIPE_TYPE_NAMED) != 0) {
1776                         PIPE_UNLOCK(cpipe);
1777                         return (vnops.fo_kqfilter(fp, kn));
1778                 }
1779                 PIPE_UNLOCK(cpipe);
1780                 return (EINVAL);
1781         }
1782
1783         kn->kn_hook = cpipe;
1784         knlist_add(&cpipe->pipe_sel.si_note, kn, 1);
1785         PIPE_UNLOCK(cpipe);
1786         return (0);
1787 }
1788
1789 static void
1790 filt_pipedetach(struct knote *kn)
1791 {
1792         struct pipe *cpipe = kn->kn_hook;
1793
1794         PIPE_LOCK(cpipe);
1795         knlist_remove(&cpipe->pipe_sel.si_note, kn, 1);
1796         PIPE_UNLOCK(cpipe);
1797 }
1798
1799 /*ARGSUSED*/
1800 static int
1801 filt_piperead(struct knote *kn, long hint)
1802 {
1803         struct file *fp = kn->kn_fp;
1804         struct pipe *rpipe = kn->kn_hook;
1805
1806         PIPE_LOCK_ASSERT(rpipe, MA_OWNED);
1807         kn->kn_data = rpipe->pipe_buffer.cnt;
1808         if (kn->kn_data == 0)
1809                 kn->kn_data = rpipe->pipe_pages.cnt;
1810
1811         if ((rpipe->pipe_state & PIPE_EOF) != 0 &&
1812             ((rpipe->pipe_type & PIPE_TYPE_NAMED) == 0 ||
1813             fp->f_pipegen != rpipe->pipe_wgen)) {
1814                 kn->kn_flags |= EV_EOF;
1815                 return (1);
1816         }
1817         kn->kn_flags &= ~EV_EOF;
1818         return (kn->kn_data > 0);
1819 }
1820
1821 /*ARGSUSED*/
1822 static int
1823 filt_pipewrite(struct knote *kn, long hint)
1824 {
1825         struct pipe *wpipe = kn->kn_hook;
1826
1827         /*
1828          * If this end of the pipe is closed, the knote was removed from the
1829          * knlist and the list lock (i.e., the pipe lock) is therefore not held.
1830          */
1831         if (wpipe->pipe_present == PIPE_ACTIVE ||
1832             (wpipe->pipe_type & PIPE_TYPE_NAMED) != 0) {
1833                 PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
1834
1835                 if (wpipe->pipe_state & PIPE_DIRECTW) {
1836                         kn->kn_data = 0;
1837                 } else if (wpipe->pipe_buffer.size > 0) {
1838                         kn->kn_data = wpipe->pipe_buffer.size -
1839                             wpipe->pipe_buffer.cnt;
1840                 } else {
1841                         kn->kn_data = PIPE_BUF;
1842                 }
1843         }
1844
1845         if (wpipe->pipe_present != PIPE_ACTIVE ||
1846             (wpipe->pipe_state & PIPE_EOF)) {
1847                 kn->kn_flags |= EV_EOF;
1848                 return (1);
1849         }
1850         kn->kn_flags &= ~EV_EOF;
1851         return (kn->kn_data >= PIPE_BUF);
1852 }
1853
1854 static void
1855 filt_pipedetach_notsup(struct knote *kn)
1856 {
1857
1858 }
1859
1860 static int
1861 filt_pipenotsup(struct knote *kn, long hint)
1862 {
1863
1864         return (0);
1865 }