sys/kern/sys_pipe.c

   1 /*-
   2  * Copyright (c) 1996 John S. Dyson
   3  * Copyright (c) 2012 Giovanni Trematerra
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice immediately at the beginning of the file, without modification,
  11  *    this list of conditions, and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. Absolutely no warranty of function or purpose is made by the author
  16  *    John S. Dyson.
  17  * 4. Modifications may be freely made to this file if the above conditions
  18  *    are met.
  19  */
  20
  21 /*
  22  * This file contains a high-performance replacement for the socket-based
  23  * pipes scheme originally used in FreeBSD/4.4Lite.  It does not support
  24  * all features of sockets, but does do everything that pipes normally
  25  * do.
  26  */
  27
  28 /*
  29  * This code has two modes of operation, a small write mode and a large
  30  * write mode.  The small write mode acts like conventional pipes with
  31  * a kernel buffer.  If the buffer is less than PIPE_MINDIRECT, then the
  32  * "normal" pipe buffering is done.  If the buffer is between PIPE_MINDIRECT
  33  * and PIPE_SIZE in size, the sending process pins the underlying pages in
  34  * memory, and the receiving process copies directly from these pinned pages
  35  * in the sending process.
  36  *
  37  * If the sending process receives a signal, it is possible that it will
  38  * go away, and certainly its address space can change, because control
  39  * is returned back to the user-mode side.  In that case, the pipe code
  40  * arranges to copy the buffer supplied by the user process, to a pageable
  41  * kernel buffer, and the receiving process will grab the data from the
  42  * pageable kernel buffer.  Since signals don't happen all that often,
  43  * the copy operation is normally eliminated.
  44  *
  45  * The constant PIPE_MINDIRECT is chosen to make sure that buffering will
  46  * happen for small transfers so that the system will not spend all of
  47  * its time context switching.
  48  *
  49  * In order to limit the resource use of pipes, two sysctls exist:
  50  *
  51  * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable
  52  * address space available to us in pipe_map. This value is normally
  53  * autotuned, but may also be loader tuned.
  54  *
  55  * kern.ipc.pipekva - This read-only sysctl tracks the current amount of
  56  * memory in use by pipes.
  57  *
  58  * Based on how large pipekva is relative to maxpipekva, the following
  59  * will happen:
  60  *
  61  * 0% - 50%:
  62  *     New pipes are given 16K of memory backing, pipes may dynamically
  63  *     grow to as large as 64K where needed.
  64  * 50% - 75%:
  65  *     New pipes are given 4K (or PAGE_SIZE) of memory backing,
  66  *     existing pipes may NOT grow.
  67  * 75% - 100%:
  68  *     New pipes are given 4K (or PAGE_SIZE) of memory backing,
  69  *     existing pipes will be shrunk down to 4K whenever possible.
  70  *
  71  * Resizing may be disabled by setting kern.ipc.piperesizeallowed=0.  If
  72  * that is set,  the only resize that will occur is the 0 -> SMALL_PIPE_SIZE
  73  * resize which MUST occur for reverse-direction pipes when they are
  74  * first used.
  75  *
  76  * Additional information about the current state of pipes may be obtained
  77  * from kern.ipc.pipes, kern.ipc.pipefragretry, kern.ipc.pipeallocfail,
  78  * and kern.ipc.piperesizefail.
  79  *
  80  * Locking rules:  There are two locks present here:  A mutex, used via
  81  * PIPE_LOCK, and a flag, used via pipelock().  All locking is done via
  82  * the flag, as mutexes can not persist over uiomove.  The mutex
  83  * exists only to guard access to the flag, and is not in itself a
  84  * locking mechanism.  Also note that there is only a single mutex for
  85  * both directions of a pipe.
  86  *
  87  * As pipelock() may have to sleep before it can acquire the flag, it
  88  * is important to reread all data after a call to pipelock(); everything
  89  * in the structure may have changed.
  90  */
  91
  92 #include <sys/cdefs.h>
  93 __FBSDID("$FreeBSD$");
  94
  95 #include <sys/param.h>
  96 #include <sys/systm.h>
  97 #include <sys/conf.h>
  98 #include <sys/fcntl.h>
  99 #include <sys/file.h>
 100 #include <sys/filedesc.h>
 101 #include <sys/filio.h>
 102 #include <sys/kernel.h>
 103 #include <sys/lock.h>
 104 #include <sys/mutex.h>
 105 #include <sys/ttycom.h>
 106 #include <sys/stat.h>
 107 #include <sys/malloc.h>
 108 #include <sys/poll.h>
 109 #include <sys/selinfo.h>
 110 #include <sys/signalvar.h>
 111 #include <sys/syscallsubr.h>
 112 #include <sys/sysctl.h>
 113 #include <sys/sysproto.h>
 114 #include <sys/pipe.h>
 115 #include <sys/proc.h>
 116 #include <sys/vnode.h>
 117 #include <sys/uio.h>
 118 #include <sys/user.h>
 119 #include <sys/event.h>
 120
 121 #include <security/mac/mac_framework.h>
 122
 123 #include <vm/vm.h>
 124 #include <vm/vm_param.h>
 125 #include <vm/vm_object.h>
 126 #include <vm/vm_kern.h>
 127 #include <vm/vm_extern.h>
 128 #include <vm/pmap.h>
 129 #include <vm/vm_map.h>
 130 #include <vm/vm_page.h>
 131 #include <vm/uma.h>
 132
 133 /*
 134  * Use this define if you want to disable *fancy* VM things.  Expect an
 135  * approx 30% decrease in transfer rate.  This could be useful for
 136  * NetBSD or OpenBSD.
 137  */
 138 /* #define PIPE_NODIRECT */
 139
 140 #define PIPE_PEER(pipe) \
 141         (((pipe)->pipe_state & PIPE_NAMED) ? (pipe) : ((pipe)->pipe_peer))
 142
 143 /*
 144  * interfaces to the outside world
 145  */
 146 static fo_rdwr_t        pipe_read;
 147 static fo_rdwr_t        pipe_write;
 148 static fo_truncate_t    pipe_truncate;
 149 static fo_ioctl_t       pipe_ioctl;
 150 static fo_poll_t        pipe_poll;
 151 static fo_kqfilter_t    pipe_kqfilter;
 152 static fo_stat_t        pipe_stat;
 153 static fo_close_t       pipe_close;
 154 static fo_chmod_t       pipe_chmod;
 155 static fo_chown_t       pipe_chown;
 156 static fo_fill_kinfo_t  pipe_fill_kinfo;
 157
 158 struct fileops pipeops = {
 159         .fo_read = pipe_read,
 160         .fo_write = pipe_write,
 161         .fo_truncate = pipe_truncate,
 162         .fo_ioctl = pipe_ioctl,
 163         .fo_poll = pipe_poll,
 164         .fo_kqfilter = pipe_kqfilter,
 165         .fo_stat = pipe_stat,
 166         .fo_close = pipe_close,
 167         .fo_chmod = pipe_chmod,
 168         .fo_chown = pipe_chown,
 169         .fo_sendfile = invfo_sendfile,
 170         .fo_fill_kinfo = pipe_fill_kinfo,
 171         .fo_flags = DFLAG_PASSABLE
 172 };
 173
 174 static void     filt_pipedetach(struct knote *kn);
 175 static void     filt_pipedetach_notsup(struct knote *kn);
 176 static int      filt_pipenotsup(struct knote *kn, long hint);
 177 static int      filt_piperead(struct knote *kn, long hint);
 178 static int      filt_pipewrite(struct knote *kn, long hint);
 179
 180 static struct filterops pipe_nfiltops = {
 181         .f_isfd = 1,
 182         .f_detach = filt_pipedetach_notsup,
 183         .f_event = filt_pipenotsup
 184 };
 185 static struct filterops pipe_rfiltops = {
 186         .f_isfd = 1,
 187         .f_detach = filt_pipedetach,
 188         .f_event = filt_piperead
 189 };
 190 static struct filterops pipe_wfiltops = {
 191         .f_isfd = 1,
 192         .f_detach = filt_pipedetach,
 193         .f_event = filt_pipewrite
 194 };
 195
 196 /*
 197  * Default pipe buffer size(s), this can be kind-of large now because pipe
 198  * space is pageable.  The pipe code will try to maintain locality of
 199  * reference for performance reasons, so small amounts of outstanding I/O
 200  * will not wipe the cache.
 201  */
 202 #define MINPIPESIZE (PIPE_SIZE/3)
 203 #define MAXPIPESIZE (2*PIPE_SIZE/3)
 204
 205 static long amountpipekva;
 206 static int pipefragretry;
 207 static int pipeallocfail;
 208 static int piperesizefail;
 209 static int piperesizeallowed = 1;
 210
 211 SYSCTL_LONG(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
 212            &maxpipekva, 0, "Pipe KVA limit");
 213 SYSCTL_LONG(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD,
 214            &amountpipekva, 0, "Pipe KVA usage");
 215 SYSCTL_INT(_kern_ipc, OID_AUTO, pipefragretry, CTLFLAG_RD,
 216           &pipefragretry, 0, "Pipe allocation retries due to fragmentation");
 217 SYSCTL_INT(_kern_ipc, OID_AUTO, pipeallocfail, CTLFLAG_RD,
 218           &pipeallocfail, 0, "Pipe allocation failures");
 219 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizefail, CTLFLAG_RD,
 220           &piperesizefail, 0, "Pipe resize failures");
 221 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizeallowed, CTLFLAG_RW,
 222           &piperesizeallowed, 0, "Pipe resizing allowed");
 223
 224 static void pipeinit(void *dummy __unused);
 225 static void pipeclose(struct pipe *cpipe);
 226 static void pipe_free_kmem(struct pipe *cpipe);
 227 static void pipe_create(struct pipe *pipe, int backing);
 228 static void pipe_paircreate(struct thread *td, struct pipepair **p_pp);
 229 static __inline int pipelock(struct pipe *cpipe, int catch);
 230 static __inline void pipeunlock(struct pipe *cpipe);
 231 #ifndef PIPE_NODIRECT
 232 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
 233 static void pipe_destroy_write_buffer(struct pipe *wpipe);
 234 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
 235 static void pipe_clone_write_buffer(struct pipe *wpipe);
 236 #endif
 237 static int pipespace(struct pipe *cpipe, int size);
 238 static int pipespace_new(struct pipe *cpipe, int size);
 239
 240 static int      pipe_zone_ctor(void *mem, int size, void *arg, int flags);
 241 static int      pipe_zone_init(void *mem, int size, int flags);
 242 static void     pipe_zone_fini(void *mem, int size);
 243
 244 static uma_zone_t pipe_zone;
 245 static struct unrhdr *pipeino_unr;
 246 static dev_t pipedev_ino;
 247
 248 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL);
 249
 250 static void
 251 pipeinit(void *dummy __unused)
 252 {
 253
 254         pipe_zone = uma_zcreate("pipe", sizeof(struct pipepair),
 255             pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini,
 256             UMA_ALIGN_PTR, 0);
 257         KASSERT(pipe_zone != NULL, ("pipe_zone not initialized"));
 258         pipeino_unr = new_unrhdr(1, INT32_MAX, NULL);
 259         KASSERT(pipeino_unr != NULL, ("pipe fake inodes not initialized"));
 260         pipedev_ino = devfs_alloc_cdp_inode();
 261         KASSERT(pipedev_ino > 0, ("pipe dev inode not initialized"));
 262 }
 263
 264 static int
 265 pipe_zone_ctor(void *mem, int size, void *arg, int flags)
 266 {
 267         struct pipepair *pp;
 268         struct pipe *rpipe, *wpipe;
 269
 270         KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size"));
 271
 272         pp = (struct pipepair *)mem;
 273
 274         /*
 275          * We zero both pipe endpoints to make sure all the kmem pointers
 276          * are NULL, flag fields are zero'd, etc.  We timestamp both
 277          * endpoints with the same time.
 278          */
 279         rpipe = &pp->pp_rpipe;
 280         bzero(rpipe, sizeof(*rpipe));
 281         vfs_timestamp(&rpipe->pipe_ctime);
 282         rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime;
 283
 284         wpipe = &pp->pp_wpipe;
 285         bzero(wpipe, sizeof(*wpipe));
 286         wpipe->pipe_ctime = rpipe->pipe_ctime;
 287         wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;
 288
 289         rpipe->pipe_peer = wpipe;
 290         rpipe->pipe_pair = pp;
 291         wpipe->pipe_peer = rpipe;
 292         wpipe->pipe_pair = pp;
 293
 294         /*
 295          * Mark both endpoints as present; they will later get free'd
 296          * one at a time.  When both are free'd, then the whole pair
 297          * is released.
 298          */
 299         rpipe->pipe_present = PIPE_ACTIVE;
 300         wpipe->pipe_present = PIPE_ACTIVE;
 301
 302         /*
 303          * Eventually, the MAC Framework may initialize the label
 304          * in ctor or init, but for now we do it elswhere to avoid
 305          * blocking in ctor or init.
 306          */
 307         pp->pp_label = NULL;
 308
 309         return (0);
 310 }
 311
 312 static int
 313 pipe_zone_init(void *mem, int size, int flags)
 314 {
 315         struct pipepair *pp;
 316
 317         KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size"));
 318
 319         pp = (struct pipepair *)mem;
 320
 321         mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_NEW);
 322         return (0);
 323 }
 324
 325 static void
 326 pipe_zone_fini(void *mem, int size)
 327 {
 328         struct pipepair *pp;
 329
 330         KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size"));
 331
 332         pp = (struct pipepair *)mem;
 333
 334         mtx_destroy(&pp->pp_mtx);
 335 }
 336
 337 static void
 338 pipe_paircreate(struct thread *td, struct pipepair **p_pp)
 339 {
 340         struct pipepair *pp;
 341         struct pipe *rpipe, *wpipe;
 342
 343         *p_pp = pp = uma_zalloc(pipe_zone, M_WAITOK);
 344 #ifdef MAC
 345         /*
 346          * The MAC label is shared between the connected endpoints.  As a
 347          * result mac_pipe_init() and mac_pipe_create() are called once
 348          * for the pair, and not on the endpoints.
 349          */
 350         mac_pipe_init(pp);
 351         mac_pipe_create(td->td_ucred, pp);
 352 #endif
 353         rpipe = &pp->pp_rpipe;
 354         wpipe = &pp->pp_wpipe;
 355
 356         knlist_init_mtx(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe));
 357         knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe));
 358
 359         /* Only the forward direction pipe is backed by default */
 360         pipe_create(rpipe, 1);
 361         pipe_create(wpipe, 0);
 362
 363         rpipe->pipe_state |= PIPE_DIRECTOK;
 364         wpipe->pipe_state |= PIPE_DIRECTOK;
 365 }
 366
 367 void
 368 pipe_named_ctor(struct pipe **ppipe, struct thread *td)
 369 {
 370         struct pipepair *pp;
 371
 372         pipe_paircreate(td, &pp);
 373         pp->pp_rpipe.pipe_state |= PIPE_NAMED;
 374         *ppipe = &pp->pp_rpipe;
 375 }
 376
 377 void
 378 pipe_dtor(struct pipe *dpipe)
 379 {
 380         struct pipe *peer;
 381         ino_t ino;
 382
 383         ino = dpipe->pipe_ino;
 384         peer = (dpipe->pipe_state & PIPE_NAMED) != 0 ? dpipe->pipe_peer : NULL;
 385         funsetown(&dpipe->pipe_sigio);
 386         pipeclose(dpipe);
 387         if (peer != NULL) {
 388                 funsetown(&peer->pipe_sigio);
 389                 pipeclose(peer);
 390         }
 391         if (ino != 0 && ino != (ino_t)-1)
 392                 free_unr(pipeino_unr, ino);
 393 }
 394
 395 /*
 396  * The pipe system call for the DTYPE_PIPE type of pipes.  If we fail, let
 397  * the zone pick up the pieces via pipeclose().
 398  */
 399 int
 400 kern_pipe(struct thread *td, int fildes[2], int flags, struct filecaps *fcaps1,
 401     struct filecaps *fcaps2)
 402 {
 403         struct file *rf, *wf;
 404         struct pipe *rpipe, *wpipe;
 405         struct pipepair *pp;
 406         int fd, fflags, error;
 407
 408         pipe_paircreate(td, &pp);
 409         rpipe = &pp->pp_rpipe;
 410         wpipe = &pp->pp_wpipe;
 411         error = falloc_caps(td, &rf, &fd, flags, fcaps1);
 412         if (error) {
 413                 pipeclose(rpipe);
 414                 pipeclose(wpipe);
 415                 return (error);
 416         }
 417         /* An extra reference on `rf' has been held for us by falloc_caps(). */
 418         fildes[0] = fd;
 419
 420         fflags = FREAD | FWRITE;
 421         if ((flags & O_NONBLOCK) != 0)
 422                 fflags |= FNONBLOCK;
 423
 424         /*
 425          * Warning: once we've gotten past allocation of the fd for the
 426          * read-side, we can only drop the read side via fdrop() in order
 427          * to avoid races against processes which manage to dup() the read
 428          * side while we are blocked trying to allocate the write side.
 429          */
 430         finit(rf, fflags, DTYPE_PIPE, rpipe, &pipeops);
 431         error = falloc_caps(td, &wf, &fd, flags, fcaps2);
 432         if (error) {
 433                 fdclose(td, rf, fildes[0]);
 434                 fdrop(rf, td);
 435                 /* rpipe has been closed by fdrop(). */
 436                 pipeclose(wpipe);
 437                 return (error);
 438         }
 439         /* An extra reference on `wf' has been held for us by falloc_caps(). */
 440         finit(wf, fflags, DTYPE_PIPE, wpipe, &pipeops);
 441         fdrop(wf, td);
 442         fildes[1] = fd;
 443         fdrop(rf, td);
 444
 445         return (0);
 446 }
 447
 448 /* ARGSUSED */
 449 int
 450 sys_pipe(struct thread *td, struct pipe_args *uap)
 451 {
 452         int error;
 453         int fildes[2];
 454
 455         error = kern_pipe(td, fildes, 0, NULL, NULL);
 456         if (error)
 457                 return (error);
 458
 459         td->td_retval[0] = fildes[0];
 460         td->td_retval[1] = fildes[1];
 461
 462         return (0);
 463 }
 464
 465 int
 466 sys_pipe2(struct thread *td, struct pipe2_args *uap)
 467 {
 468         int error, fildes[2];
 469
 470         if (uap->flags & ~(O_CLOEXEC | O_NONBLOCK))
 471                 return (EINVAL);
 472         error = kern_pipe(td, fildes, uap->flags, NULL, NULL);
 473         if (error)
 474                 return (error);
 475         error = copyout(fildes, uap->fildes, 2 * sizeof(int));
 476         if (error) {
 477                 (void)kern_close(td, fildes[0]);
 478                 (void)kern_close(td, fildes[1]);
 479         }
 480         return (error);
 481 }
 482
 483 /*
 484  * Allocate kva for pipe circular buffer, the space is pageable
 485  * This routine will 'realloc' the size of a pipe safely, if it fails
 486  * it will retain the old buffer.
 487  * If it fails it will return ENOMEM.
 488  */
 489 static int
 490 pipespace_new(cpipe, size)
 491         struct pipe *cpipe;
 492         int size;
 493 {
 494         caddr_t buffer;
 495         int error, cnt, firstseg;
 496         static int curfail = 0;
 497         static struct timeval lastfail;
 498
 499         KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked"));
 500         KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW),
 501                 ("pipespace: resize of direct writes not allowed"));
 502 retry:
 503         cnt = cpipe->pipe_buffer.cnt;
 504         if (cnt > size)
 505                 size = cnt;
 506
 507         size = round_page(size);
 508         buffer = (caddr_t) vm_map_min(pipe_map);
 509
 510         error = vm_map_find(pipe_map, NULL, 0,
 511                 (vm_offset_t *) &buffer, size, 0, VMFS_ANY_SPACE,
 512                 VM_PROT_ALL, VM_PROT_ALL, 0);
 513         if (error != KERN_SUCCESS) {
 514                 if ((cpipe->pipe_buffer.buffer == NULL) &&
 515                         (size > SMALL_PIPE_SIZE)) {
 516                         size = SMALL_PIPE_SIZE;
 517                         pipefragretry++;
 518                         goto retry;
 519                 }
 520                 if (cpipe->pipe_buffer.buffer == NULL) {
 521                         pipeallocfail++;
 522                         if (ppsratecheck(&lastfail, &curfail, 1))
 523                                 printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n");
 524                 } else {
 525                         piperesizefail++;
 526                 }
 527                 return (ENOMEM);
 528         }
 529
 530         /* copy data, then free old resources if we're resizing */
 531         if (cnt > 0) {
 532                 if (cpipe->pipe_buffer.in <= cpipe->pipe_buffer.out) {
 533                         firstseg = cpipe->pipe_buffer.size - cpipe->pipe_buffer.out;
 534                         bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out],
 535                                 buffer, firstseg);
 536                         if ((cnt - firstseg) > 0)
 537                                 bcopy(cpipe->pipe_buffer.buffer, &buffer[firstseg],
 538                                         cpipe->pipe_buffer.in);
 539                 } else {
 540                         bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out],
 541                                 buffer, cnt);
 542                 }
 543         }
 544         pipe_free_kmem(cpipe);
 545         cpipe->pipe_buffer.buffer = buffer;
 546         cpipe->pipe_buffer.size = size;
 547         cpipe->pipe_buffer.in = cnt;
 548         cpipe->pipe_buffer.out = 0;
 549         cpipe->pipe_buffer.cnt = cnt;
 550         atomic_add_long(&amountpipekva, cpipe->pipe_buffer.size);
 551         return (0);
 552 }
 553
 554 /*
 555  * Wrapper for pipespace_new() that performs locking assertions.
 556  */
 557 static int
 558 pipespace(cpipe, size)
 559         struct pipe *cpipe;
 560         int size;
 561 {
 562
 563         KASSERT(cpipe->pipe_state & PIPE_LOCKFL,
 564                 ("Unlocked pipe passed to pipespace"));
 565         return (pipespace_new(cpipe, size));
 566 }
 567
 568 /*
 569  * lock a pipe for I/O, blocking other access
 570  */
 571 static __inline int
 572 pipelock(cpipe, catch)
 573         struct pipe *cpipe;
 574         int catch;
 575 {
 576         int error;
 577
 578         PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
 579         while (cpipe->pipe_state & PIPE_LOCKFL) {
 580                 cpipe->pipe_state |= PIPE_LWANT;
 581                 error = msleep(cpipe, PIPE_MTX(cpipe),
 582                     catch ? (PRIBIO | PCATCH) : PRIBIO,
 583                     "pipelk", 0);
 584                 if (error != 0)
 585                         return (error);
 586         }
 587         cpipe->pipe_state |= PIPE_LOCKFL;
 588         return (0);
 589 }
 590
 591 /*
 592  * unlock a pipe I/O lock
 593  */
 594 static __inline void
 595 pipeunlock(cpipe)
 596         struct pipe *cpipe;
 597 {
 598
 599         PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
 600         KASSERT(cpipe->pipe_state & PIPE_LOCKFL,
 601                 ("Unlocked pipe passed to pipeunlock"));
 602         cpipe->pipe_state &= ~PIPE_LOCKFL;
 603         if (cpipe->pipe_state & PIPE_LWANT) {
 604                 cpipe->pipe_state &= ~PIPE_LWANT;
 605                 wakeup(cpipe);
 606         }
 607 }
 608
 609 void
 610 pipeselwakeup(cpipe)
 611         struct pipe *cpipe;
 612 {
 613
 614         PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
 615         if (cpipe->pipe_state & PIPE_SEL) {
 616                 selwakeuppri(&cpipe->pipe_sel, PSOCK);
 617                 if (!SEL_WAITING(&cpipe->pipe_sel))
 618                         cpipe->pipe_state &= ~PIPE_SEL;
 619         }
 620         if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio)
 621                 pgsigio(&cpipe->pipe_sigio, SIGIO, 0);
 622         KNOTE_LOCKED(&cpipe->pipe_sel.si_note, 0);
 623 }
 624
 625 /*
 626  * Initialize and allocate VM and memory for pipe.  The structure
 627  * will start out zero'd from the ctor, so we just manage the kmem.
 628  */
 629 static void
 630 pipe_create(pipe, backing)
 631         struct pipe *pipe;
 632         int backing;
 633 {
 634
 635         if (backing) {
 636                 /*
 637                  * Note that these functions can fail if pipe map is exhausted
 638                  * (as a result of too many pipes created), but we ignore the
 639                  * error as it is not fatal and could be provoked by
 640                  * unprivileged users. The only consequence is worse performance
 641                  * with given pipe.
 642                  */
 643                 if (amountpipekva > maxpipekva / 2)
 644                         (void)pipespace_new(pipe, SMALL_PIPE_SIZE);
 645                 else
 646                         (void)pipespace_new(pipe, PIPE_SIZE);
 647         }
 648
 649         pipe->pipe_ino = -1;
 650 }
 651
 652 /* ARGSUSED */
 653 static int
 654 pipe_read(fp, uio, active_cred, flags, td)
 655         struct file *fp;
 656         struct uio *uio;
 657         struct ucred *active_cred;
 658         struct thread *td;
 659         int flags;
 660 {
 661         struct pipe *rpipe;
 662         int error;
 663         int nread = 0;
 664         int size;
 665
 666         rpipe = fp->f_data;
 667         PIPE_LOCK(rpipe);
 668         ++rpipe->pipe_busy;
 669         error = pipelock(rpipe, 1);
 670         if (error)
 671                 goto unlocked_error;
 672
 673 #ifdef MAC
 674         error = mac_pipe_check_read(active_cred, rpipe->pipe_pair);
 675         if (error)
 676                 goto locked_error;
 677 #endif
 678         if (amountpipekva > (3 * maxpipekva) / 4) {
 679                 if (!(rpipe->pipe_state & PIPE_DIRECTW) &&
 680                         (rpipe->pipe_buffer.size > SMALL_PIPE_SIZE) &&
 681                         (rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) &&
 682                         (piperesizeallowed == 1)) {
 683                         PIPE_UNLOCK(rpipe);
 684                         pipespace(rpipe, SMALL_PIPE_SIZE);
 685                         PIPE_LOCK(rpipe);
 686                 }
 687         }
 688
 689         while (uio->uio_resid) {
 690                 /*
 691                  * normal pipe buffer receive
 692                  */
 693                 if (rpipe->pipe_buffer.cnt > 0) {
 694                         size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out;
 695                         if (size > rpipe->pipe_buffer.cnt)
 696                                 size = rpipe->pipe_buffer.cnt;
 697                         if (size > uio->uio_resid)
 698                                 size = uio->uio_resid;
 699
 700                         PIPE_UNLOCK(rpipe);
 701                         error = uiomove(
 702                             &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out],
 703                             size, uio);
 704                         PIPE_LOCK(rpipe);
 705                         if (error)
 706                                 break;
 707
 708                         rpipe->pipe_buffer.out += size;
 709                         if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size)
 710                                 rpipe->pipe_buffer.out = 0;
 711
 712                         rpipe->pipe_buffer.cnt -= size;
 713
 714                         /*
 715                          * If there is no more to read in the pipe, reset
 716                          * its pointers to the beginning.  This improves
 717                          * cache hit stats.
 718                          */
 719                         if (rpipe->pipe_buffer.cnt == 0) {
 720                                 rpipe->pipe_buffer.in = 0;
 721                                 rpipe->pipe_buffer.out = 0;
 722                         }
 723                         nread += size;
 724 #ifndef PIPE_NODIRECT
 725                 /*
 726                  * Direct copy, bypassing a kernel buffer.
 727                  */
 728                 } else if ((size = rpipe->pipe_map.cnt) &&
 729                            (rpipe->pipe_state & PIPE_DIRECTW)) {
 730                         if (size > uio->uio_resid)
 731                                 size = (u_int) uio->uio_resid;
 732
 733                         PIPE_UNLOCK(rpipe);
 734                         error = uiomove_fromphys(rpipe->pipe_map.ms,
 735                             rpipe->pipe_map.pos, size, uio);
 736                         PIPE_LOCK(rpipe);
 737                         if (error)
 738                                 break;
 739                         nread += size;
 740                         rpipe->pipe_map.pos += size;
 741                         rpipe->pipe_map.cnt -= size;
 742                         if (rpipe->pipe_map.cnt == 0) {
 743                                 rpipe->pipe_state &= ~(PIPE_DIRECTW|PIPE_WANTW);
 744                                 wakeup(rpipe);
 745                         }
 746 #endif
 747                 } else {
 748                         /*
 749                          * detect EOF condition
 750                          * read returns 0 on EOF, no need to set error
 751                          */
 752                         if (rpipe->pipe_state & PIPE_EOF)
 753                                 break;
 754
 755                         /*
 756                          * If the "write-side" has been blocked, wake it up now.
 757                          */
 758                         if (rpipe->pipe_state & PIPE_WANTW) {
 759                                 rpipe->pipe_state &= ~PIPE_WANTW;
 760                                 wakeup(rpipe);
 761                         }
 762
 763                         /*
 764                          * Break if some data was read.
 765                          */
 766                         if (nread > 0)
 767                                 break;
 768
 769                         /*
 770                          * Unlock the pipe buffer for our remaining processing.
 771                          * We will either break out with an error or we will
 772                          * sleep and relock to loop.
 773                          */
 774                         pipeunlock(rpipe);
 775
 776                         /*
 777                          * Handle non-blocking mode operation or
 778                          * wait for more data.
 779                          */
 780                         if (fp->f_flag & FNONBLOCK) {
 781                                 error = EAGAIN;
 782                         } else {
 783                                 rpipe->pipe_state |= PIPE_WANTR;
 784                                 if ((error = msleep(rpipe, PIPE_MTX(rpipe),
 785                                     PRIBIO | PCATCH,
 786                                     "piperd", 0)) == 0)
 787                                         error = pipelock(rpipe, 1);
 788                         }
 789                         if (error)
 790                                 goto unlocked_error;
 791                 }
 792         }
 793 #ifdef MAC
 794 locked_error:
 795 #endif
 796         pipeunlock(rpipe);
 797
 798         /* XXX: should probably do this before getting any locks. */
 799         if (error == 0)
 800                 vfs_timestamp(&rpipe->pipe_atime);
 801 unlocked_error:
 802         --rpipe->pipe_busy;
 803
 804         /*
 805          * PIPE_WANT processing only makes sense if pipe_busy is 0.
 806          */
 807         if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) {
 808                 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW);
 809                 wakeup(rpipe);
 810         } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) {
 811                 /*
 812                  * Handle write blocking hysteresis.
 813                  */
 814                 if (rpipe->pipe_state & PIPE_WANTW) {
 815                         rpipe->pipe_state &= ~PIPE_WANTW;
 816                         wakeup(rpipe);
 817                 }
 818         }
 819
 820         if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF)
 821                 pipeselwakeup(rpipe);
 822
 823         PIPE_UNLOCK(rpipe);
 824         return (error);
 825 }
 826
 827 #ifndef PIPE_NODIRECT
 828 /*
 829  * Map the sending processes' buffer into kernel space and wire it.
 830  * This is similar to a physical write operation.
 831  */
 832 static int
 833 pipe_build_write_buffer(wpipe, uio)
 834         struct pipe *wpipe;
 835         struct uio *uio;
 836 {
 837         u_int size;
 838         int i;
 839
 840         PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED);
 841         KASSERT(wpipe->pipe_state & PIPE_DIRECTW,
 842                 ("Clone attempt on non-direct write pipe!"));
 843
 844         if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size)
 845                 size = wpipe->pipe_buffer.size;
 846         else
 847                 size = uio->uio_iov->iov_len;
 848
 849         if ((i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
 850             (vm_offset_t)uio->uio_iov->iov_base, size, VM_PROT_READ,
 851             wpipe->pipe_map.ms, PIPENPAGES)) < 0)
 852                 return (EFAULT);
 853
 854 /*
 855  * set up the control block
 856  */
 857         wpipe->pipe_map.npages = i;
 858         wpipe->pipe_map.pos =
 859             ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK;
 860         wpipe->pipe_map.cnt = size;
 861
 862 /*
 863  * and update the uio data
 864  */
 865
 866         uio->uio_iov->iov_len -= size;
 867         uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size;
 868         if (uio->uio_iov->iov_len == 0)
 869                 uio->uio_iov++;
 870         uio->uio_resid -= size;
 871         uio->uio_offset += size;
 872         return (0);
 873 }
 874
 875 /*
 876  * unmap and unwire the process buffer
 877  */
 878 static void
 879 pipe_destroy_write_buffer(wpipe)
 880         struct pipe *wpipe;
 881 {
 882
 883         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
 884         vm_page_unhold_pages(wpipe->pipe_map.ms, wpipe->pipe_map.npages);
 885         wpipe->pipe_map.npages = 0;
 886 }
 887
 888 /*
 889  * In the case of a signal, the writing process might go away.  This
 890  * code copies the data into the circular buffer so that the source
 891  * pages can be freed without loss of data.
 892  */
 893 static void
 894 pipe_clone_write_buffer(wpipe)
 895         struct pipe *wpipe;
 896 {
 897         struct uio uio;
 898         struct iovec iov;
 899         int size;
 900         int pos;
 901
 902         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
 903         size = wpipe->pipe_map.cnt;
 904         pos = wpipe->pipe_map.pos;
 905
 906         wpipe->pipe_buffer.in = size;
 907         wpipe->pipe_buffer.out = 0;
 908         wpipe->pipe_buffer.cnt = size;
 909         wpipe->pipe_state &= ~PIPE_DIRECTW;
 910
 911         PIPE_UNLOCK(wpipe);
 912         iov.iov_base = wpipe->pipe_buffer.buffer;
 913         iov.iov_len = size;
 914         uio.uio_iov = &iov;
 915         uio.uio_iovcnt = 1;
 916         uio.uio_offset = 0;
 917         uio.uio_resid = size;
 918         uio.uio_segflg = UIO_SYSSPACE;
 919         uio.uio_rw = UIO_READ;
 920         uio.uio_td = curthread;
 921         uiomove_fromphys(wpipe->pipe_map.ms, pos, size, &uio);
 922         PIPE_LOCK(wpipe);
 923         pipe_destroy_write_buffer(wpipe);
 924 }
 925
 926 /*
 927  * This implements the pipe buffer write mechanism.  Note that only
 928  * a direct write OR a normal pipe write can be pending at any given time.
 929  * If there are any characters in the pipe buffer, the direct write will
 930  * be deferred until the receiving process grabs all of the bytes from
 931  * the pipe buffer.  Then the direct mapping write is set-up.
 932  */
 933 static int
 934 pipe_direct_write(wpipe, uio)
 935         struct pipe *wpipe;
 936         struct uio *uio;
 937 {
 938         int error;
 939
 940 retry:
 941         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
 942         error = pipelock(wpipe, 1);
 943         if (error != 0)
 944                 goto error1;
 945         if ((wpipe->pipe_state & PIPE_EOF) != 0) {
 946                 error = EPIPE;
 947                 pipeunlock(wpipe);
 948                 goto error1;
 949         }
 950         while (wpipe->pipe_state & PIPE_DIRECTW) {
 951                 if (wpipe->pipe_state & PIPE_WANTR) {
 952                         wpipe->pipe_state &= ~PIPE_WANTR;
 953                         wakeup(wpipe);
 954                 }
 955                 pipeselwakeup(wpipe);
 956                 wpipe->pipe_state |= PIPE_WANTW;
 957                 pipeunlock(wpipe);
 958                 error = msleep(wpipe, PIPE_MTX(wpipe),
 959                     PRIBIO | PCATCH, "pipdww", 0);
 960                 if (error)
 961                         goto error1;
 962                 else
 963                         goto retry;
 964         }
 965         wpipe->pipe_map.cnt = 0;        /* transfer not ready yet */
 966         if (wpipe->pipe_buffer.cnt > 0) {
 967                 if (wpipe->pipe_state & PIPE_WANTR) {
 968                         wpipe->pipe_state &= ~PIPE_WANTR;
 969                         wakeup(wpipe);
 970                 }
 971                 pipeselwakeup(wpipe);
 972                 wpipe->pipe_state |= PIPE_WANTW;
 973                 pipeunlock(wpipe);
 974                 error = msleep(wpipe, PIPE_MTX(wpipe),
 975                     PRIBIO | PCATCH, "pipdwc", 0);
 976                 if (error)
 977                         goto error1;
 978                 else
 979                         goto retry;
 980         }
 981
 982         wpipe->pipe_state |= PIPE_DIRECTW;
 983
 984         PIPE_UNLOCK(wpipe);
 985         error = pipe_build_write_buffer(wpipe, uio);
 986         PIPE_LOCK(wpipe);
 987         if (error) {
 988                 wpipe->pipe_state &= ~PIPE_DIRECTW;
 989                 pipeunlock(wpipe);
 990                 goto error1;
 991         }
 992
 993         error = 0;
 994         while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) {
 995                 if (wpipe->pipe_state & PIPE_EOF) {
 996                         pipe_destroy_write_buffer(wpipe);
 997                         pipeselwakeup(wpipe);
 998                         pipeunlock(wpipe);
 999                         error = EPIPE;
1000                         goto error1;
1001                 }
1002                 if (wpipe->pipe_state & PIPE_WANTR) {
1003                         wpipe->pipe_state &= ~PIPE_WANTR;
1004                         wakeup(wpipe);
1005                 }
1006                 pipeselwakeup(wpipe);
1007                 wpipe->pipe_state |= PIPE_WANTW;
1008                 pipeunlock(wpipe);
1009                 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,
1010                     "pipdwt", 0);
1011                 pipelock(wpipe, 0);
1012         }
1013
1014         if (wpipe->pipe_state & PIPE_EOF)
1015                 error = EPIPE;
1016         if (wpipe->pipe_state & PIPE_DIRECTW) {
1017                 /*
1018                  * this bit of trickery substitutes a kernel buffer for
1019                  * the process that might be going away.
1020                  */
1021                 pipe_clone_write_buffer(wpipe);
1022         } else {
1023                 pipe_destroy_write_buffer(wpipe);
1024         }
1025         pipeunlock(wpipe);
1026         return (error);
1027
1028 error1:
1029         wakeup(wpipe);
1030         return (error);
1031 }
1032 #endif
1033
1034 static int
1035 pipe_write(fp, uio, active_cred, flags, td)
1036         struct file *fp;
1037         struct uio *uio;
1038         struct ucred *active_cred;
1039         struct thread *td;
1040         int flags;
1041 {
1042         int error = 0;
1043         int desiredsize;
1044         ssize_t orig_resid;
1045         struct pipe *wpipe, *rpipe;
1046
1047         rpipe = fp->f_data;
1048         wpipe = PIPE_PEER(rpipe);
1049         PIPE_LOCK(rpipe);
1050         error = pipelock(wpipe, 1);
1051         if (error) {
1052                 PIPE_UNLOCK(rpipe);
1053                 return (error);
1054         }
1055         /*
1056          * detect loss of pipe read side, issue SIGPIPE if lost.
1057          */
1058         if (wpipe->pipe_present != PIPE_ACTIVE ||
1059             (wpipe->pipe_state & PIPE_EOF)) {
1060                 pipeunlock(wpipe);
1061                 PIPE_UNLOCK(rpipe);
1062                 return (EPIPE);
1063         }
1064 #ifdef MAC
1065         error = mac_pipe_check_write(active_cred, wpipe->pipe_pair);
1066         if (error) {
1067                 pipeunlock(wpipe);
1068                 PIPE_UNLOCK(rpipe);
1069                 return (error);
1070         }
1071 #endif
1072         ++wpipe->pipe_busy;
1073
1074         /* Choose a larger size if it's advantageous */
1075         desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size);
1076         while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) {
1077                 if (piperesizeallowed != 1)
1078                         break;
1079                 if (amountpipekva > maxpipekva / 2)
1080                         break;
1081                 if (desiredsize == BIG_PIPE_SIZE)
1082                         break;
1083                 desiredsize = desiredsize * 2;
1084         }
1085
1086         /* Choose a smaller size if we're in a OOM situation */
1087         if ((amountpipekva > (3 * maxpipekva) / 4) &&
1088                 (wpipe->pipe_buffer.size > SMALL_PIPE_SIZE) &&
1089                 (wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) &&
1090                 (piperesizeallowed == 1))
1091                 desiredsize = SMALL_PIPE_SIZE;
1092
1093         /* Resize if the above determined that a new size was necessary */
1094         if ((desiredsize != wpipe->pipe_buffer.size) &&
1095                 ((wpipe->pipe_state & PIPE_DIRECTW) == 0)) {
1096                 PIPE_UNLOCK(wpipe);
1097                 pipespace(wpipe, desiredsize);
1098                 PIPE_LOCK(wpipe);
1099         }
1100         if (wpipe->pipe_buffer.size == 0) {
1101                 /*
1102                  * This can only happen for reverse direction use of pipes
1103                  * in a complete OOM situation.
1104                  */
1105                 error = ENOMEM;
1106                 --wpipe->pipe_busy;
1107                 pipeunlock(wpipe);
1108                 PIPE_UNLOCK(wpipe);
1109                 return (error);
1110         }
1111
1112         pipeunlock(wpipe);
1113
1114         orig_resid = uio->uio_resid;
1115
1116         while (uio->uio_resid) {
1117                 int space;
1118
1119                 pipelock(wpipe, 0);
1120                 if (wpipe->pipe_state & PIPE_EOF) {
1121                         pipeunlock(wpipe);
1122                         error = EPIPE;
1123                         break;
1124                 }
1125 #ifndef PIPE_NODIRECT
1126                 /*
1127                  * If the transfer is large, we can gain performance if
1128                  * we do process-to-process copies directly.
1129                  * If the write is non-blocking, we don't use the
1130                  * direct write mechanism.
1131                  *
1132                  * The direct write mechanism will detect the reader going
1133                  * away on us.
1134                  */
1135                 if (uio->uio_segflg == UIO_USERSPACE &&
1136                     uio->uio_iov->iov_len >= PIPE_MINDIRECT &&
1137                     wpipe->pipe_buffer.size >= PIPE_MINDIRECT &&
1138                     (fp->f_flag & FNONBLOCK) == 0) {
1139                         pipeunlock(wpipe);
1140                         error = pipe_direct_write(wpipe, uio);
1141                         if (error)
1142                                 break;
1143                         continue;
1144                 }
1145 #endif
1146
1147                 /*
1148                  * Pipe buffered writes cannot be coincidental with
1149                  * direct writes.  We wait until the currently executing
1150                  * direct write is completed before we start filling the
1151                  * pipe buffer.  We break out if a signal occurs or the
1152                  * reader goes away.
1153                  */
1154                 if (wpipe->pipe_state & PIPE_DIRECTW) {
1155                         if (wpipe->pipe_state & PIPE_WANTR) {
1156                                 wpipe->pipe_state &= ~PIPE_WANTR;
1157                                 wakeup(wpipe);
1158                         }
1159                         pipeselwakeup(wpipe);
1160                         wpipe->pipe_state |= PIPE_WANTW;
1161                         pipeunlock(wpipe);
1162                         error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,
1163                             "pipbww", 0);
1164                         if (error)
1165                                 break;
1166                         else
1167                                 continue;
1168                 }
1169
1170                 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
1171
1172                 /* Writes of size <= PIPE_BUF must be atomic. */
1173                 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF))
1174                         space = 0;
1175
1176                 if (space > 0) {
1177                         int size;       /* Transfer size */
1178                         int segsize;    /* first segment to transfer */
1179
1180                         /*
1181                          * Transfer size is minimum of uio transfer
1182                          * and free space in pipe buffer.
1183                          */
1184                         if (space > uio->uio_resid)
1185                                 size = uio->uio_resid;
1186                         else
1187                                 size = space;
1188                         /*
1189                          * First segment to transfer is minimum of
1190                          * transfer size and contiguous space in
1191                          * pipe buffer.  If first segment to transfer
1192                          * is less than the transfer size, we've got
1193                          * a wraparound in the buffer.
1194                          */
1195                         segsize = wpipe->pipe_buffer.size -
1196                                 wpipe->pipe_buffer.in;
1197                         if (segsize > size)
1198                                 segsize = size;
1199
1200                         /* Transfer first segment */
1201
1202                         PIPE_UNLOCK(rpipe);
1203                         error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],
1204                                         segsize, uio);
1205                         PIPE_LOCK(rpipe);
1206
1207                         if (error == 0 && segsize < size) {
1208                                 KASSERT(wpipe->pipe_buffer.in + segsize ==
1209                                         wpipe->pipe_buffer.size,
1210                                         ("Pipe buffer wraparound disappeared"));
1211                                 /*
1212                                  * Transfer remaining part now, to
1213                                  * support atomic writes.  Wraparound
1214                                  * happened.
1215                                  */
1216
1217                                 PIPE_UNLOCK(rpipe);
1218                                 error = uiomove(
1219                                     &wpipe->pipe_buffer.buffer[0],
1220                                     size - segsize, uio);
1221                                 PIPE_LOCK(rpipe);
1222                         }
1223                         if (error == 0) {
1224                                 wpipe->pipe_buffer.in += size;
1225                                 if (wpipe->pipe_buffer.in >=
1226                                     wpipe->pipe_buffer.size) {
1227                                         KASSERT(wpipe->pipe_buffer.in ==
1228                                                 size - segsize +
1229                                                 wpipe->pipe_buffer.size,
1230                                                 ("Expected wraparound bad"));
1231                                         wpipe->pipe_buffer.in = size - segsize;
1232                                 }
1233
1234                                 wpipe->pipe_buffer.cnt += size;
1235                                 KASSERT(wpipe->pipe_buffer.cnt <=
1236                                         wpipe->pipe_buffer.size,
1237                                         ("Pipe buffer overflow"));
1238                         }
1239                         pipeunlock(wpipe);
1240                         if (error != 0)
1241                                 break;
1242                 } else {
1243                         /*
1244                          * If the "read-side" has been blocked, wake it up now.
1245                          */
1246                         if (wpipe->pipe_state & PIPE_WANTR) {
1247                                 wpipe->pipe_state &= ~PIPE_WANTR;
1248                                 wakeup(wpipe);
1249                         }
1250
1251                         /*
1252                          * don't block on non-blocking I/O
1253                          */
1254                         if (fp->f_flag & FNONBLOCK) {
1255                                 error = EAGAIN;
1256                                 pipeunlock(wpipe);
1257                                 break;
1258                         }
1259
1260                         /*
1261                          * We have no more space and have something to offer,
1262                          * wake up select/poll.
1263                          */
1264                         pipeselwakeup(wpipe);
1265
1266                         wpipe->pipe_state |= PIPE_WANTW;
1267                         pipeunlock(wpipe);
1268                         error = msleep(wpipe, PIPE_MTX(rpipe),
1269                             PRIBIO | PCATCH, "pipewr", 0);
1270                         if (error != 0)
1271                                 break;
1272                 }
1273         }
1274
1275         pipelock(wpipe, 0);
1276         --wpipe->pipe_busy;
1277
1278         if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {
1279                 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
1280                 wakeup(wpipe);
1281         } else if (wpipe->pipe_buffer.cnt > 0) {
1282                 /*
1283                  * If we have put any characters in the buffer, we wake up
1284                  * the reader.
1285                  */
1286                 if (wpipe->pipe_state & PIPE_WANTR) {
1287                         wpipe->pipe_state &= ~PIPE_WANTR;
1288                         wakeup(wpipe);
1289                 }
1290         }
1291
1292         /*
1293          * Don't return EPIPE if any byte was written.
1294          * EINTR and other interrupts are handled by generic I/O layer.
1295          * Do not pretend that I/O succeeded for obvious user error
1296          * like EFAULT.
1297          */
1298         if (uio->uio_resid != orig_resid && error == EPIPE)
1299                 error = 0;
1300
1301         if (error == 0)
1302                 vfs_timestamp(&wpipe->pipe_mtime);
1303
1304         /*
1305          * We have something to offer,
1306          * wake up select/poll.
1307          */
1308         if (wpipe->pipe_buffer.cnt)
1309                 pipeselwakeup(wpipe);
1310
1311         pipeunlock(wpipe);
1312         PIPE_UNLOCK(rpipe);
1313         return (error);
1314 }
1315
1316 /* ARGSUSED */
1317 static int
1318 pipe_truncate(fp, length, active_cred, td)
1319         struct file *fp;
1320         off_t length;
1321         struct ucred *active_cred;
1322         struct thread *td;
1323 {
1324         struct pipe *cpipe;
1325         int error;
1326
1327         cpipe = fp->f_data;
1328         if (cpipe->pipe_state & PIPE_NAMED)
1329                 error = vnops.fo_truncate(fp, length, active_cred, td);
1330         else
1331                 error = invfo_truncate(fp, length, active_cred, td);
1332         return (error);
1333 }
1334
1335 /*
1336  * we implement a very minimal set of ioctls for compatibility with sockets.
1337  */
1338 static int
1339 pipe_ioctl(fp, cmd, data, active_cred, td)
1340         struct file *fp;
1341         u_long cmd;
1342         void *data;
1343         struct ucred *active_cred;
1344         struct thread *td;
1345 {
1346         struct pipe *mpipe = fp->f_data;
1347         int error;
1348
1349         PIPE_LOCK(mpipe);
1350
1351 #ifdef MAC
1352         error = mac_pipe_check_ioctl(active_cred, mpipe->pipe_pair, cmd, data);
1353         if (error) {
1354                 PIPE_UNLOCK(mpipe);
1355                 return (error);
1356         }
1357 #endif
1358
1359         error = 0;
1360         switch (cmd) {
1361
1362         case FIONBIO:
1363                 break;
1364
1365         case FIOASYNC:
1366                 if (*(int *)data) {
1367                         mpipe->pipe_state |= PIPE_ASYNC;
1368                 } else {
1369                         mpipe->pipe_state &= ~PIPE_ASYNC;
1370                 }
1371                 break;
1372
1373         case FIONREAD:
1374                 if (!(fp->f_flag & FREAD)) {
1375                         *(int *)data = 0;
1376                         PIPE_UNLOCK(mpipe);
1377                         return (0);
1378                 }
1379                 if (mpipe->pipe_state & PIPE_DIRECTW)
1380                         *(int *)data = mpipe->pipe_map.cnt;
1381                 else
1382                         *(int *)data = mpipe->pipe_buffer.cnt;
1383                 break;
1384
1385         case FIOSETOWN:
1386                 PIPE_UNLOCK(mpipe);
1387                 error = fsetown(*(int *)data, &mpipe->pipe_sigio);
1388                 goto out_unlocked;
1389
1390         case FIOGETOWN:
1391                 *(int *)data = fgetown(&mpipe->pipe_sigio);
1392                 break;
1393
1394         /* This is deprecated, FIOSETOWN should be used instead. */
1395         case TIOCSPGRP:
1396                 PIPE_UNLOCK(mpipe);
1397                 error = fsetown(-(*(int *)data), &mpipe->pipe_sigio);
1398                 goto out_unlocked;
1399
1400         /* This is deprecated, FIOGETOWN should be used instead. */
1401         case TIOCGPGRP:
1402                 *(int *)data = -fgetown(&mpipe->pipe_sigio);
1403                 break;
1404
1405         default:
1406                 error = ENOTTY;
1407                 break;
1408         }
1409         PIPE_UNLOCK(mpipe);
1410 out_unlocked:
1411         return (error);
1412 }
1413
1414 static int
1415 pipe_poll(fp, events, active_cred, td)
1416         struct file *fp;
1417         int events;
1418         struct ucred *active_cred;
1419         struct thread *td;
1420 {
1421         struct pipe *rpipe;
1422         struct pipe *wpipe;
1423         int levents, revents;
1424 #ifdef MAC
1425         int error;
1426 #endif
1427
1428         revents = 0;
1429         rpipe = fp->f_data;
1430         wpipe = PIPE_PEER(rpipe);
1431         PIPE_LOCK(rpipe);
1432 #ifdef MAC
1433         error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair);
1434         if (error)
1435                 goto locked_error;
1436 #endif
1437         if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM))
1438                 if ((rpipe->pipe_state & PIPE_DIRECTW) ||
1439                     (rpipe->pipe_buffer.cnt > 0))
1440                         revents |= events & (POLLIN | POLLRDNORM);
1441
1442         if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM))
1443                 if (wpipe->pipe_present != PIPE_ACTIVE ||
1444                     (wpipe->pipe_state & PIPE_EOF) ||
1445                     (((wpipe->pipe_state & PIPE_DIRECTW) == 0) &&
1446                      ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF ||
1447                          wpipe->pipe_buffer.size == 0)))
1448                         revents |= events & (POLLOUT | POLLWRNORM);
1449
1450         levents = events &
1451             (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | POLLRDBAND);
1452         if (rpipe->pipe_state & PIPE_NAMED && fp->f_flag & FREAD && levents &&
1453             fp->f_seqcount == rpipe->pipe_wgen)
1454                 events |= POLLINIGNEOF;
1455
1456         if ((events & POLLINIGNEOF) == 0) {
1457                 if (rpipe->pipe_state & PIPE_EOF) {
1458                         revents |= (events & (POLLIN | POLLRDNORM));
1459                         if (wpipe->pipe_present != PIPE_ACTIVE ||
1460                             (wpipe->pipe_state & PIPE_EOF))
1461                                 revents |= POLLHUP;
1462                 }
1463         }
1464
1465         if (revents == 0) {
1466                 if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM)) {
1467                         selrecord(td, &rpipe->pipe_sel);
1468                         if (SEL_WAITING(&rpipe->pipe_sel))
1469                                 rpipe->pipe_state |= PIPE_SEL;
1470                 }
1471
1472                 if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM)) {
1473                         selrecord(td, &wpipe->pipe_sel);
1474                         if (SEL_WAITING(&wpipe->pipe_sel))
1475                                 wpipe->pipe_state |= PIPE_SEL;
1476                 }
1477         }
1478 #ifdef MAC
1479 locked_error:
1480 #endif
1481         PIPE_UNLOCK(rpipe);
1482
1483         return (revents);
1484 }
1485
1486 /*
1487  * We shouldn't need locks here as we're doing a read and this should
1488  * be a natural race.
1489  */
1490 static int
1491 pipe_stat(fp, ub, active_cred, td)
1492         struct file *fp;
1493         struct stat *ub;
1494         struct ucred *active_cred;
1495         struct thread *td;
1496 {
1497         struct pipe *pipe;
1498         int new_unr;
1499 #ifdef MAC
1500         int error;
1501 #endif
1502
1503         pipe = fp->f_data;
1504         PIPE_LOCK(pipe);
1505 #ifdef MAC
1506         error = mac_pipe_check_stat(active_cred, pipe->pipe_pair);
1507         if (error) {
1508                 PIPE_UNLOCK(pipe);
1509                 return (error);
1510         }
1511 #endif
1512
1513         /* For named pipes ask the underlying filesystem. */
1514         if (pipe->pipe_state & PIPE_NAMED) {
1515                 PIPE_UNLOCK(pipe);
1516                 return (vnops.fo_stat(fp, ub, active_cred, td));
1517         }
1518
1519         /*
1520          * Lazily allocate an inode number for the pipe.  Most pipe
1521          * users do not call fstat(2) on the pipe, which means that
1522          * postponing the inode allocation until it is must be
1523          * returned to userland is useful.  If alloc_unr failed,
1524          * assign st_ino zero instead of returning an error.
1525          * Special pipe_ino values:
1526          *  -1 - not yet initialized;
1527          *  0  - alloc_unr failed, return 0 as st_ino forever.
1528          */
1529         if (pipe->pipe_ino == (ino_t)-1) {
1530                 new_unr = alloc_unr(pipeino_unr);
1531                 if (new_unr != -1)
1532                         pipe->pipe_ino = new_unr;
1533                 else
1534                         pipe->pipe_ino = 0;
1535         }
1536         PIPE_UNLOCK(pipe);
1537
1538         bzero(ub, sizeof(*ub));
1539         ub->st_mode = S_IFIFO;
1540         ub->st_blksize = PAGE_SIZE;
1541         if (pipe->pipe_state & PIPE_DIRECTW)
1542                 ub->st_size = pipe->pipe_map.cnt;
1543         else
1544                 ub->st_size = pipe->pipe_buffer.cnt;
1545         ub->st_blocks = howmany(ub->st_size, ub->st_blksize);
1546         ub->st_atim = pipe->pipe_atime;
1547         ub->st_mtim = pipe->pipe_mtime;
1548         ub->st_ctim = pipe->pipe_ctime;
1549         ub->st_uid = fp->f_cred->cr_uid;
1550         ub->st_gid = fp->f_cred->cr_gid;
1551         ub->st_dev = pipedev_ino;
1552         ub->st_ino = pipe->pipe_ino;
1553         /*
1554          * Left as 0: st_nlink, st_rdev, st_flags, st_gen.
1555          */
1556         return (0);
1557 }
1558
1559 /* ARGSUSED */
1560 static int
1561 pipe_close(fp, td)
1562         struct file *fp;
1563         struct thread *td;
1564 {
1565
1566         if (fp->f_vnode != NULL)
1567                 return vnops.fo_close(fp, td);
1568         fp->f_ops = &badfileops;
1569         pipe_dtor(fp->f_data);
1570         fp->f_data = NULL;
1571         return (0);
1572 }
1573
1574 static int
1575 pipe_chmod(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td)
1576 {
1577         struct pipe *cpipe;
1578         int error;
1579
1580         cpipe = fp->f_data;
1581         if (cpipe->pipe_state & PIPE_NAMED)
1582                 error = vn_chmod(fp, mode, active_cred, td);
1583         else
1584                 error = invfo_chmod(fp, mode, active_cred, td);
1585         return (error);
1586 }
1587
1588 static int
1589 pipe_chown(fp, uid, gid, active_cred, td)
1590         struct file *fp;
1591         uid_t uid;
1592         gid_t gid;
1593         struct ucred *active_cred;
1594         struct thread *td;
1595 {
1596         struct pipe *cpipe;
1597         int error;
1598
1599         cpipe = fp->f_data;
1600         if (cpipe->pipe_state & PIPE_NAMED)
1601                 error = vn_chown(fp, uid, gid, active_cred, td);
1602         else
1603                 error = invfo_chown(fp, uid, gid, active_cred, td);
1604         return (error);
1605 }
1606
1607 static int
1608 pipe_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
1609 {
1610         struct pipe *pi;
1611
1612         if (fp->f_type == DTYPE_FIFO)
1613                 return (vn_fill_kinfo(fp, kif, fdp));
1614         kif->kf_type = KF_TYPE_PIPE;
1615         pi = fp->f_data;
1616         kif->kf_un.kf_pipe.kf_pipe_addr = (uintptr_t)pi;
1617         kif->kf_un.kf_pipe.kf_pipe_peer = (uintptr_t)pi->pipe_peer;
1618         kif->kf_un.kf_pipe.kf_pipe_buffer_cnt = pi->pipe_buffer.cnt;
1619         return (0);
1620 }
1621
1622 static void
1623 pipe_free_kmem(cpipe)
1624         struct pipe *cpipe;
1625 {
1626
1627         KASSERT(!mtx_owned(PIPE_MTX(cpipe)),
1628             ("pipe_free_kmem: pipe mutex locked"));
1629
1630         if (cpipe->pipe_buffer.buffer != NULL) {
1631                 atomic_subtract_long(&amountpipekva, cpipe->pipe_buffer.size);
1632                 vm_map_remove(pipe_map,
1633                     (vm_offset_t)cpipe->pipe_buffer.buffer,
1634                     (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size);
1635                 cpipe->pipe_buffer.buffer = NULL;
1636         }
1637 #ifndef PIPE_NODIRECT
1638         {
1639                 cpipe->pipe_map.cnt = 0;
1640                 cpipe->pipe_map.pos = 0;
1641                 cpipe->pipe_map.npages = 0;
1642         }
1643 #endif
1644 }
1645
1646 /*
1647  * shutdown the pipe
1648  */
1649 static void
1650 pipeclose(cpipe)
1651         struct pipe *cpipe;
1652 {
1653         struct pipepair *pp;
1654         struct pipe *ppipe;
1655
1656         KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL"));
1657
1658         PIPE_LOCK(cpipe);
1659         pipelock(cpipe, 0);
1660         pp = cpipe->pipe_pair;
1661
1662         pipeselwakeup(cpipe);
1663
1664         /*
1665          * If the other side is blocked, wake it up saying that
1666          * we want to close it down.
1667          */
1668         cpipe->pipe_state |= PIPE_EOF;
1669         while (cpipe->pipe_busy) {
1670                 wakeup(cpipe);
1671                 cpipe->pipe_state |= PIPE_WANT;
1672                 pipeunlock(cpipe);
1673                 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0);
1674                 pipelock(cpipe, 0);
1675         }
1676
1677
1678         /*
1679          * Disconnect from peer, if any.
1680          */
1681         ppipe = cpipe->pipe_peer;
1682         if (ppipe->pipe_present == PIPE_ACTIVE) {
1683                 pipeselwakeup(ppipe);
1684
1685                 ppipe->pipe_state |= PIPE_EOF;
1686                 wakeup(ppipe);
1687                 KNOTE_LOCKED(&ppipe->pipe_sel.si_note, 0);
1688         }
1689
1690         /*
1691          * Mark this endpoint as free.  Release kmem resources.  We
1692          * don't mark this endpoint as unused until we've finished
1693          * doing that, or the pipe might disappear out from under
1694          * us.
1695          */
1696         PIPE_UNLOCK(cpipe);
1697         pipe_free_kmem(cpipe);
1698         PIPE_LOCK(cpipe);
1699         cpipe->pipe_present = PIPE_CLOSING;
1700         pipeunlock(cpipe);
1701
1702         /*
1703          * knlist_clear() may sleep dropping the PIPE_MTX. Set the
1704          * PIPE_FINALIZED, that allows other end to free the
1705          * pipe_pair, only after the knotes are completely dismantled.
1706          */
1707         knlist_clear(&cpipe->pipe_sel.si_note, 1);
1708         cpipe->pipe_present = PIPE_FINALIZED;
1709         seldrain(&cpipe->pipe_sel);
1710         knlist_destroy(&cpipe->pipe_sel.si_note);
1711
1712         /*
1713          * If both endpoints are now closed, release the memory for the
1714          * pipe pair.  If not, unlock.
1715          */
1716         if (ppipe->pipe_present == PIPE_FINALIZED) {
1717                 PIPE_UNLOCK(cpipe);
1718 #ifdef MAC
1719                 mac_pipe_destroy(pp);
1720 #endif
1721                 uma_zfree(pipe_zone, cpipe->pipe_pair);
1722         } else
1723                 PIPE_UNLOCK(cpipe);
1724 }
1725
1726 /*ARGSUSED*/
1727 static int
1728 pipe_kqfilter(struct file *fp, struct knote *kn)
1729 {
1730         struct pipe *cpipe;
1731
1732         /*
1733          * If a filter is requested that is not supported by this file
1734          * descriptor, don't return an error, but also don't ever generate an
1735          * event.
1736          */
1737         if ((kn->kn_filter == EVFILT_READ) && !(fp->f_flag & FREAD)) {
1738                 kn->kn_fop = &pipe_nfiltops;
1739                 return (0);
1740         }
1741         if ((kn->kn_filter == EVFILT_WRITE) && !(fp->f_flag & FWRITE)) {
1742                 kn->kn_fop = &pipe_nfiltops;
1743                 return (0);
1744         }
1745         cpipe = fp->f_data;
1746         PIPE_LOCK(cpipe);
1747         switch (kn->kn_filter) {
1748         case EVFILT_READ:
1749                 kn->kn_fop = &pipe_rfiltops;
1750                 break;
1751         case EVFILT_WRITE:
1752                 kn->kn_fop = &pipe_wfiltops;
1753                 if (cpipe->pipe_peer->pipe_present != PIPE_ACTIVE) {
1754                         /* other end of pipe has been closed */
1755                         PIPE_UNLOCK(cpipe);
1756                         return (EPIPE);
1757                 }
1758                 cpipe = PIPE_PEER(cpipe);
1759                 break;
1760         default:
1761                 PIPE_UNLOCK(cpipe);
1762                 return (EINVAL);
1763         }
1764
1765         kn->kn_hook = cpipe;
1766         knlist_add(&cpipe->pipe_sel.si_note, kn, 1);
1767         PIPE_UNLOCK(cpipe);
1768         return (0);
1769 }
1770
1771 static void
1772 filt_pipedetach(struct knote *kn)
1773 {
1774         struct pipe *cpipe = kn->kn_hook;
1775
1776         PIPE_LOCK(cpipe);
1777         knlist_remove(&cpipe->pipe_sel.si_note, kn, 1);
1778         PIPE_UNLOCK(cpipe);
1779 }
1780
1781 /*ARGSUSED*/
1782 static int
1783 filt_piperead(struct knote *kn, long hint)
1784 {
1785         struct pipe *rpipe = kn->kn_hook;
1786         struct pipe *wpipe = rpipe->pipe_peer;
1787         int ret;
1788
1789         PIPE_LOCK_ASSERT(rpipe, MA_OWNED);
1790         kn->kn_data = rpipe->pipe_buffer.cnt;
1791         if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW))
1792                 kn->kn_data = rpipe->pipe_map.cnt;
1793
1794         if ((rpipe->pipe_state & PIPE_EOF) ||
1795             wpipe->pipe_present != PIPE_ACTIVE ||
1796             (wpipe->pipe_state & PIPE_EOF)) {
1797                 kn->kn_flags |= EV_EOF;
1798                 return (1);
1799         }
1800         ret = kn->kn_data > 0;
1801         return ret;
1802 }
1803
1804 /*ARGSUSED*/
1805 static int
1806 filt_pipewrite(struct knote *kn, long hint)
1807 {
1808         struct pipe *wpipe;
1809
1810         wpipe = kn->kn_hook;
1811         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
1812         if (wpipe->pipe_present != PIPE_ACTIVE ||
1813             (wpipe->pipe_state & PIPE_EOF)) {
1814                 kn->kn_data = 0;
1815                 kn->kn_flags |= EV_EOF;
1816                 return (1);
1817         }
1818         kn->kn_data = (wpipe->pipe_buffer.size > 0) ?
1819             (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) : PIPE_BUF;
1820         if (wpipe->pipe_state & PIPE_DIRECTW)
1821                 kn->kn_data = 0;
1822
1823         return (kn->kn_data >= PIPE_BUF);
1824 }
1825
1826 static void
1827 filt_pipedetach_notsup(struct knote *kn)
1828 {
1829
1830 }
1831
1832 static int
1833 filt_pipenotsup(struct knote *kn, long hint)
1834 {
1835
1836         return (0);
1837 }