sys/kern/sys_pipe.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-4-Clause
   3  *
   4  * Copyright (c) 1996 John S. Dyson
   5  * Copyright (c) 2012 Giovanni Trematerra
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice immediately at the beginning of the file, without modification,
  13  *    this list of conditions, and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  * 3. Absolutely no warranty of function or purpose is made by the author
  18  *    John S. Dyson.
  19  * 4. Modifications may be freely made to this file if the above conditions
  20  *    are met.
  21  */
  22
  23 /*
  24  * This file contains a high-performance replacement for the socket-based
  25  * pipes scheme originally used in FreeBSD/4.4Lite.  It does not support
  26  * all features of sockets, but does do everything that pipes normally
  27  * do.
  28  */
  29
  30 /*
  31  * This code has two modes of operation, a small write mode and a large
  32  * write mode.  The small write mode acts like conventional pipes with
  33  * a kernel buffer.  If the buffer is less than PIPE_MINDIRECT, then the
  34  * "normal" pipe buffering is done.  If the buffer is between PIPE_MINDIRECT
  35  * and PIPE_SIZE in size, the sending process pins the underlying pages in
  36  * memory, and the receiving process copies directly from these pinned pages
  37  * in the sending process.
  38  *
  39  * If the sending process receives a signal, it is possible that it will
  40  * go away, and certainly its address space can change, because control
  41  * is returned back to the user-mode side.  In that case, the pipe code
  42  * arranges to copy the buffer supplied by the user process, to a pageable
  43  * kernel buffer, and the receiving process will grab the data from the
  44  * pageable kernel buffer.  Since signals don't happen all that often,
  45  * the copy operation is normally eliminated.
  46  *
  47  * The constant PIPE_MINDIRECT is chosen to make sure that buffering will
  48  * happen for small transfers so that the system will not spend all of
  49  * its time context switching.
  50  *
  51  * In order to limit the resource use of pipes, two sysctls exist:
  52  *
  53  * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable
  54  * address space available to us in pipe_map. This value is normally
  55  * autotuned, but may also be loader tuned.
  56  *
  57  * kern.ipc.pipekva - This read-only sysctl tracks the current amount of
  58  * memory in use by pipes.
  59  *
  60  * Based on how large pipekva is relative to maxpipekva, the following
  61  * will happen:
  62  *
  63  * 0% - 50%:
  64  *     New pipes are given 16K of memory backing, pipes may dynamically
  65  *     grow to as large as 64K where needed.
  66  * 50% - 75%:
  67  *     New pipes are given 4K (or PAGE_SIZE) of memory backing,
  68  *     existing pipes may NOT grow.
  69  * 75% - 100%:
  70  *     New pipes are given 4K (or PAGE_SIZE) of memory backing,
  71  *     existing pipes will be shrunk down to 4K whenever possible.
  72  *
  73  * Resizing may be disabled by setting kern.ipc.piperesizeallowed=0.  If
  74  * that is set,  the only resize that will occur is the 0 -> SMALL_PIPE_SIZE
  75  * resize which MUST occur for reverse-direction pipes when they are
  76  * first used.
  77  *
  78  * Additional information about the current state of pipes may be obtained
  79  * from kern.ipc.pipes, kern.ipc.pipefragretry, kern.ipc.pipeallocfail,
  80  * and kern.ipc.piperesizefail.
  81  *
  82  * Locking rules:  There are two locks present here:  A mutex, used via
  83  * PIPE_LOCK, and a flag, used via pipelock().  All locking is done via
  84  * the flag, as mutexes can not persist over uiomove.  The mutex
  85  * exists only to guard access to the flag, and is not in itself a
  86  * locking mechanism.  Also note that there is only a single mutex for
  87  * both directions of a pipe.
  88  *
  89  * As pipelock() may have to sleep before it can acquire the flag, it
  90  * is important to reread all data after a call to pipelock(); everything
  91  * in the structure may have changed.
  92  */
  93
  94 #include <sys/cdefs.h>
  95 __FBSDID("$FreeBSD$");
  96
  97 #include <sys/param.h>
  98 #include <sys/systm.h>
  99 #include <sys/conf.h>
 100 #include <sys/fcntl.h>
 101 #include <sys/file.h>
 102 #include <sys/filedesc.h>
 103 #include <sys/filio.h>
 104 #include <sys/kernel.h>
 105 #include <sys/lock.h>
 106 #include <sys/mutex.h>
 107 #include <sys/ttycom.h>
 108 #include <sys/stat.h>
 109 #include <sys/malloc.h>
 110 #include <sys/poll.h>
 111 #include <sys/selinfo.h>
 112 #include <sys/signalvar.h>
 113 #include <sys/syscallsubr.h>
 114 #include <sys/sysctl.h>
 115 #include <sys/sysproto.h>
 116 #include <sys/pipe.h>
 117 #include <sys/proc.h>
 118 #include <sys/vnode.h>
 119 #include <sys/uio.h>
 120 #include <sys/user.h>
 121 #include <sys/event.h>
 122
 123 #include <security/mac/mac_framework.h>
 124
 125 #include <vm/vm.h>
 126 #include <vm/vm_param.h>
 127 #include <vm/vm_object.h>
 128 #include <vm/vm_kern.h>
 129 #include <vm/vm_extern.h>
 130 #include <vm/pmap.h>
 131 #include <vm/vm_map.h>
 132 #include <vm/vm_page.h>
 133 #include <vm/uma.h>
 134
 135 /*
 136  * Use this define if you want to disable *fancy* VM things.  Expect an
 137  * approx 30% decrease in transfer rate.  This could be useful for
 138  * NetBSD or OpenBSD.
 139  */
 140 /* #define PIPE_NODIRECT */
 141
 142 #define PIPE_PEER(pipe) \
 143         (((pipe)->pipe_type & PIPE_TYPE_NAMED) ? (pipe) : ((pipe)->pipe_peer))
 144
 145 /*
 146  * interfaces to the outside world
 147  */
 148 static fo_rdwr_t        pipe_read;
 149 static fo_rdwr_t        pipe_write;
 150 static fo_truncate_t    pipe_truncate;
 151 static fo_ioctl_t       pipe_ioctl;
 152 static fo_poll_t        pipe_poll;
 153 static fo_kqfilter_t    pipe_kqfilter;
 154 static fo_stat_t        pipe_stat;
 155 static fo_close_t       pipe_close;
 156 static fo_chmod_t       pipe_chmod;
 157 static fo_chown_t       pipe_chown;
 158 static fo_fill_kinfo_t  pipe_fill_kinfo;
 159
 160 struct fileops pipeops = {
 161         .fo_read = pipe_read,
 162         .fo_write = pipe_write,
 163         .fo_truncate = pipe_truncate,
 164         .fo_ioctl = pipe_ioctl,
 165         .fo_poll = pipe_poll,
 166         .fo_kqfilter = pipe_kqfilter,
 167         .fo_stat = pipe_stat,
 168         .fo_close = pipe_close,
 169         .fo_chmod = pipe_chmod,
 170         .fo_chown = pipe_chown,
 171         .fo_sendfile = invfo_sendfile,
 172         .fo_fill_kinfo = pipe_fill_kinfo,
 173         .fo_flags = DFLAG_PASSABLE
 174 };
 175
 176 static void     filt_pipedetach(struct knote *kn);
 177 static void     filt_pipedetach_notsup(struct knote *kn);
 178 static int      filt_pipenotsup(struct knote *kn, long hint);
 179 static int      filt_piperead(struct knote *kn, long hint);
 180 static int      filt_pipewrite(struct knote *kn, long hint);
 181
 182 static struct filterops pipe_nfiltops = {
 183         .f_isfd = 1,
 184         .f_detach = filt_pipedetach_notsup,
 185         .f_event = filt_pipenotsup
 186 };
 187 static struct filterops pipe_rfiltops = {
 188         .f_isfd = 1,
 189         .f_detach = filt_pipedetach,
 190         .f_event = filt_piperead
 191 };
 192 static struct filterops pipe_wfiltops = {
 193         .f_isfd = 1,
 194         .f_detach = filt_pipedetach,
 195         .f_event = filt_pipewrite
 196 };
 197
 198 /*
 199  * Default pipe buffer size(s), this can be kind-of large now because pipe
 200  * space is pageable.  The pipe code will try to maintain locality of
 201  * reference for performance reasons, so small amounts of outstanding I/O
 202  * will not wipe the cache.
 203  */
 204 #define MINPIPESIZE (PIPE_SIZE/3)
 205 #define MAXPIPESIZE (2*PIPE_SIZE/3)
 206
 207 static long amountpipekva;
 208 static int pipefragretry;
 209 static int pipeallocfail;
 210 static int piperesizefail;
 211 static int piperesizeallowed = 1;
 212
 213 SYSCTL_LONG(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
 214            &maxpipekva, 0, "Pipe KVA limit");
 215 SYSCTL_LONG(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD,
 216            &amountpipekva, 0, "Pipe KVA usage");
 217 SYSCTL_INT(_kern_ipc, OID_AUTO, pipefragretry, CTLFLAG_RD,
 218           &pipefragretry, 0, "Pipe allocation retries due to fragmentation");
 219 SYSCTL_INT(_kern_ipc, OID_AUTO, pipeallocfail, CTLFLAG_RD,
 220           &pipeallocfail, 0, "Pipe allocation failures");
 221 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizefail, CTLFLAG_RD,
 222           &piperesizefail, 0, "Pipe resize failures");
 223 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizeallowed, CTLFLAG_RW,
 224           &piperesizeallowed, 0, "Pipe resizing allowed");
 225
 226 static void pipeinit(void *dummy __unused);
 227 static void pipeclose(struct pipe *cpipe);
 228 static void pipe_free_kmem(struct pipe *cpipe);
 229 static int pipe_create(struct pipe *pipe, bool backing);
 230 static int pipe_paircreate(struct thread *td, struct pipepair **p_pp);
 231 static __inline int pipelock(struct pipe *cpipe, int catch);
 232 static __inline void pipeunlock(struct pipe *cpipe);
 233 static void pipe_timestamp(struct timespec *tsp);
 234 #ifndef PIPE_NODIRECT
 235 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
 236 static void pipe_destroy_write_buffer(struct pipe *wpipe);
 237 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
 238 static void pipe_clone_write_buffer(struct pipe *wpipe);
 239 #endif
 240 static int pipespace(struct pipe *cpipe, int size);
 241 static int pipespace_new(struct pipe *cpipe, int size);
 242
 243 static int      pipe_zone_ctor(void *mem, int size, void *arg, int flags);
 244 static int      pipe_zone_init(void *mem, int size, int flags);
 245 static void     pipe_zone_fini(void *mem, int size);
 246
 247 static uma_zone_t pipe_zone;
 248 static struct unrhdr64 pipeino_unr;
 249 static dev_t pipedev_ino;
 250
 251 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL);
 252
 253 static void
 254 pipeinit(void *dummy __unused)
 255 {
 256
 257         pipe_zone = uma_zcreate("pipe", sizeof(struct pipepair),
 258             pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini,
 259             UMA_ALIGN_PTR, 0);
 260         KASSERT(pipe_zone != NULL, ("pipe_zone not initialized"));
 261         new_unrhdr64(&pipeino_unr, 1);
 262         pipedev_ino = devfs_alloc_cdp_inode();
 263         KASSERT(pipedev_ino > 0, ("pipe dev inode not initialized"));
 264 }
 265
 266 static int
 267 pipe_zone_ctor(void *mem, int size, void *arg, int flags)
 268 {
 269         struct pipepair *pp;
 270         struct pipe *rpipe, *wpipe;
 271
 272         KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size"));
 273
 274         pp = (struct pipepair *)mem;
 275
 276         /*
 277          * We zero both pipe endpoints to make sure all the kmem pointers
 278          * are NULL, flag fields are zero'd, etc.  We timestamp both
 279          * endpoints with the same time.
 280          */
 281         rpipe = &pp->pp_rpipe;
 282         bzero(rpipe, sizeof(*rpipe));
 283         pipe_timestamp(&rpipe->pipe_ctime);
 284         rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime;
 285
 286         wpipe = &pp->pp_wpipe;
 287         bzero(wpipe, sizeof(*wpipe));
 288         wpipe->pipe_ctime = rpipe->pipe_ctime;
 289         wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;
 290
 291         rpipe->pipe_peer = wpipe;
 292         rpipe->pipe_pair = pp;
 293         wpipe->pipe_peer = rpipe;
 294         wpipe->pipe_pair = pp;
 295
 296         /*
 297          * Mark both endpoints as present; they will later get free'd
 298          * one at a time.  When both are free'd, then the whole pair
 299          * is released.
 300          */
 301         rpipe->pipe_present = PIPE_ACTIVE;
 302         wpipe->pipe_present = PIPE_ACTIVE;
 303
 304         /*
 305          * Eventually, the MAC Framework may initialize the label
 306          * in ctor or init, but for now we do it elswhere to avoid
 307          * blocking in ctor or init.
 308          */
 309         pp->pp_label = NULL;
 310
 311         return (0);
 312 }
 313
 314 static int
 315 pipe_zone_init(void *mem, int size, int flags)
 316 {
 317         struct pipepair *pp;
 318
 319         KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size"));
 320
 321         pp = (struct pipepair *)mem;
 322
 323         mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_NEW);
 324         return (0);
 325 }
 326
 327 static void
 328 pipe_zone_fini(void *mem, int size)
 329 {
 330         struct pipepair *pp;
 331
 332         KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size"));
 333
 334         pp = (struct pipepair *)mem;
 335
 336         mtx_destroy(&pp->pp_mtx);
 337 }
 338
 339 static int
 340 pipe_paircreate(struct thread *td, struct pipepair **p_pp)
 341 {
 342         struct pipepair *pp;
 343         struct pipe *rpipe, *wpipe;
 344         int error;
 345
 346         *p_pp = pp = uma_zalloc(pipe_zone, M_WAITOK);
 347 #ifdef MAC
 348         /*
 349          * The MAC label is shared between the connected endpoints.  As a
 350          * result mac_pipe_init() and mac_pipe_create() are called once
 351          * for the pair, and not on the endpoints.
 352          */
 353         mac_pipe_init(pp);
 354         mac_pipe_create(td->td_ucred, pp);
 355 #endif
 356         rpipe = &pp->pp_rpipe;
 357         wpipe = &pp->pp_wpipe;
 358
 359         knlist_init_mtx(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe));
 360         knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe));
 361
 362         /*
 363          * Only the forward direction pipe is backed by big buffer by
 364          * default.
 365          */
 366         error = pipe_create(rpipe, true);
 367         if (error != 0)
 368                 goto fail;
 369         error = pipe_create(wpipe, false);
 370         if (error != 0) {
 371                 /*
 372                  * This cleanup leaves the pipe inode number for rpipe
 373                  * still allocated, but never used.  We do not free
 374                  * inode numbers for opened pipes, which is required
 375                  * for correctness because numbers must be unique.
 376                  * But also it avoids any memory use by the unr
 377                  * allocator, so stashing away the transient inode
 378                  * number is reasonable.
 379                  */
 380                 pipe_free_kmem(rpipe);
 381                 goto fail;
 382         }
 383
 384         rpipe->pipe_state |= PIPE_DIRECTOK;
 385         wpipe->pipe_state |= PIPE_DIRECTOK;
 386         return (0);
 387
 388 fail:
 389         knlist_destroy(&rpipe->pipe_sel.si_note);
 390         knlist_destroy(&wpipe->pipe_sel.si_note);
 391 #ifdef MAC
 392         mac_pipe_destroy(pp);
 393 #endif
 394         return (error);
 395 }
 396
 397 int
 398 pipe_named_ctor(struct pipe **ppipe, struct thread *td)
 399 {
 400         struct pipepair *pp;
 401         int error;
 402
 403         error = pipe_paircreate(td, &pp);
 404         if (error != 0)
 405                 return (error);
 406         pp->pp_rpipe.pipe_type |= PIPE_TYPE_NAMED;
 407         *ppipe = &pp->pp_rpipe;
 408         return (0);
 409 }
 410
 411 void
 412 pipe_dtor(struct pipe *dpipe)
 413 {
 414         struct pipe *peer;
 415
 416         peer = (dpipe->pipe_type & PIPE_TYPE_NAMED) != 0 ? dpipe->pipe_peer : NULL;
 417         funsetown(&dpipe->pipe_sigio);
 418         pipeclose(dpipe);
 419         if (peer != NULL) {
 420                 funsetown(&peer->pipe_sigio);
 421                 pipeclose(peer);
 422         }
 423 }
 424
 425 /*
 426  * Get a timestamp.
 427  *
 428  * This used to be vfs_timestamp but the higher precision is unnecessary and
 429  * can very negatively affect performance in virtualized environments (e.g., on
 430  * vms running on amd64 when using the rdtscp instruction).
 431  */
 432 static void
 433 pipe_timestamp(struct timespec *tsp)
 434 {
 435
 436         getnanotime(tsp);
 437 }
 438
 439 /*
 440  * The pipe system call for the DTYPE_PIPE type of pipes.  If we fail, let
 441  * the zone pick up the pieces via pipeclose().
 442  */
 443 int
 444 kern_pipe(struct thread *td, int fildes[2], int flags, struct filecaps *fcaps1,
 445     struct filecaps *fcaps2)
 446 {
 447         struct file *rf, *wf;
 448         struct pipe *rpipe, *wpipe;
 449         struct pipepair *pp;
 450         int fd, fflags, error;
 451
 452         error = pipe_paircreate(td, &pp);
 453         if (error != 0)
 454                 return (error);
 455         rpipe = &pp->pp_rpipe;
 456         wpipe = &pp->pp_wpipe;
 457         error = falloc_caps(td, &rf, &fd, flags, fcaps1);
 458         if (error) {
 459                 pipeclose(rpipe);
 460                 pipeclose(wpipe);
 461                 return (error);
 462         }
 463         /* An extra reference on `rf' has been held for us by falloc_caps(). */
 464         fildes[0] = fd;
 465
 466         fflags = FREAD | FWRITE;
 467         if ((flags & O_NONBLOCK) != 0)
 468                 fflags |= FNONBLOCK;
 469
 470         /*
 471          * Warning: once we've gotten past allocation of the fd for the
 472          * read-side, we can only drop the read side via fdrop() in order
 473          * to avoid races against processes which manage to dup() the read
 474          * side while we are blocked trying to allocate the write side.
 475          */
 476         finit(rf, fflags, DTYPE_PIPE, rpipe, &pipeops);
 477         error = falloc_caps(td, &wf, &fd, flags, fcaps2);
 478         if (error) {
 479                 fdclose(td, rf, fildes[0]);
 480                 fdrop(rf, td);
 481                 /* rpipe has been closed by fdrop(). */
 482                 pipeclose(wpipe);
 483                 return (error);
 484         }
 485         /* An extra reference on `wf' has been held for us by falloc_caps(). */
 486         finit(wf, fflags, DTYPE_PIPE, wpipe, &pipeops);
 487         fdrop(wf, td);
 488         fildes[1] = fd;
 489         fdrop(rf, td);
 490
 491         return (0);
 492 }
 493
 494 #ifdef COMPAT_FREEBSD10
 495 /* ARGSUSED */
 496 int
 497 freebsd10_pipe(struct thread *td, struct freebsd10_pipe_args *uap __unused)
 498 {
 499         int error;
 500         int fildes[2];
 501
 502         error = kern_pipe(td, fildes, 0, NULL, NULL);
 503         if (error)
 504                 return (error);
 505
 506         td->td_retval[0] = fildes[0];
 507         td->td_retval[1] = fildes[1];
 508
 509         return (0);
 510 }
 511 #endif
 512
 513 int
 514 sys_pipe2(struct thread *td, struct pipe2_args *uap)
 515 {
 516         int error, fildes[2];
 517
 518         if (uap->flags & ~(O_CLOEXEC | O_NONBLOCK))
 519                 return (EINVAL);
 520         error = kern_pipe(td, fildes, uap->flags, NULL, NULL);
 521         if (error)
 522                 return (error);
 523         error = copyout(fildes, uap->fildes, 2 * sizeof(int));
 524         if (error) {
 525                 (void)kern_close(td, fildes[0]);
 526                 (void)kern_close(td, fildes[1]);
 527         }
 528         return (error);
 529 }
 530
 531 /*
 532  * Allocate kva for pipe circular buffer, the space is pageable
 533  * This routine will 'realloc' the size of a pipe safely, if it fails
 534  * it will retain the old buffer.
 535  * If it fails it will return ENOMEM.
 536  */
 537 static int
 538 pipespace_new(struct pipe *cpipe, int size)
 539 {
 540         caddr_t buffer;
 541         int error, cnt, firstseg;
 542         static int curfail = 0;
 543         static struct timeval lastfail;
 544
 545         KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked"));
 546         KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW),
 547                 ("pipespace: resize of direct writes not allowed"));
 548 retry:
 549         cnt = cpipe->pipe_buffer.cnt;
 550         if (cnt > size)
 551                 size = cnt;
 552
 553         size = round_page(size);
 554         buffer = (caddr_t) vm_map_min(pipe_map);
 555
 556         error = vm_map_find(pipe_map, NULL, 0, (vm_offset_t *)&buffer, size, 0,
 557             VMFS_ANY_SPACE, VM_PROT_RW, VM_PROT_RW, 0);
 558         if (error != KERN_SUCCESS) {
 559                 if (cpipe->pipe_buffer.buffer == NULL &&
 560                     size > SMALL_PIPE_SIZE) {
 561                         size = SMALL_PIPE_SIZE;
 562                         pipefragretry++;
 563                         goto retry;
 564                 }
 565                 if (cpipe->pipe_buffer.buffer == NULL) {
 566                         pipeallocfail++;
 567                         if (ppsratecheck(&lastfail, &curfail, 1))
 568                                 printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n");
 569                 } else {
 570                         piperesizefail++;
 571                 }
 572                 return (ENOMEM);
 573         }
 574
 575         /* copy data, then free old resources if we're resizing */
 576         if (cnt > 0) {
 577                 if (cpipe->pipe_buffer.in <= cpipe->pipe_buffer.out) {
 578                         firstseg = cpipe->pipe_buffer.size - cpipe->pipe_buffer.out;
 579                         bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out],
 580                                 buffer, firstseg);
 581                         if ((cnt - firstseg) > 0)
 582                                 bcopy(cpipe->pipe_buffer.buffer, &buffer[firstseg],
 583                                         cpipe->pipe_buffer.in);
 584                 } else {
 585                         bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out],
 586                                 buffer, cnt);
 587                 }
 588         }
 589         pipe_free_kmem(cpipe);
 590         cpipe->pipe_buffer.buffer = buffer;
 591         cpipe->pipe_buffer.size = size;
 592         cpipe->pipe_buffer.in = cnt;
 593         cpipe->pipe_buffer.out = 0;
 594         cpipe->pipe_buffer.cnt = cnt;
 595         atomic_add_long(&amountpipekva, cpipe->pipe_buffer.size);
 596         return (0);
 597 }
 598
 599 /*
 600  * Wrapper for pipespace_new() that performs locking assertions.
 601  */
 602 static int
 603 pipespace(struct pipe *cpipe, int size)
 604 {
 605
 606         KASSERT(cpipe->pipe_state & PIPE_LOCKFL,
 607             ("Unlocked pipe passed to pipespace"));
 608         return (pipespace_new(cpipe, size));
 609 }
 610
 611 /*
 612  * lock a pipe for I/O, blocking other access
 613  */
 614 static __inline int
 615 pipelock(struct pipe *cpipe, int catch)
 616 {
 617         int error, prio;
 618
 619         PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
 620
 621         prio = PRIBIO;
 622         if (catch)
 623                 prio |= PCATCH;
 624         while (cpipe->pipe_state & PIPE_LOCKFL) {
 625                 KASSERT(cpipe->pipe_waiters >= 0,
 626                     ("%s: bad waiter count %d", __func__,
 627                     cpipe->pipe_waiters));
 628                 cpipe->pipe_waiters++;
 629                 error = msleep(cpipe, PIPE_MTX(cpipe),
 630                     prio, "pipelk", 0);
 631                 cpipe->pipe_waiters--;
 632                 if (error != 0)
 633                         return (error);
 634         }
 635         cpipe->pipe_state |= PIPE_LOCKFL;
 636         return (0);
 637 }
 638
 639 /*
 640  * unlock a pipe I/O lock
 641  */
 642 static __inline void
 643 pipeunlock(struct pipe *cpipe)
 644 {
 645
 646         PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
 647         KASSERT(cpipe->pipe_state & PIPE_LOCKFL,
 648                 ("Unlocked pipe passed to pipeunlock"));
 649         KASSERT(cpipe->pipe_waiters >= 0,
 650             ("%s: bad waiter count %d", __func__,
 651             cpipe->pipe_waiters));
 652         cpipe->pipe_state &= ~PIPE_LOCKFL;
 653         if (cpipe->pipe_waiters > 0) {
 654                 wakeup_one(cpipe);
 655         }
 656 }
 657
 658 void
 659 pipeselwakeup(struct pipe *cpipe)
 660 {
 661
 662         PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
 663         if (cpipe->pipe_state & PIPE_SEL) {
 664                 selwakeuppri(&cpipe->pipe_sel, PSOCK);
 665                 if (!SEL_WAITING(&cpipe->pipe_sel))
 666                         cpipe->pipe_state &= ~PIPE_SEL;
 667         }
 668         if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio)
 669                 pgsigio(&cpipe->pipe_sigio, SIGIO, 0);
 670         KNOTE_LOCKED(&cpipe->pipe_sel.si_note, 0);
 671 }
 672
 673 /*
 674  * Initialize and allocate VM and memory for pipe.  The structure
 675  * will start out zero'd from the ctor, so we just manage the kmem.
 676  */
 677 static int
 678 pipe_create(struct pipe *pipe, bool large_backing)
 679 {
 680         int error;
 681
 682         error = pipespace_new(pipe, !large_backing || amountpipekva >
 683             maxpipekva / 2 ? SMALL_PIPE_SIZE : PIPE_SIZE);
 684         if (error == 0)
 685                 pipe->pipe_ino = alloc_unr64(&pipeino_unr);
 686         return (error);
 687 }
 688
 689 /* ARGSUSED */
 690 static int
 691 pipe_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
 692     int flags, struct thread *td)
 693 {
 694         struct pipe *rpipe;
 695         int error;
 696         int nread = 0;
 697         int size;
 698
 699         rpipe = fp->f_data;
 700         PIPE_LOCK(rpipe);
 701         ++rpipe->pipe_busy;
 702         error = pipelock(rpipe, 1);
 703         if (error)
 704                 goto unlocked_error;
 705
 706 #ifdef MAC
 707         error = mac_pipe_check_read(active_cred, rpipe->pipe_pair);
 708         if (error)
 709                 goto locked_error;
 710 #endif
 711         if (amountpipekva > (3 * maxpipekva) / 4) {
 712                 if ((rpipe->pipe_state & PIPE_DIRECTW) == 0 &&
 713                     rpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&
 714                     rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&
 715                     piperesizeallowed == 1) {
 716                         PIPE_UNLOCK(rpipe);
 717                         pipespace(rpipe, SMALL_PIPE_SIZE);
 718                         PIPE_LOCK(rpipe);
 719                 }
 720         }
 721
 722         while (uio->uio_resid) {
 723                 /*
 724                  * normal pipe buffer receive
 725                  */
 726                 if (rpipe->pipe_buffer.cnt > 0) {
 727                         size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out;
 728                         if (size > rpipe->pipe_buffer.cnt)
 729                                 size = rpipe->pipe_buffer.cnt;
 730                         if (size > uio->uio_resid)
 731                                 size = uio->uio_resid;
 732
 733                         PIPE_UNLOCK(rpipe);
 734                         error = uiomove(
 735                             &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out],
 736                             size, uio);
 737                         PIPE_LOCK(rpipe);
 738                         if (error)
 739                                 break;
 740
 741                         rpipe->pipe_buffer.out += size;
 742                         if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size)
 743                                 rpipe->pipe_buffer.out = 0;
 744
 745                         rpipe->pipe_buffer.cnt -= size;
 746
 747                         /*
 748                          * If there is no more to read in the pipe, reset
 749                          * its pointers to the beginning.  This improves
 750                          * cache hit stats.
 751                          */
 752                         if (rpipe->pipe_buffer.cnt == 0) {
 753                                 rpipe->pipe_buffer.in = 0;
 754                                 rpipe->pipe_buffer.out = 0;
 755                         }
 756                         nread += size;
 757 #ifndef PIPE_NODIRECT
 758                 /*
 759                  * Direct copy, bypassing a kernel buffer.
 760                  */
 761                 } else if ((size = rpipe->pipe_pages.cnt) != 0) {
 762                         if (size > uio->uio_resid)
 763                                 size = (u_int) uio->uio_resid;
 764                         PIPE_UNLOCK(rpipe);
 765                         error = uiomove_fromphys(rpipe->pipe_pages.ms,
 766                             rpipe->pipe_pages.pos, size, uio);
 767                         PIPE_LOCK(rpipe);
 768                         if (error)
 769                                 break;
 770                         nread += size;
 771                         rpipe->pipe_pages.pos += size;
 772                         rpipe->pipe_pages.cnt -= size;
 773                         if (rpipe->pipe_pages.cnt == 0) {
 774                                 rpipe->pipe_state &= ~PIPE_WANTW;
 775                                 wakeup(rpipe);
 776                         }
 777 #endif
 778                 } else {
 779                         /*
 780                          * detect EOF condition
 781                          * read returns 0 on EOF, no need to set error
 782                          */
 783                         if (rpipe->pipe_state & PIPE_EOF)
 784                                 break;
 785
 786                         /*
 787                          * If the "write-side" has been blocked, wake it up now.
 788                          */
 789                         if (rpipe->pipe_state & PIPE_WANTW) {
 790                                 rpipe->pipe_state &= ~PIPE_WANTW;
 791                                 wakeup(rpipe);
 792                         }
 793
 794                         /*
 795                          * Break if some data was read.
 796                          */
 797                         if (nread > 0)
 798                                 break;
 799
 800                         /*
 801                          * Unlock the pipe buffer for our remaining processing.
 802                          * We will either break out with an error or we will
 803                          * sleep and relock to loop.
 804                          */
 805                         pipeunlock(rpipe);
 806
 807                         /*
 808                          * Handle non-blocking mode operation or
 809                          * wait for more data.
 810                          */
 811                         if (fp->f_flag & FNONBLOCK) {
 812                                 error = EAGAIN;
 813                         } else {
 814                                 rpipe->pipe_state |= PIPE_WANTR;
 815                                 if ((error = msleep(rpipe, PIPE_MTX(rpipe),
 816                                     PRIBIO | PCATCH,
 817                                     "piperd", 0)) == 0)
 818                                         error = pipelock(rpipe, 1);
 819                         }
 820                         if (error)
 821                                 goto unlocked_error;
 822                 }
 823         }
 824 #ifdef MAC
 825 locked_error:
 826 #endif
 827         pipeunlock(rpipe);
 828
 829         /* XXX: should probably do this before getting any locks. */
 830         if (error == 0)
 831                 pipe_timestamp(&rpipe->pipe_atime);
 832 unlocked_error:
 833         --rpipe->pipe_busy;
 834
 835         /*
 836          * PIPE_WANT processing only makes sense if pipe_busy is 0.
 837          */
 838         if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) {
 839                 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW);
 840                 wakeup(rpipe);
 841         } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) {
 842                 /*
 843                  * Handle write blocking hysteresis.
 844                  */
 845                 if (rpipe->pipe_state & PIPE_WANTW) {
 846                         rpipe->pipe_state &= ~PIPE_WANTW;
 847                         wakeup(rpipe);
 848                 }
 849         }
 850
 851         /*
 852          * Only wake up writers if there was actually something read.
 853          * Otherwise, when calling read(2) at EOF, a spurious wakeup occurs.
 854          */
 855         if (nread > 0 &&
 856             rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt >= PIPE_BUF)
 857                 pipeselwakeup(rpipe);
 858
 859         PIPE_UNLOCK(rpipe);
 860         return (error);
 861 }
 862
 863 #ifndef PIPE_NODIRECT
 864 /*
 865  * Map the sending processes' buffer into kernel space and wire it.
 866  * This is similar to a physical write operation.
 867  */
 868 static int
 869 pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio)
 870 {
 871         u_int size;
 872         int i;
 873
 874         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
 875         KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,
 876             ("%s: PIPE_DIRECTW set on %p", __func__, wpipe));
 877         KASSERT(wpipe->pipe_pages.cnt == 0,
 878             ("%s: pipe map for %p contains residual data", __func__, wpipe));
 879
 880         if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size)
 881                 size = wpipe->pipe_buffer.size;
 882         else
 883                 size = uio->uio_iov->iov_len;
 884
 885         wpipe->pipe_state |= PIPE_DIRECTW;
 886         PIPE_UNLOCK(wpipe);
 887         i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
 888             (vm_offset_t)uio->uio_iov->iov_base, size, VM_PROT_READ,
 889             wpipe->pipe_pages.ms, PIPENPAGES);
 890         PIPE_LOCK(wpipe);
 891         if (i < 0) {
 892                 wpipe->pipe_state &= ~PIPE_DIRECTW;
 893                 return (EFAULT);
 894         }
 895
 896         wpipe->pipe_pages.npages = i;
 897         wpipe->pipe_pages.pos =
 898             ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK;
 899         wpipe->pipe_pages.cnt = size;
 900
 901         uio->uio_iov->iov_len -= size;
 902         uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size;
 903         if (uio->uio_iov->iov_len == 0)
 904                 uio->uio_iov++;
 905         uio->uio_resid -= size;
 906         uio->uio_offset += size;
 907         return (0);
 908 }
 909
 910 /*
 911  * Unwire the process buffer.
 912  */
 913 static void
 914 pipe_destroy_write_buffer(struct pipe *wpipe)
 915 {
 916
 917         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
 918         KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,
 919             ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));
 920         KASSERT(wpipe->pipe_pages.cnt == 0,
 921             ("%s: pipe map for %p contains residual data", __func__, wpipe));
 922
 923         wpipe->pipe_state &= ~PIPE_DIRECTW;
 924         vm_page_unhold_pages(wpipe->pipe_pages.ms, wpipe->pipe_pages.npages);
 925         wpipe->pipe_pages.npages = 0;
 926 }
 927
 928 /*
 929  * In the case of a signal, the writing process might go away.  This
 930  * code copies the data into the circular buffer so that the source
 931  * pages can be freed without loss of data.
 932  */
 933 static void
 934 pipe_clone_write_buffer(struct pipe *wpipe)
 935 {
 936         struct uio uio;
 937         struct iovec iov;
 938         int size;
 939         int pos;
 940
 941         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
 942         KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,
 943             ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));
 944
 945         size = wpipe->pipe_pages.cnt;
 946         pos = wpipe->pipe_pages.pos;
 947         wpipe->pipe_pages.cnt = 0;
 948
 949         wpipe->pipe_buffer.in = size;
 950         wpipe->pipe_buffer.out = 0;
 951         wpipe->pipe_buffer.cnt = size;
 952
 953         PIPE_UNLOCK(wpipe);
 954         iov.iov_base = wpipe->pipe_buffer.buffer;
 955         iov.iov_len = size;
 956         uio.uio_iov = &iov;
 957         uio.uio_iovcnt = 1;
 958         uio.uio_offset = 0;
 959         uio.uio_resid = size;
 960         uio.uio_segflg = UIO_SYSSPACE;
 961         uio.uio_rw = UIO_READ;
 962         uio.uio_td = curthread;
 963         uiomove_fromphys(wpipe->pipe_pages.ms, pos, size, &uio);
 964         PIPE_LOCK(wpipe);
 965         pipe_destroy_write_buffer(wpipe);
 966 }
 967
 968 /*
 969  * This implements the pipe buffer write mechanism.  Note that only
 970  * a direct write OR a normal pipe write can be pending at any given time.
 971  * If there are any characters in the pipe buffer, the direct write will
 972  * be deferred until the receiving process grabs all of the bytes from
 973  * the pipe buffer.  Then the direct mapping write is set-up.
 974  */
 975 static int
 976 pipe_direct_write(struct pipe *wpipe, struct uio *uio)
 977 {
 978         int error;
 979
 980 retry:
 981         PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
 982         if ((wpipe->pipe_state & PIPE_EOF) != 0) {
 983                 error = EPIPE;
 984                 goto error1;
 985         }
 986         if (wpipe->pipe_state & PIPE_DIRECTW) {
 987                 if (wpipe->pipe_state & PIPE_WANTR) {
 988                         wpipe->pipe_state &= ~PIPE_WANTR;
 989                         wakeup(wpipe);
 990                 }
 991                 pipeselwakeup(wpipe);
 992                 wpipe->pipe_state |= PIPE_WANTW;
 993                 pipeunlock(wpipe);
 994                 error = msleep(wpipe, PIPE_MTX(wpipe),
 995                     PRIBIO | PCATCH, "pipdww", 0);
 996                 pipelock(wpipe, 0);
 997                 if (error != 0)
 998                         goto error1;
 999                 goto retry;
1000         }
1001         if (wpipe->pipe_buffer.cnt > 0) {
1002                 if (wpipe->pipe_state & PIPE_WANTR) {
1003                         wpipe->pipe_state &= ~PIPE_WANTR;
1004                         wakeup(wpipe);
1005                 }
1006                 pipeselwakeup(wpipe);
1007                 wpipe->pipe_state |= PIPE_WANTW;
1008                 pipeunlock(wpipe);
1009                 error = msleep(wpipe, PIPE_MTX(wpipe),
1010                     PRIBIO | PCATCH, "pipdwc", 0);
1011                 pipelock(wpipe, 0);
1012                 if (error != 0)
1013                         goto error1;
1014                 goto retry;
1015         }
1016
1017         error = pipe_build_write_buffer(wpipe, uio);
1018         if (error) {
1019                 goto error1;
1020         }
1021
1022         while (wpipe->pipe_pages.cnt != 0 &&
1023             (wpipe->pipe_state & PIPE_EOF) == 0) {
1024                 if (wpipe->pipe_state & PIPE_WANTR) {
1025                         wpipe->pipe_state &= ~PIPE_WANTR;
1026                         wakeup(wpipe);
1027                 }
1028                 pipeselwakeup(wpipe);
1029                 wpipe->pipe_state |= PIPE_WANTW;
1030                 pipeunlock(wpipe);
1031                 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,
1032                     "pipdwt", 0);
1033                 pipelock(wpipe, 0);
1034                 if (error != 0)
1035                         break;
1036         }
1037
1038         if ((wpipe->pipe_state & PIPE_EOF) != 0) {
1039                 wpipe->pipe_pages.cnt = 0;
1040                 pipe_destroy_write_buffer(wpipe);
1041                 pipeselwakeup(wpipe);
1042                 error = EPIPE;
1043         } else if (error == EINTR || error == ERESTART) {
1044                 pipe_clone_write_buffer(wpipe);
1045         } else {
1046                 pipe_destroy_write_buffer(wpipe);
1047         }
1048         KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,
1049             ("pipe %p leaked PIPE_DIRECTW", wpipe));
1050         return (error);
1051
1052 error1:
1053         wakeup(wpipe);
1054         return (error);
1055 }
1056 #endif
1057
1058 static int
1059 pipe_write(struct file *fp, struct uio *uio, struct ucred *active_cred,
1060     int flags, struct thread *td)
1061 {
1062         struct pipe *wpipe, *rpipe;
1063         ssize_t orig_resid;
1064         int desiredsize, error;
1065
1066         rpipe = fp->f_data;
1067         wpipe = PIPE_PEER(rpipe);
1068         PIPE_LOCK(rpipe);
1069         error = pipelock(wpipe, 1);
1070         if (error) {
1071                 PIPE_UNLOCK(rpipe);
1072                 return (error);
1073         }
1074         /*
1075          * detect loss of pipe read side, issue SIGPIPE if lost.
1076          */
1077         if (wpipe->pipe_present != PIPE_ACTIVE ||
1078             (wpipe->pipe_state & PIPE_EOF)) {
1079                 pipeunlock(wpipe);
1080                 PIPE_UNLOCK(rpipe);
1081                 return (EPIPE);
1082         }
1083 #ifdef MAC
1084         error = mac_pipe_check_write(active_cred, wpipe->pipe_pair);
1085         if (error) {
1086                 pipeunlock(wpipe);
1087                 PIPE_UNLOCK(rpipe);
1088                 return (error);
1089         }
1090 #endif
1091         ++wpipe->pipe_busy;
1092
1093         /* Choose a larger size if it's advantageous */
1094         desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size);
1095         while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) {
1096                 if (piperesizeallowed != 1)
1097                         break;
1098                 if (amountpipekva > maxpipekva / 2)
1099                         break;
1100                 if (desiredsize == BIG_PIPE_SIZE)
1101                         break;
1102                 desiredsize = desiredsize * 2;
1103         }
1104
1105         /* Choose a smaller size if we're in a OOM situation */
1106         if (amountpipekva > (3 * maxpipekva) / 4 &&
1107             wpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&
1108             wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&
1109             piperesizeallowed == 1)
1110                 desiredsize = SMALL_PIPE_SIZE;
1111
1112         /* Resize if the above determined that a new size was necessary */
1113         if (desiredsize != wpipe->pipe_buffer.size &&
1114             (wpipe->pipe_state & PIPE_DIRECTW) == 0) {
1115                 PIPE_UNLOCK(wpipe);
1116                 pipespace(wpipe, desiredsize);
1117                 PIPE_LOCK(wpipe);
1118         }
1119         MPASS(wpipe->pipe_buffer.size != 0);
1120
1121         orig_resid = uio->uio_resid;
1122
1123         while (uio->uio_resid) {
1124                 int space;
1125
1126                 if (wpipe->pipe_state & PIPE_EOF) {
1127                         error = EPIPE;
1128                         break;
1129                 }
1130 #ifndef PIPE_NODIRECT
1131                 /*
1132                  * If the transfer is large, we can gain performance if
1133                  * we do process-to-process copies directly.
1134                  * If the write is non-blocking, we don't use the
1135                  * direct write mechanism.
1136                  *
1137                  * The direct write mechanism will detect the reader going
1138                  * away on us.
1139                  */
1140                 if (uio->uio_segflg == UIO_USERSPACE &&
1141                     uio->uio_iov->iov_len >= PIPE_MINDIRECT &&
1142                     wpipe->pipe_buffer.size >= PIPE_MINDIRECT &&
1143                     (fp->f_flag & FNONBLOCK) == 0) {
1144                         error = pipe_direct_write(wpipe, uio);
1145                         if (error != 0)
1146                                 break;
1147                         continue;
1148                 }
1149 #endif
1150
1151                 /*
1152                  * Pipe buffered writes cannot be coincidental with
1153                  * direct writes.  We wait until the currently executing
1154                  * direct write is completed before we start filling the
1155                  * pipe buffer.  We break out if a signal occurs or the
1156                  * reader goes away.
1157                  */
1158                 if (wpipe->pipe_pages.cnt != 0) {
1159                         if (wpipe->pipe_state & PIPE_WANTR) {
1160                                 wpipe->pipe_state &= ~PIPE_WANTR;
1161                                 wakeup(wpipe);
1162                         }
1163                         pipeselwakeup(wpipe);
1164                         wpipe->pipe_state |= PIPE_WANTW;
1165                         pipeunlock(wpipe);
1166                         error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,
1167                             "pipbww", 0);
1168                         pipelock(wpipe, 0);
1169                         if (error != 0)
1170                                 break;
1171                         continue;
1172                 }
1173
1174                 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
1175
1176                 /* Writes of size <= PIPE_BUF must be atomic. */
1177                 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF))
1178                         space = 0;
1179
1180                 if (space > 0) {
1181                         int size;       /* Transfer size */
1182                         int segsize;    /* first segment to transfer */
1183
1184                         /*
1185                          * Transfer size is minimum of uio transfer
1186                          * and free space in pipe buffer.
1187                          */
1188                         if (space > uio->uio_resid)
1189                                 size = uio->uio_resid;
1190                         else
1191                                 size = space;
1192                         /*
1193                          * First segment to transfer is minimum of
1194                          * transfer size and contiguous space in
1195                          * pipe buffer.  If first segment to transfer
1196                          * is less than the transfer size, we've got
1197                          * a wraparound in the buffer.
1198                          */
1199                         segsize = wpipe->pipe_buffer.size -
1200                                 wpipe->pipe_buffer.in;
1201                         if (segsize > size)
1202                                 segsize = size;
1203
1204                         /* Transfer first segment */
1205
1206                         PIPE_UNLOCK(rpipe);
1207                         error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],
1208                                         segsize, uio);
1209                         PIPE_LOCK(rpipe);
1210
1211                         if (error == 0 && segsize < size) {
1212                                 KASSERT(wpipe->pipe_buffer.in + segsize ==
1213                                         wpipe->pipe_buffer.size,
1214                                         ("Pipe buffer wraparound disappeared"));
1215                                 /*
1216                                  * Transfer remaining part now, to
1217                                  * support atomic writes.  Wraparound
1218                                  * happened.
1219                                  */
1220
1221                                 PIPE_UNLOCK(rpipe);
1222                                 error = uiomove(
1223                                     &wpipe->pipe_buffer.buffer[0],
1224                                     size - segsize, uio);
1225                                 PIPE_LOCK(rpipe);
1226                         }
1227                         if (error == 0) {
1228                                 wpipe->pipe_buffer.in += size;
1229                                 if (wpipe->pipe_buffer.in >=
1230                                     wpipe->pipe_buffer.size) {
1231                                         KASSERT(wpipe->pipe_buffer.in ==
1232                                                 size - segsize +
1233                                                 wpipe->pipe_buffer.size,
1234                                                 ("Expected wraparound bad"));
1235                                         wpipe->pipe_buffer.in = size - segsize;
1236                                 }
1237
1238                                 wpipe->pipe_buffer.cnt += size;
1239                                 KASSERT(wpipe->pipe_buffer.cnt <=
1240                                         wpipe->pipe_buffer.size,
1241                                         ("Pipe buffer overflow"));
1242                         }
1243                         if (error != 0)
1244                                 break;
1245                         continue;
1246                 } else {
1247                         /*
1248                          * If the "read-side" has been blocked, wake it up now.
1249                          */
1250                         if (wpipe->pipe_state & PIPE_WANTR) {
1251                                 wpipe->pipe_state &= ~PIPE_WANTR;
1252                                 wakeup(wpipe);
1253                         }
1254
1255                         /*
1256                          * don't block on non-blocking I/O
1257                          */
1258                         if (fp->f_flag & FNONBLOCK) {
1259                                 error = EAGAIN;
1260                                 break;
1261                         }
1262
1263                         /*
1264                          * We have no more space and have something to offer,
1265                          * wake up select/poll.
1266                          */
1267                         pipeselwakeup(wpipe);
1268
1269                         wpipe->pipe_state |= PIPE_WANTW;
1270                         pipeunlock(wpipe);
1271                         error = msleep(wpipe, PIPE_MTX(rpipe),
1272                             PRIBIO | PCATCH, "pipewr", 0);
1273                         pipelock(wpipe, 0);
1274                         if (error != 0)
1275                                 break;
1276                         continue;
1277                 }
1278         }
1279
1280         --wpipe->pipe_busy;
1281
1282         if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {
1283                 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
1284                 wakeup(wpipe);
1285         } else if (wpipe->pipe_buffer.cnt > 0) {
1286                 /*
1287                  * If we have put any characters in the buffer, we wake up
1288                  * the reader.
1289                  */
1290                 if (wpipe->pipe_state & PIPE_WANTR) {
1291                         wpipe->pipe_state &= ~PIPE_WANTR;
1292                         wakeup(wpipe);
1293                 }
1294         }
1295
1296         /*
1297          * Don't return EPIPE if any byte was written.
1298          * EINTR and other interrupts are handled by generic I/O layer.
1299          * Do not pretend that I/O succeeded for obvious user error
1300          * like EFAULT.
1301          */
1302         if (uio->uio_resid != orig_resid && error == EPIPE)
1303                 error = 0;
1304
1305         if (error == 0)
1306                 pipe_timestamp(&wpipe->pipe_mtime);
1307
1308         /*
1309          * We have something to offer,
1310          * wake up select/poll.
1311          */
1312         if (wpipe->pipe_buffer.cnt)
1313                 pipeselwakeup(wpipe);
1314
1315         pipeunlock(wpipe);
1316         PIPE_UNLOCK(rpipe);
1317         return (error);
1318 }
1319
1320 /* ARGSUSED */
1321 static int
1322 pipe_truncate(struct file *fp, off_t length, struct ucred *active_cred,
1323     struct thread *td)
1324 {
1325         struct pipe *cpipe;
1326         int error;
1327
1328         cpipe = fp->f_data;
1329         if (cpipe->pipe_type & PIPE_TYPE_NAMED)
1330                 error = vnops.fo_truncate(fp, length, active_cred, td);
1331         else
1332                 error = invfo_truncate(fp, length, active_cred, td);
1333         return (error);
1334 }
1335
1336 /*
1337  * we implement a very minimal set of ioctls for compatibility with sockets.
1338  */
1339 static int
1340 pipe_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *active_cred,
1341     struct thread *td)
1342 {
1343         struct pipe *mpipe = fp->f_data;
1344         int error;
1345
1346         PIPE_LOCK(mpipe);
1347
1348 #ifdef MAC
1349         error = mac_pipe_check_ioctl(active_cred, mpipe->pipe_pair, cmd, data);
1350         if (error) {
1351                 PIPE_UNLOCK(mpipe);
1352                 return (error);
1353         }
1354 #endif
1355
1356         error = 0;
1357         switch (cmd) {
1358         case FIONBIO:
1359                 break;
1360
1361         case FIOASYNC:
1362                 if (*(int *)data) {
1363                         mpipe->pipe_state |= PIPE_ASYNC;
1364                 } else {
1365                         mpipe->pipe_state &= ~PIPE_ASYNC;
1366                 }
1367                 break;
1368
1369         case FIONREAD:
1370                 if (!(fp->f_flag & FREAD)) {
1371                         *(int *)data = 0;
1372                         PIPE_UNLOCK(mpipe);
1373                         return (0);
1374                 }
1375                 if (mpipe->pipe_pages.cnt != 0)
1376                         *(int *)data = mpipe->pipe_pages.cnt;
1377                 else
1378                         *(int *)data = mpipe->pipe_buffer.cnt;
1379                 break;
1380
1381         case FIOSETOWN:
1382                 PIPE_UNLOCK(mpipe);
1383                 error = fsetown(*(int *)data, &mpipe->pipe_sigio);
1384                 goto out_unlocked;
1385
1386         case FIOGETOWN:
1387                 *(int *)data = fgetown(&mpipe->pipe_sigio);
1388                 break;
1389
1390         /* This is deprecated, FIOSETOWN should be used instead. */
1391         case TIOCSPGRP:
1392                 PIPE_UNLOCK(mpipe);
1393                 error = fsetown(-(*(int *)data), &mpipe->pipe_sigio);
1394                 goto out_unlocked;
1395
1396         /* This is deprecated, FIOGETOWN should be used instead. */
1397         case TIOCGPGRP:
1398                 *(int *)data = -fgetown(&mpipe->pipe_sigio);
1399                 break;
1400
1401         default:
1402                 error = ENOTTY;
1403                 break;
1404         }
1405         PIPE_UNLOCK(mpipe);
1406 out_unlocked:
1407         return (error);
1408 }
1409
1410 static int
1411 pipe_poll(struct file *fp, int events, struct ucred *active_cred,
1412     struct thread *td)
1413 {
1414         struct pipe *rpipe;
1415         struct pipe *wpipe;
1416         int levents, revents;
1417 #ifdef MAC
1418         int error;
1419 #endif
1420
1421         revents = 0;
1422         rpipe = fp->f_data;
1423         wpipe = PIPE_PEER(rpipe);
1424         PIPE_LOCK(rpipe);
1425 #ifdef MAC
1426         error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair);
1427         if (error)
1428                 goto locked_error;
1429 #endif
1430         if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM))
1431                 if (rpipe->pipe_pages.cnt > 0 || rpipe->pipe_buffer.cnt > 0)
1432                         revents |= events & (POLLIN | POLLRDNORM);
1433
1434         if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM))
1435                 if (wpipe->pipe_present != PIPE_ACTIVE ||
1436                     (wpipe->pipe_state & PIPE_EOF) ||
1437                     ((wpipe->pipe_state & PIPE_DIRECTW) == 0 &&
1438                      ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF ||
1439                          wpipe->pipe_buffer.size == 0)))
1440                         revents |= events & (POLLOUT | POLLWRNORM);
1441
1442         levents = events &
1443             (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | POLLRDBAND);
1444         if (rpipe->pipe_type & PIPE_TYPE_NAMED && fp->f_flag & FREAD && levents &&
1445             fp->f_pipegen == rpipe->pipe_wgen)
1446                 events |= POLLINIGNEOF;
1447
1448         if ((events & POLLINIGNEOF) == 0) {
1449                 if (rpipe->pipe_state & PIPE_EOF) {
1450                         if (fp->f_flag & FREAD)
1451                                 revents |= (events & (POLLIN | POLLRDNORM));
1452                         if (wpipe->pipe_present != PIPE_ACTIVE ||
1453                             (wpipe->pipe_state & PIPE_EOF))
1454                                 revents |= POLLHUP;
1455                 }
1456         }
1457
1458         if (revents == 0) {
1459                 /*
1460                  * Add ourselves regardless of eventmask as we have to return
1461                  * POLLHUP even if it was not asked for.
1462                  */
1463                 if ((fp->f_flag & FREAD) != 0) {
1464                         selrecord(td, &rpipe->pipe_sel);
1465                         if (SEL_WAITING(&rpipe->pipe_sel))
1466                                 rpipe->pipe_state |= PIPE_SEL;
1467                 }
1468
1469                 if ((fp->f_flag & FWRITE) != 0) {
1470                         selrecord(td, &wpipe->pipe_sel);
1471                         if (SEL_WAITING(&wpipe->pipe_sel))
1472                                 wpipe->pipe_state |= PIPE_SEL;
1473                 }
1474         }
1475 #ifdef MAC
1476 locked_error:
1477 #endif
1478         PIPE_UNLOCK(rpipe);
1479
1480         return (revents);
1481 }
1482
1483 /*
1484  * We shouldn't need locks here as we're doing a read and this should
1485  * be a natural race.
1486  */
1487 static int
1488 pipe_stat(struct file *fp, struct stat *ub, struct ucred *active_cred,
1489     struct thread *td)
1490 {
1491         struct pipe *pipe;
1492 #ifdef MAC
1493         int error;
1494 #endif
1495
1496         pipe = fp->f_data;
1497 #ifdef MAC
1498         if (mac_pipe_check_stat_enabled()) {
1499                 PIPE_LOCK(pipe);
1500                 error = mac_pipe_check_stat(active_cred, pipe->pipe_pair);
1501                 PIPE_UNLOCK(pipe);
1502                 if (error) {
1503                         return (error);
1504                 }
1505         }
1506 #endif
1507
1508         /* For named pipes ask the underlying filesystem. */
1509         if (pipe->pipe_type & PIPE_TYPE_NAMED) {
1510                 return (vnops.fo_stat(fp, ub, active_cred, td));
1511         }
1512
1513         bzero(ub, sizeof(*ub));
1514         ub->st_mode = S_IFIFO;
1515         ub->st_blksize = PAGE_SIZE;
1516         if (pipe->pipe_pages.cnt != 0)
1517                 ub->st_size = pipe->pipe_pages.cnt;
1518         else
1519                 ub->st_size = pipe->pipe_buffer.cnt;
1520         ub->st_blocks = howmany(ub->st_size, ub->st_blksize);
1521         ub->st_atim = pipe->pipe_atime;
1522         ub->st_mtim = pipe->pipe_mtime;
1523         ub->st_ctim = pipe->pipe_ctime;
1524         ub->st_uid = fp->f_cred->cr_uid;
1525         ub->st_gid = fp->f_cred->cr_gid;
1526         ub->st_dev = pipedev_ino;
1527         ub->st_ino = pipe->pipe_ino;
1528         /*
1529          * Left as 0: st_nlink, st_rdev, st_flags, st_gen.
1530          */
1531         return (0);
1532 }
1533
1534 /* ARGSUSED */
1535 static int
1536 pipe_close(struct file *fp, struct thread *td)
1537 {
1538
1539         if (fp->f_vnode != NULL)
1540                 return vnops.fo_close(fp, td);
1541         fp->f_ops = &badfileops;
1542         pipe_dtor(fp->f_data);
1543         fp->f_data = NULL;
1544         return (0);
1545 }
1546
1547 static int
1548 pipe_chmod(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td)
1549 {
1550         struct pipe *cpipe;
1551         int error;
1552
1553         cpipe = fp->f_data;
1554         if (cpipe->pipe_type & PIPE_TYPE_NAMED)
1555                 error = vn_chmod(fp, mode, active_cred, td);
1556         else
1557                 error = invfo_chmod(fp, mode, active_cred, td);
1558         return (error);
1559 }
1560
1561 static int
1562 pipe_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
1563     struct thread *td)
1564 {
1565         struct pipe *cpipe;
1566         int error;
1567
1568         cpipe = fp->f_data;
1569         if (cpipe->pipe_type & PIPE_TYPE_NAMED)
1570                 error = vn_chown(fp, uid, gid, active_cred, td);
1571         else
1572                 error = invfo_chown(fp, uid, gid, active_cred, td);
1573         return (error);
1574 }
1575
1576 static int
1577 pipe_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
1578 {
1579         struct pipe *pi;
1580
1581         if (fp->f_type == DTYPE_FIFO)
1582                 return (vn_fill_kinfo(fp, kif, fdp));
1583         kif->kf_type = KF_TYPE_PIPE;
1584         pi = fp->f_data;
1585         kif->kf_un.kf_pipe.kf_pipe_addr = (uintptr_t)pi;
1586         kif->kf_un.kf_pipe.kf_pipe_peer = (uintptr_t)pi->pipe_peer;
1587         kif->kf_un.kf_pipe.kf_pipe_buffer_cnt = pi->pipe_buffer.cnt;
1588         return (0);
1589 }
1590
1591 static void
1592 pipe_free_kmem(struct pipe *cpipe)
1593 {
1594
1595         KASSERT(!mtx_owned(PIPE_MTX(cpipe)),
1596             ("pipe_free_kmem: pipe mutex locked"));
1597
1598         if (cpipe->pipe_buffer.buffer != NULL) {
1599                 atomic_subtract_long(&amountpipekva, cpipe->pipe_buffer.size);
1600                 vm_map_remove(pipe_map,
1601                     (vm_offset_t)cpipe->pipe_buffer.buffer,
1602                     (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size);
1603                 cpipe->pipe_buffer.buffer = NULL;
1604         }
1605 #ifndef PIPE_NODIRECT
1606         {
1607                 cpipe->pipe_pages.cnt = 0;
1608                 cpipe->pipe_pages.pos = 0;
1609                 cpipe->pipe_pages.npages = 0;
1610         }
1611 #endif
1612 }
1613
1614 /*
1615  * shutdown the pipe
1616  */
1617 static void
1618 pipeclose(struct pipe *cpipe)
1619 {
1620         struct pipepair *pp;
1621         struct pipe *ppipe;
1622
1623         KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL"));
1624
1625         PIPE_LOCK(cpipe);
1626         pipelock(cpipe, 0);
1627         pp = cpipe->pipe_pair;
1628
1629         /*
1630          * If the other side is blocked, wake it up saying that
1631          * we want to close it down.
1632          */
1633         cpipe->pipe_state |= PIPE_EOF;
1634         while (cpipe->pipe_busy) {
1635                 wakeup(cpipe);
1636                 cpipe->pipe_state |= PIPE_WANT;
1637                 pipeunlock(cpipe);
1638                 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0);
1639                 pipelock(cpipe, 0);
1640         }
1641
1642         pipeselwakeup(cpipe);
1643
1644         /*
1645          * Disconnect from peer, if any.
1646          */
1647         ppipe = cpipe->pipe_peer;
1648         if (ppipe->pipe_present == PIPE_ACTIVE) {
1649                 ppipe->pipe_state |= PIPE_EOF;
1650                 wakeup(ppipe);
1651                 pipeselwakeup(ppipe);
1652         }
1653
1654         /*
1655          * Mark this endpoint as free.  Release kmem resources.  We
1656          * don't mark this endpoint as unused until we've finished
1657          * doing that, or the pipe might disappear out from under
1658          * us.
1659          */
1660         PIPE_UNLOCK(cpipe);
1661         pipe_free_kmem(cpipe);
1662         PIPE_LOCK(cpipe);
1663         cpipe->pipe_present = PIPE_CLOSING;
1664         pipeunlock(cpipe);
1665
1666         /*
1667          * knlist_clear() may sleep dropping the PIPE_MTX. Set the
1668          * PIPE_FINALIZED, that allows other end to free the
1669          * pipe_pair, only after the knotes are completely dismantled.
1670          */
1671         knlist_clear(&cpipe->pipe_sel.si_note, 1);
1672         cpipe->pipe_present = PIPE_FINALIZED;
1673         seldrain(&cpipe->pipe_sel);
1674         knlist_destroy(&cpipe->pipe_sel.si_note);
1675
1676         /*
1677          * If both endpoints are now closed, release the memory for the
1678          * pipe pair.  If not, unlock.
1679          */
1680         if (ppipe->pipe_present == PIPE_FINALIZED) {
1681                 PIPE_UNLOCK(cpipe);
1682 #ifdef MAC
1683                 mac_pipe_destroy(pp);
1684 #endif
1685                 uma_zfree(pipe_zone, cpipe->pipe_pair);
1686         } else
1687                 PIPE_UNLOCK(cpipe);
1688 }
1689
1690 /*ARGSUSED*/
1691 static int
1692 pipe_kqfilter(struct file *fp, struct knote *kn)
1693 {
1694         struct pipe *cpipe;
1695
1696         /*
1697          * If a filter is requested that is not supported by this file
1698          * descriptor, don't return an error, but also don't ever generate an
1699          * event.
1700          */
1701         if ((kn->kn_filter == EVFILT_READ) && !(fp->f_flag & FREAD)) {
1702                 kn->kn_fop = &pipe_nfiltops;
1703                 return (0);
1704         }
1705         if ((kn->kn_filter == EVFILT_WRITE) && !(fp->f_flag & FWRITE)) {
1706                 kn->kn_fop = &pipe_nfiltops;
1707                 return (0);
1708         }
1709         cpipe = fp->f_data;
1710         PIPE_LOCK(cpipe);
1711         switch (kn->kn_filter) {
1712         case EVFILT_READ:
1713                 kn->kn_fop = &pipe_rfiltops;
1714                 break;
1715         case EVFILT_WRITE:
1716                 kn->kn_fop = &pipe_wfiltops;
1717                 if (cpipe->pipe_peer->pipe_present != PIPE_ACTIVE) {
1718                         /* other end of pipe has been closed */
1719                         PIPE_UNLOCK(cpipe);
1720                         return (EPIPE);
1721                 }
1722                 cpipe = PIPE_PEER(cpipe);
1723                 break;
1724         default:
1725                 PIPE_UNLOCK(cpipe);
1726                 return (EINVAL);
1727         }
1728
1729         kn->kn_hook = cpipe;
1730         knlist_add(&cpipe->pipe_sel.si_note, kn, 1);
1731         PIPE_UNLOCK(cpipe);
1732         return (0);
1733 }
1734
1735 static void
1736 filt_pipedetach(struct knote *kn)
1737 {
1738         struct pipe *cpipe = kn->kn_hook;
1739
1740         PIPE_LOCK(cpipe);
1741         knlist_remove(&cpipe->pipe_sel.si_note, kn, 1);
1742         PIPE_UNLOCK(cpipe);
1743 }
1744
1745 /*ARGSUSED*/
1746 static int
1747 filt_piperead(struct knote *kn, long hint)
1748 {
1749         struct file *fp = kn->kn_fp;
1750         struct pipe *rpipe = kn->kn_hook;
1751
1752         PIPE_LOCK_ASSERT(rpipe, MA_OWNED);
1753         kn->kn_data = rpipe->pipe_buffer.cnt;
1754         if (kn->kn_data == 0)
1755                 kn->kn_data = rpipe->pipe_pages.cnt;
1756
1757         if ((rpipe->pipe_state & PIPE_EOF) != 0 &&
1758             ((rpipe->pipe_type & PIPE_TYPE_NAMED) == 0 ||
1759             fp->f_pipegen != rpipe->pipe_wgen)) {
1760                 kn->kn_flags |= EV_EOF;
1761                 return (1);
1762         }
1763         kn->kn_flags &= ~EV_EOF;
1764         return (kn->kn_data > 0);
1765 }
1766
1767 /*ARGSUSED*/
1768 static int
1769 filt_pipewrite(struct knote *kn, long hint)
1770 {
1771         struct pipe *wpipe = kn->kn_hook;
1772
1773         /*
1774          * If this end of the pipe is closed, the knote was removed from the
1775          * knlist and the list lock (i.e., the pipe lock) is therefore not held.
1776          */
1777         if (wpipe->pipe_present == PIPE_ACTIVE ||
1778             (wpipe->pipe_type & PIPE_TYPE_NAMED) != 0) {
1779                 PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
1780
1781                 if (wpipe->pipe_state & PIPE_DIRECTW) {
1782                         kn->kn_data = 0;
1783                 } else if (wpipe->pipe_buffer.size > 0) {
1784                         kn->kn_data = wpipe->pipe_buffer.size -
1785                             wpipe->pipe_buffer.cnt;
1786                 } else {
1787                         kn->kn_data = PIPE_BUF;
1788                 }
1789         }
1790
1791         if (wpipe->pipe_present != PIPE_ACTIVE ||
1792             (wpipe->pipe_state & PIPE_EOF)) {
1793                 kn->kn_flags |= EV_EOF;
1794                 return (1);
1795         }
1796         kn->kn_flags &= ~EV_EOF;
1797         return (kn->kn_data >= PIPE_BUF);
1798 }
1799
1800 static void
1801 filt_pipedetach_notsup(struct knote *kn)
1802 {
1803
1804 }
1805
1806 static int
1807 filt_pipenotsup(struct knote *kn, long hint)
1808 {
1809
1810         return (0);
1811 }