sys/fs/nfsclient/nfs_clstate.c

   1 /*-
   2  * Copyright (c) 2009 Rick Macklem, University of Guelph
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 /*
  32  * These functions implement the client side state handling for NFSv4.
  33  * NFSv4 state handling:
  34  * - A lockowner is used to determine lock contention, so it
  35  *   corresponds directly to a Posix pid. (1 to 1 mapping)
  36  * - The correct granularity of an OpenOwner is not nearly so
  37  *   obvious. An OpenOwner does the following:
  38  *   - provides a serial sequencing of Open/Close/Lock-with-new-lockowner
  39  *   - is used to check for Open/SHare contention (not applicable to
  40  *     this client, since all Opens are Deny_None)
  41  *   As such, I considered both extrema.
  42  *   1 OpenOwner per ClientID - Simple to manage, but fully serializes
  43  *   all Open, Close and Lock (with a new lockowner) Ops.
  44  *   1 OpenOwner for each Open - This one results in an OpenConfirm for
  45  *   every Open, for most servers.
  46  *   So, I chose to use the same mapping as I did for LockOwnwers.
  47  *   The main concern here is that you can end up with multiple Opens
  48  *   for the same File Handle, but on different OpenOwners (opens
  49  *   inherited from parents, grandparents...) and you do not know
  50  *   which of these the vnodeop close applies to. This is handled by
  51  *   delaying the Close Op(s) until all of the Opens have been closed.
  52  *   (It is not yet obvious if this is the correct granularity.)
  53  * - How the code handles serailization:
  54  *   - For the ClientId, is uses an exclusive lock while getting its
  55  *     SetClientId and during recovery. Otherwise, it uses a shared
  56  *     lock via a reference count.
  57  *   - For the rest of the data structures, it uses an SMP mutex
  58  *     (once the nfs client is SMP safe) and doesn't sleep while
  59  *     manipulating the linked lists.
  60  *   - The serialization of Open/Close/Lock/LockU falls out in the
  61  *     "wash", since OpenOwners and LockOwners are both mapped from
  62  *     Posix pid. In other words, there is only one Posix pid using
  63  *     any given owner, so that owner is serialized. (If you change
  64  *     the granularity of the OpenOwner, then code must be added to
  65  *     serialize Ops on the OpenOwner.)
  66  * - When to get rid of OpenOwners and LockOwners.
  67  *   - When a process exits, it calls nfscl_cleanup(), which goes
  68  *     through the client list looking for all Open and Lock Owners.
  69  *     When one is found, it is marked "defunct" or in the case of
  70  *     an OpenOwner without any Opens, freed.
  71  *     The renew thread scans for defunct Owners and gets rid of them,
  72  *     if it can. The LockOwners will also be deleted when the
  73  *     associated Open is closed.
  74  *   - If the LockU or Close Op(s) fail during close in a way
  75  *     that could be recovered upon retry, they are relinked to the
  76  *     ClientId's defunct open list and retried by the renew thread
  77  *     until they succeed or an unmount/recovery occurs.
  78  *     (Since we are done with them, they do not need to be recovered.)
  79  */
  80
  81 #ifndef APPLEKEXT
  82 #include <fs/nfs/nfsport.h>
  83
  84 /*
  85  * Global variables
  86  */
  87 extern struct nfsstats newnfsstats;
  88 extern struct nfsreqhead nfsd_reqq;
  89 NFSREQSPINLOCK;
  90 NFSCLSTATEMUTEX;
  91 int nfscl_inited = 0;
  92 struct nfsclhead nfsclhead;     /* Head of clientid list */
  93 int nfscl_deleghighwater = NFSCLDELEGHIGHWATER;
  94 #endif  /* !APPLEKEXT */
  95
  96 static int nfscl_delegcnt = 0;
  97 static int nfscl_getopen(struct nfsclownerhead *, u_int8_t *, int, u_int8_t *,
  98     NFSPROC_T *, u_int32_t, struct nfsclowner **, struct nfsclopen **);
  99 static void nfscl_clrelease(struct nfsclclient *);
 100 static void nfscl_cleanclient(struct nfsclclient *);
 101 static void nfscl_expireclient(struct nfsclclient *, struct nfsmount *,
 102     struct ucred *, NFSPROC_T *);
 103 static int nfscl_expireopen(struct nfsclclient *, struct nfsclopen *,
 104     struct nfsmount *, struct ucred *, NFSPROC_T *);
 105 static void nfscl_recover(struct nfsclclient *, struct ucred *, NFSPROC_T *);
 106 static void nfscl_insertlock(struct nfscllockowner *, struct nfscllock *,
 107     struct nfscllock *, int);
 108 static int nfscl_updatelock(struct nfscllockowner *, struct nfscllock **,
 109     struct nfscllock **, int);
 110 static void nfscl_delegreturnall(struct nfsclclient *, NFSPROC_T *);
 111 static u_int32_t nfscl_nextcbident(void);
 112 static mount_t nfscl_getmnt(u_int32_t);
 113 static struct nfscldeleg *nfscl_finddeleg(struct nfsclclient *, u_int8_t *,
 114     int);
 115 static int nfscl_checkconflict(struct nfscllockownerhead *, struct nfscllock *,
 116     u_int8_t *, struct nfscllock **);
 117 static void nfscl_freelockowner(struct nfscllockowner *, int);
 118 static void nfscl_freealllocks(struct nfscllockownerhead *, int);
 119 static int nfscl_localconflict(struct nfsclclient *, struct nfscllock *,
 120     u_int8_t *, struct nfscldeleg *, struct nfscllock **);
 121 static void nfscl_newopen(struct nfsclclient *, struct nfscldeleg *,
 122     struct nfsclowner **, struct nfsclowner **, struct nfsclopen **,
 123     struct nfsclopen **, u_int8_t *, u_int8_t *, int, int *);
 124 static int nfscl_moveopen(vnode_t , struct nfsclclient *,
 125     struct nfsmount *, struct nfsclopen *, struct nfsclowner *,
 126     struct nfscldeleg *, struct ucred *, NFSPROC_T *);
 127 static void nfscl_totalrecall(struct nfsclclient *);
 128 static int nfscl_relock(vnode_t , struct nfsclclient *, struct nfsmount *,
 129     struct nfscllockowner *, struct nfscllock *, struct ucred *, NFSPROC_T *);
 130 static int nfscl_tryopen(struct nfsmount *, vnode_t , u_int8_t *, int,
 131     u_int8_t *, int, u_int32_t, struct nfsclopen *, u_int8_t *, int,
 132     struct nfscldeleg **, int, u_int32_t, struct ucred *, NFSPROC_T *);
 133 static int nfscl_trylock(struct nfsmount *, vnode_t , u_int8_t *,
 134     int, struct nfscllockowner *, int, int, u_int64_t, u_int64_t, short,
 135     struct ucred *, NFSPROC_T *);
 136 static int nfsrpc_reopen(struct nfsmount *, u_int8_t *, int, u_int32_t,
 137     struct nfsclopen *, struct nfscldeleg **, struct ucred *, NFSPROC_T *);
 138 static void nfscl_freedeleg(struct nfscldeleghead *, struct nfscldeleg *);
 139 static int nfscl_errmap(struct nfsrv_descript *);
 140 static void nfscl_cleanup_common(struct nfsclclient *, u_int8_t *);
 141 static int nfscl_recalldeleg(struct nfsclclient *, struct nfsmount *,
 142     struct nfscldeleg *, vnode_t, struct ucred *, NFSPROC_T *);
 143 static void nfscl_freeopenowner(struct nfsclowner *, int);
 144 static void nfscl_cleandeleg(struct nfscldeleg *);
 145 static int nfscl_trydelegreturn(struct nfscldeleg *, struct ucred *,
 146     struct nfsmount *, NFSPROC_T *);
 147
 148 static short nfscberr_null[] = {
 149         0,
 150         0,
 151 };
 152
 153 static short nfscberr_getattr[] = {
 154         NFSERR_RESOURCE,
 155         NFSERR_BADHANDLE,
 156         NFSERR_BADXDR,
 157         NFSERR_RESOURCE,
 158         NFSERR_SERVERFAULT,
 159         0,
 160 };
 161
 162 static short nfscberr_recall[] = {
 163         NFSERR_RESOURCE,
 164         NFSERR_BADHANDLE,
 165         NFSERR_BADSTATEID,
 166         NFSERR_BADXDR,
 167         NFSERR_RESOURCE,
 168         NFSERR_SERVERFAULT,
 169         0,
 170 };
 171
 172 static short *nfscl_cberrmap[] = {
 173         nfscberr_null,
 174         nfscberr_null,
 175         nfscberr_null,
 176         nfscberr_getattr,
 177         nfscberr_recall
 178 };
 179
 180 #define NETFAMILY(clp) \
 181                 (((clp)->nfsc_flags & NFSCLFLAGS_AFINET6) ? AF_INET6 : AF_INET)
 182
 183 /*
 184  * Called for an open operation.
 185  * If the nfhp argument is NULL, just get an openowner.
 186  */
 187 APPLESTATIC int
 188 nfscl_open(vnode_t vp, u_int8_t *nfhp, int fhlen, u_int32_t amode, int usedeleg,
 189     struct ucred *cred, NFSPROC_T *p, struct nfsclowner **owpp,
 190     struct nfsclopen **opp, int *newonep, int *retp, int lockit)
 191 {
 192         struct nfsclclient *clp;
 193         struct nfsclowner *owp, *nowp;
 194         struct nfsclopen *op = NULL, *nop = NULL;
 195         struct nfscldeleg *dp;
 196         struct nfsclownerhead *ohp;
 197         u_int8_t own[NFSV4CL_LOCKNAMELEN];
 198         int ret;
 199
 200         if (newonep != NULL)
 201                 *newonep = 0;
 202         if (opp != NULL)
 203                 *opp = NULL;
 204         if (owpp != NULL)
 205                 *owpp = NULL;
 206
 207         /*
 208          * Might need one or both of these, so MALLOC them now, to
 209          * avoid a tsleep() in MALLOC later.
 210          */
 211         MALLOC(nowp, struct nfsclowner *, sizeof (struct nfsclowner),
 212             M_NFSCLOWNER, M_WAITOK);
 213         if (nfhp != NULL)
 214             MALLOC(nop, struct nfsclopen *, sizeof (struct nfsclopen) +
 215                 fhlen - 1, M_NFSCLOPEN, M_WAITOK);
 216         ret = nfscl_getcl(vp, cred, p, &clp);
 217         if (ret != 0) {
 218                 FREE((caddr_t)nowp, M_NFSCLOWNER);
 219                 if (nop != NULL)
 220                         FREE((caddr_t)nop, M_NFSCLOPEN);
 221                 return (ret);
 222         }
 223
 224         /*
 225          * Get the Open iff it already exists.
 226          * If none found, add the new one or return error, depending upon
 227          * "create".
 228          */
 229         nfscl_filllockowner(p, own);
 230         NFSLOCKCLSTATE();
 231         dp = NULL;
 232         /* First check the delegation list */
 233         if (nfhp != NULL && usedeleg) {
 234                 LIST_FOREACH(dp, NFSCLDELEGHASH(clp, nfhp, fhlen), nfsdl_hash) {
 235                         if (dp->nfsdl_fhlen == fhlen &&
 236                             !NFSBCMP(nfhp, dp->nfsdl_fh, fhlen)) {
 237                                 if (!(amode & NFSV4OPEN_ACCESSWRITE) ||
 238                                     (dp->nfsdl_flags & NFSCLDL_WRITE))
 239                                         break;
 240                                 dp = NULL;
 241                                 break;
 242                         }
 243                 }
 244         }
 245
 246         if (dp != NULL)
 247                 ohp = &dp->nfsdl_owner;
 248         else
 249                 ohp = &clp->nfsc_owner;
 250         /* Now, search for an openowner */
 251         LIST_FOREACH(owp, ohp, nfsow_list) {
 252                 if (!NFSBCMP(owp->nfsow_owner, own, NFSV4CL_LOCKNAMELEN))
 253                         break;
 254         }
 255
 256         /*
 257          * Create a new open, as required.
 258          */
 259         nfscl_newopen(clp, dp, &owp, &nowp, &op, &nop, own, nfhp, fhlen,
 260             newonep);
 261
 262         /*
 263          * Serialize modifications to the open owner for multiple threads
 264          * within the same process using a read/write sleep lock.
 265          */
 266         if (lockit)
 267                 nfscl_lockexcl(&owp->nfsow_rwlock, NFSCLSTATEMUTEXPTR);
 268         NFSUNLOCKCLSTATE();
 269         if (nowp != NULL)
 270                 FREE((caddr_t)nowp, M_NFSCLOWNER);
 271         if (nop != NULL)
 272                 FREE((caddr_t)nop, M_NFSCLOPEN);
 273         if (owpp != NULL)
 274                 *owpp = owp;
 275         if (opp != NULL)
 276                 *opp = op;
 277         if (retp != NULL)
 278                 *retp = NFSCLOPEN_OK;
 279
 280         /*
 281          * Now, check the mode on the open and return the appropriate
 282          * value.
 283          */
 284         if (op != NULL && (amode & ~(op->nfso_mode))) {
 285                 op->nfso_mode |= amode;
 286                 if (retp != NULL && dp == NULL)
 287                         *retp = NFSCLOPEN_DOOPEN;
 288         }
 289         return (0);
 290 }
 291
 292 /*
 293  * Create a new open, as required.
 294  */
 295 static void
 296 nfscl_newopen(struct nfsclclient *clp, struct nfscldeleg *dp,
 297     struct nfsclowner **owpp, struct nfsclowner **nowpp, struct nfsclopen **opp,
 298     struct nfsclopen **nopp, u_int8_t *own, u_int8_t *fhp, int fhlen,
 299     int *newonep)
 300 {
 301         struct nfsclowner *owp = *owpp, *nowp;
 302         struct nfsclopen *op, *nop;
 303
 304         if (nowpp != NULL)
 305                 nowp = *nowpp;
 306         else
 307                 nowp = NULL;
 308         if (nopp != NULL)
 309                 nop = *nopp;
 310         else
 311                 nop = NULL;
 312         if (owp == NULL && nowp != NULL) {
 313                 NFSBCOPY(own, nowp->nfsow_owner, NFSV4CL_LOCKNAMELEN);
 314                 LIST_INIT(&nowp->nfsow_open);
 315                 nowp->nfsow_clp = clp;
 316                 nowp->nfsow_seqid = 0;
 317                 nowp->nfsow_defunct = 0;
 318                 nfscl_lockinit(&nowp->nfsow_rwlock);
 319                 if (dp != NULL) {
 320                         newnfsstats.cllocalopenowners++;
 321                         LIST_INSERT_HEAD(&dp->nfsdl_owner, nowp, nfsow_list);
 322                 } else {
 323                         newnfsstats.clopenowners++;
 324                         LIST_INSERT_HEAD(&clp->nfsc_owner, nowp, nfsow_list);
 325                 }
 326                 owp = *owpp = nowp;
 327                 *nowpp = NULL;
 328                 if (newonep != NULL)
 329                         *newonep = 1;
 330         }
 331
 332          /* If an fhp has been specified, create an Open as well. */
 333         if (fhp != NULL) {
 334                 /* and look for the correct open, based upon FH */
 335                 LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 336                         if (op->nfso_fhlen == fhlen &&
 337                             !NFSBCMP(op->nfso_fh, fhp, fhlen))
 338                                 break;
 339                 }
 340                 if (op == NULL && nop != NULL) {
 341                         nop->nfso_own = owp;
 342                         nop->nfso_mode = 0;
 343                         nop->nfso_opencnt = 0;
 344                         nop->nfso_posixlock = 1;
 345                         nop->nfso_fhlen = fhlen;
 346                         NFSBCOPY(fhp, nop->nfso_fh, fhlen);
 347                         LIST_INIT(&nop->nfso_lock);
 348                         nop->nfso_stateid.seqid = 0;
 349                         nop->nfso_stateid.other[0] = 0;
 350                         nop->nfso_stateid.other[1] = 0;
 351                         nop->nfso_stateid.other[2] = 0;
 352                         if (dp != NULL) {
 353                                 TAILQ_REMOVE(&clp->nfsc_deleg, dp, nfsdl_list);
 354                                 TAILQ_INSERT_HEAD(&clp->nfsc_deleg, dp,
 355                                     nfsdl_list);
 356                                 dp->nfsdl_timestamp = NFSD_MONOSEC + 120;
 357                                 newnfsstats.cllocalopens++;
 358                         } else {
 359                                 newnfsstats.clopens++;
 360                         }
 361                         LIST_INSERT_HEAD(&owp->nfsow_open, nop, nfso_list);
 362                         *opp = nop;
 363                         *nopp = NULL;
 364                         if (newonep != NULL)
 365                                 *newonep = 1;
 366                 } else {
 367                         *opp = op;
 368                 }
 369         }
 370 }
 371
 372 /*
 373  * Called to find/add a delegation to a client.
 374  */
 375 APPLESTATIC int
 376 nfscl_deleg(mount_t mp, struct nfsclclient *clp, u_int8_t *nfhp,
 377     int fhlen, struct ucred *cred, NFSPROC_T *p, struct nfscldeleg **dpp)
 378 {
 379         struct nfscldeleg *dp = *dpp, *tdp;
 380
 381         /*
 382          * First, if we have received a Read delegation for a file on a
 383          * read/write file system, just return it, because they aren't
 384          * useful, imho.
 385          */
 386         if (mp != NULL && dp != NULL && !NFSMNT_RDONLY(mp) &&
 387             (dp->nfsdl_flags & NFSCLDL_READ)) {
 388                 (void) nfscl_trydelegreturn(dp, cred, VFSTONFS(mp), p);
 389                 FREE((caddr_t)dp, M_NFSCLDELEG);
 390                 *dpp = NULL;
 391                 return (0);
 392         }
 393
 394         /* Look for the correct deleg, based upon FH */
 395         NFSLOCKCLSTATE();
 396         tdp = nfscl_finddeleg(clp, nfhp, fhlen);
 397         if (tdp == NULL) {
 398                 if (dp == NULL) {
 399                         NFSUNLOCKCLSTATE();
 400                         return (NFSERR_BADSTATEID);
 401                 }
 402                 *dpp = NULL;
 403                 TAILQ_INSERT_HEAD(&clp->nfsc_deleg, dp, nfsdl_list);
 404                 LIST_INSERT_HEAD(NFSCLDELEGHASH(clp, nfhp, fhlen), dp,
 405                     nfsdl_hash);
 406                 dp->nfsdl_timestamp = NFSD_MONOSEC + 120;
 407                 newnfsstats.cldelegates++;
 408                 nfscl_delegcnt++;
 409         } else {
 410                 /*
 411                  * Delegation already exists, what do we do if a new one??
 412                  */
 413                 if (dp != NULL) {
 414                         printf("Deleg already exists!\n");
 415                         FREE((caddr_t)dp, M_NFSCLDELEG);
 416                         *dpp = NULL;
 417                 } else {
 418                         *dpp = tdp;
 419                 }
 420         }
 421         NFSUNLOCKCLSTATE();
 422         return (0);
 423 }
 424
 425 /*
 426  * Find a delegation for this file handle. Return NULL upon failure.
 427  */
 428 static struct nfscldeleg *
 429 nfscl_finddeleg(struct nfsclclient *clp, u_int8_t *fhp, int fhlen)
 430 {
 431         struct nfscldeleg *dp;
 432
 433         LIST_FOREACH(dp, NFSCLDELEGHASH(clp, fhp, fhlen), nfsdl_hash) {
 434             if (dp->nfsdl_fhlen == fhlen &&
 435                 !NFSBCMP(dp->nfsdl_fh, fhp, fhlen))
 436                 break;
 437         }
 438         return (dp);
 439 }
 440
 441 /*
 442  * Get a stateid for an I/O operation. First, look for an open and iff
 443  * found, return either a lockowner stateid or the open stateid.
 444  * If no Open is found, just return error and the special stateid of all zeros.
 445  */
 446 APPLESTATIC int
 447 nfscl_getstateid(vnode_t vp, u_int8_t *nfhp, int fhlen, u_int32_t mode,
 448     struct ucred *cred, NFSPROC_T *p, nfsv4stateid_t *stateidp,
 449     void **lckpp)
 450 {
 451         struct nfsclclient *clp;
 452         struct nfsclowner *owp;
 453         struct nfsclopen *op;
 454         struct nfscllockowner *lp;
 455         struct nfscldeleg *dp;
 456         struct nfsnode *np;
 457         u_int8_t own[NFSV4CL_LOCKNAMELEN];
 458         int error, done;
 459
 460         *lckpp = NULL;
 461         /*
 462          * Initially, just set the special stateid of all zeros.
 463          */
 464         stateidp->seqid = 0;
 465         stateidp->other[0] = 0;
 466         stateidp->other[1] = 0;
 467         stateidp->other[2] = 0;
 468         if (vnode_vtype(vp) != VREG)
 469                 return (EISDIR);
 470         np = VTONFS(vp);
 471         NFSLOCKCLSTATE();
 472         clp = nfscl_findcl(VFSTONFS(vnode_mount(vp)));
 473         if (clp == NULL) {
 474                 NFSUNLOCKCLSTATE();
 475                 return (EACCES);
 476         }
 477
 478         /*
 479          * First, look for a delegation.
 480          */
 481         LIST_FOREACH(dp, NFSCLDELEGHASH(clp, nfhp, fhlen), nfsdl_hash) {
 482                 if (dp->nfsdl_fhlen == fhlen &&
 483                     !NFSBCMP(nfhp, dp->nfsdl_fh, fhlen)) {
 484                         if (!(mode & NFSV4OPEN_ACCESSWRITE) ||
 485                             (dp->nfsdl_flags & NFSCLDL_WRITE)) {
 486                                 stateidp->seqid = dp->nfsdl_stateid.seqid;
 487                                 stateidp->other[0] = dp->nfsdl_stateid.other[0];
 488                                 stateidp->other[1] = dp->nfsdl_stateid.other[1];
 489                                 stateidp->other[2] = dp->nfsdl_stateid.other[2];
 490                                 if (!(np->n_flag & NDELEGRECALL)) {
 491                                         TAILQ_REMOVE(&clp->nfsc_deleg, dp,
 492                                             nfsdl_list);
 493                                         TAILQ_INSERT_HEAD(&clp->nfsc_deleg, dp,
 494                                             nfsdl_list);
 495                                         dp->nfsdl_timestamp = NFSD_MONOSEC +
 496                                             120;
 497                                         dp->nfsdl_rwlock.nfslock_usecnt++;
 498                                         *lckpp = (void *)&dp->nfsdl_rwlock;
 499                                 }
 500                                 NFSUNLOCKCLSTATE();
 501                                 return (0);
 502                         }
 503                         break;
 504                 }
 505         }
 506
 507         if (p != NULL) {
 508                 /*
 509                  * If p != NULL, we want to search the parentage tree
 510                  * for a matching OpenOwner and use that.
 511                  */
 512                 nfscl_filllockowner(p, own);
 513                 error = nfscl_getopen(&clp->nfsc_owner, nfhp, fhlen, NULL, p,
 514                     mode, NULL, &op);
 515                 if (error) {
 516                         NFSUNLOCKCLSTATE();
 517                         return (error);
 518                 }
 519
 520                 /* now look for a lockowner */
 521                 LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
 522                     if (!NFSBCMP(lp->nfsl_owner, own, NFSV4CL_LOCKNAMELEN)) {
 523                         stateidp->seqid = lp->nfsl_stateid.seqid;
 524                         stateidp->other[0] = lp->nfsl_stateid.other[0];
 525                         stateidp->other[1] = lp->nfsl_stateid.other[1];
 526                         stateidp->other[2] = lp->nfsl_stateid.other[2];
 527                         NFSUNLOCKCLSTATE();
 528                         return (0);
 529                     }
 530                 }
 531         } else  {
 532                 /*
 533                  * If p == NULL, it is a read ahead or write behind,
 534                  * so just look for any OpenOwner that will work.
 535                  */
 536                 done = 0;
 537                 owp = LIST_FIRST(&clp->nfsc_owner);
 538                 while (!done && owp != NULL) {
 539                     LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 540                         if (op->nfso_fhlen == fhlen &&
 541                             !NFSBCMP(op->nfso_fh, nfhp, fhlen) &&
 542                             (mode & op->nfso_mode) == mode) {
 543                             done = 1;
 544                             break;
 545                         }
 546                     }
 547                     if (!done)
 548                         owp = LIST_NEXT(owp, nfsow_list);
 549                 }
 550                 if (!done) {
 551                         NFSUNLOCKCLSTATE();
 552                         return (ENOENT);
 553                 }
 554                 /* for read aheads or write behinds, use the open cred */
 555                 newnfs_copycred(&op->nfso_cred, cred);
 556         }
 557
 558         /*
 559          * No lock stateid, so return the open stateid.
 560          */
 561         stateidp->seqid = op->nfso_stateid.seqid;
 562         stateidp->other[0] = op->nfso_stateid.other[0];
 563         stateidp->other[1] = op->nfso_stateid.other[1];
 564         stateidp->other[2] = op->nfso_stateid.other[2];
 565         NFSUNLOCKCLSTATE();
 566         return (0);
 567 }
 568
 569 /*
 570  * Get an existing open. Search up the parentage tree for a match and
 571  * return with the first one found.
 572  */
 573 static int
 574 nfscl_getopen(struct nfsclownerhead *ohp, u_int8_t *nfhp, int fhlen,
 575     u_int8_t *rown, NFSPROC_T *p, u_int32_t mode, struct nfsclowner **owpp,
 576     struct nfsclopen **opp)
 577 {
 578         struct nfsclowner *owp = NULL;
 579         struct nfsclopen *op;
 580         NFSPROC_T *nproc;
 581         u_int8_t own[NFSV4CL_LOCKNAMELEN], *ownp;
 582
 583         nproc = p;
 584         op = NULL;
 585         while (op == NULL && (nproc != NULL || rown != NULL)) {
 586                 if (nproc != NULL) {
 587                         nfscl_filllockowner(nproc, own);
 588                         ownp = own;
 589                 } else {
 590                         ownp = rown;
 591                 }
 592                 /* Search the client list */
 593                 LIST_FOREACH(owp, ohp, nfsow_list) {
 594                         if (!NFSBCMP(owp->nfsow_owner, ownp,
 595                             NFSV4CL_LOCKNAMELEN))
 596                                 break;
 597                 }
 598                 if (owp != NULL) {
 599                         /* and look for the correct open */
 600                         LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 601                                 if (op->nfso_fhlen == fhlen &&
 602                                     !NFSBCMP(op->nfso_fh, nfhp, fhlen)
 603                                     && (op->nfso_mode & mode) == mode) {
 604                                         break;
 605                                 }
 606                         }
 607                 }
 608                 if (rown != NULL)
 609                         break;
 610                 if (op == NULL)
 611                         nproc = nfscl_getparent(nproc);
 612         }
 613         if (op == NULL) {
 614                 return (EBADF);
 615         }
 616         if (owpp)
 617                 *owpp = owp;
 618         *opp = op;
 619         return (0);
 620 }
 621
 622 /*
 623  * Release use of an open owner. Called when open operations are done
 624  * with the open owner.
 625  */
 626 APPLESTATIC void
 627 nfscl_ownerrelease(struct nfsclowner *owp, __unused int error,
 628     __unused int candelete, int unlocked)
 629 {
 630
 631         if (owp == NULL)
 632                 return;
 633         NFSLOCKCLSTATE();
 634         if (!unlocked)
 635                 nfscl_lockunlock(&owp->nfsow_rwlock);
 636         nfscl_clrelease(owp->nfsow_clp);
 637         NFSUNLOCKCLSTATE();
 638 }
 639
 640 /*
 641  * Release use of an open structure under an open owner.
 642  */
 643 APPLESTATIC void
 644 nfscl_openrelease(struct nfsclopen *op, int error, int candelete)
 645 {
 646         struct nfsclclient *clp;
 647         struct nfsclowner *owp;
 648
 649         if (op == NULL)
 650                 return;
 651         NFSLOCKCLSTATE();
 652         owp = op->nfso_own;
 653         nfscl_lockunlock(&owp->nfsow_rwlock);
 654         clp = owp->nfsow_clp;
 655         if (error && candelete && op->nfso_opencnt == 0)
 656                 nfscl_freeopen(op, 0);
 657         nfscl_clrelease(clp);
 658         NFSUNLOCKCLSTATE();
 659 }
 660
 661 /*
 662  * Called to get a clientid structure. It will optionally lock the
 663  * client data structures to do the SetClientId/SetClientId_confirm,
 664  * but will release that lock and return the clientid with a refernce
 665  * count on it.
 666  * If the p argument is NULL, it will not do the SetClientId/Confirm
 667  * and the cred argument is not used, so it can be NULL too.
 668  * It always clpp with a reference count on it, unless returning an error.
 669  */
 670 APPLESTATIC int
 671 nfscl_getcl(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
 672     struct nfsclclient **clpp)
 673 {
 674         struct nfsclclient *clp;
 675         struct nfsclclient *newclp;
 676         struct nfscllockowner *lp, *nlp;
 677         struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
 678         int igotlock = 0, error, trystalecnt, clidinusedelay, i;
 679         u_int16_t idlen;
 680
 681         idlen = strlen(hostuuid);
 682         if (idlen > 0)
 683                 idlen += sizeof (u_int64_t);
 684         else
 685                 idlen += sizeof (u_int64_t) + 16;       /* 16 random bytes */
 686         MALLOC(newclp, struct nfsclclient *, sizeof (struct nfsclclient) +
 687             idlen - 1, M_NFSCLCLIENT, M_WAITOK);
 688         NFSLOCKCLSTATE();
 689         clp = nmp->nm_clp;
 690         if (clp == NULL) {
 691                 clp = newclp;
 692                 NFSBZERO((caddr_t)clp, sizeof(struct nfsclclient) + idlen - 1);
 693                 clp->nfsc_idlen = idlen;
 694                 LIST_INIT(&clp->nfsc_owner);
 695                 TAILQ_INIT(&clp->nfsc_deleg);
 696                 for (i = 0; i < NFSCLDELEGHASHSIZE; i++)
 697                         LIST_INIT(&clp->nfsc_deleghash[i]);
 698                 LIST_INIT(&clp->nfsc_defunctlockowner);
 699                 clp->nfsc_flags = NFSCLFLAGS_INITED;
 700                 clp->nfsc_clientidrev = 1;
 701                 clp->nfsc_cbident = nfscl_nextcbident();
 702                 nfscl_fillclid(nmp->nm_clval, hostuuid, clp->nfsc_id,
 703                     clp->nfsc_idlen);
 704                 LIST_INSERT_HEAD(&nfsclhead, clp, nfsc_list);
 705                 nmp->nm_clp = clp;
 706                 clp->nfsc_nmp = nmp;
 707                 NFSUNLOCKCLSTATE();
 708                 nfscl_start_renewthread(clp);
 709         } else {
 710                 NFSUNLOCKCLSTATE();
 711                 FREE((caddr_t)newclp, M_NFSCLCLIENT);
 712         }
 713         NFSLOCKCLSTATE();
 714         while ((clp->nfsc_flags & NFSCLFLAGS_HASCLIENTID) == 0 && !igotlock)
 715                 igotlock = nfsv4_lock(&clp->nfsc_lock, 1, NULL,
 716                     NFSCLSTATEMUTEXPTR);
 717         if (!igotlock)
 718                 nfsv4_getref(&clp->nfsc_lock, NULL, NFSCLSTATEMUTEXPTR);
 719         NFSUNLOCKCLSTATE();
 720
 721         /*
 722          * If it needs a clientid, do the setclientid now.
 723          */
 724         if ((clp->nfsc_flags & NFSCLFLAGS_HASCLIENTID) == 0) {
 725                 if (!igotlock)
 726                         panic("nfscl_clget");
 727                 if (p == NULL) {
 728                         NFSLOCKCLSTATE();
 729                         nfsv4_unlock(&clp->nfsc_lock, 0);
 730                         NFSUNLOCKCLSTATE();
 731                         return (EACCES);
 732                 }
 733                 /* get rid of defunct lockowners */
 734                 LIST_FOREACH_SAFE(lp, &clp->nfsc_defunctlockowner, nfsl_list,
 735                     nlp) {
 736                         nfscl_freelockowner(lp, 0);
 737                 }
 738                 /*
 739                  * If RFC3530 Sec. 14.2.33 is taken literally,
 740                  * NFSERR_CLIDINUSE will be returned persistently for the
 741                  * case where a new mount of the same file system is using
 742                  * a different principal. In practice, NFSERR_CLIDINUSE is
 743                  * only returned when there is outstanding unexpired state
 744                  * on the clientid. As such, try for twice the lease
 745                  * interval, if we know what that is. Otherwise, make a
 746                  * wild ass guess.
 747                  * The case of returning NFSERR_STALECLIENTID is far less
 748                  * likely, but might occur if there is a significant delay
 749                  * between doing the SetClientID and SetClientIDConfirm Ops,
 750                  * such that the server throws away the clientid before
 751                  * receiving the SetClientIDConfirm.
 752                  */
 753                 if (clp->nfsc_renew > 0)
 754                         clidinusedelay = NFSCL_LEASE(clp->nfsc_renew) * 2;
 755                 else
 756                         clidinusedelay = 120;
 757                 trystalecnt = 3;
 758                 do {
 759                         error = nfsrpc_setclient(VFSTONFS(vnode_mount(vp)), clp,
 760                             cred, p);
 761                         if (error == NFSERR_STALECLIENTID ||
 762                             error == NFSERR_STALEDONTRECOVER ||
 763                             error == NFSERR_CLIDINUSE) {
 764                                 (void) nfs_catnap(PZERO, "nfs_setcl");
 765                         }
 766                 } while (((error == NFSERR_STALECLIENTID ||
 767                      error == NFSERR_STALEDONTRECOVER) && --trystalecnt > 0) ||
 768                     (error == NFSERR_CLIDINUSE && --clidinusedelay > 0));
 769                 if (error) {
 770                         NFSLOCKCLSTATE();
 771                         nfsv4_unlock(&clp->nfsc_lock, 0);
 772                         NFSUNLOCKCLSTATE();
 773                         return (error);
 774                 }
 775                 clp->nfsc_flags |= NFSCLFLAGS_HASCLIENTID;
 776         }
 777         if (igotlock) {
 778                 NFSLOCKCLSTATE();
 779                 nfsv4_unlock(&clp->nfsc_lock, 1);
 780                 NFSUNLOCKCLSTATE();
 781         }
 782
 783         *clpp = clp;
 784         return (0);
 785 }
 786
 787 /*
 788  * Get a reference to a clientid and return it, if valid.
 789  */
 790 APPLESTATIC struct nfsclclient *
 791 nfscl_findcl(struct nfsmount *nmp)
 792 {
 793         struct nfsclclient *clp;
 794
 795         clp = nmp->nm_clp;
 796         if (clp == NULL || !(clp->nfsc_flags & NFSCLFLAGS_HASCLIENTID))
 797                 return (NULL);
 798         return (clp);
 799 }
 800
 801 /*
 802  * Release the clientid structure. It may be locked or reference counted.
 803  */
 804 static void
 805 nfscl_clrelease(struct nfsclclient *clp)
 806 {
 807
 808         if (clp->nfsc_lock.nfslock_lock & NFSV4LOCK_LOCK)
 809                 nfsv4_unlock(&clp->nfsc_lock, 0);
 810         else
 811                 nfsv4_relref(&clp->nfsc_lock);
 812 }
 813
 814 /*
 815  * External call for nfscl_clrelease.
 816  */
 817 APPLESTATIC void
 818 nfscl_clientrelease(struct nfsclclient *clp)
 819 {
 820
 821         NFSLOCKCLSTATE();
 822         if (clp->nfsc_lock.nfslock_lock & NFSV4LOCK_LOCK)
 823                 nfsv4_unlock(&clp->nfsc_lock, 0);
 824         else
 825                 nfsv4_relref(&clp->nfsc_lock);
 826         NFSUNLOCKCLSTATE();
 827 }
 828
 829 /*
 830  * Called when wanting to lock a byte region.
 831  */
 832 APPLESTATIC int
 833 nfscl_getbytelock(vnode_t vp, u_int64_t off, u_int64_t len,
 834     short type, struct ucred *cred, NFSPROC_T *p, struct nfsclclient *rclp,
 835     int recovery, u_int8_t *rownp, u_int8_t *ropenownp,
 836     struct nfscllockowner **lpp, int *newonep, int *donelocallyp)
 837 {
 838         struct nfscllockowner *lp;
 839         struct nfsclopen *op;
 840         struct nfsclclient *clp;
 841         struct nfscllockowner *nlp;
 842         struct nfscllock *nlop, *otherlop;
 843         struct nfscldeleg *dp = NULL, *ldp = NULL;
 844         struct nfscllockownerhead *lhp = NULL;
 845         struct nfsnode *np;
 846         u_int8_t own[NFSV4CL_LOCKNAMELEN], *ownp;
 847         int error = 0, ret, donelocally = 0;
 848         u_int32_t mode;
 849
 850         if (type == F_WRLCK)
 851                 mode = NFSV4OPEN_ACCESSWRITE;
 852         else
 853                 mode = NFSV4OPEN_ACCESSREAD;
 854         np = VTONFS(vp);
 855         *lpp = NULL;
 856         *newonep = 0;
 857         *donelocallyp = 0;
 858
 859         /*
 860          * Might need these, so MALLOC them now, to
 861          * avoid a tsleep() in MALLOC later.
 862          */
 863         MALLOC(nlp, struct nfscllockowner *,
 864             sizeof (struct nfscllockowner), M_NFSCLLOCKOWNER, M_WAITOK);
 865         MALLOC(otherlop, struct nfscllock *,
 866             sizeof (struct nfscllock), M_NFSCLLOCK, M_WAITOK);
 867         MALLOC(nlop, struct nfscllock *,
 868             sizeof (struct nfscllock), M_NFSCLLOCK, M_WAITOK);
 869         nlop->nfslo_type = type;
 870         nlop->nfslo_first = off;
 871         if (len == NFS64BITSSET) {
 872                 nlop->nfslo_end = NFS64BITSSET;
 873         } else {
 874                 nlop->nfslo_end = off + len;
 875                 if (nlop->nfslo_end <= nlop->nfslo_first)
 876                         error = NFSERR_INVAL;
 877         }
 878
 879         if (!error) {
 880                 if (recovery)
 881                         clp = rclp;
 882                 else
 883                         error = nfscl_getcl(vp, cred, p, &clp);
 884         }
 885         if (error) {
 886                 FREE((caddr_t)nlp, M_NFSCLLOCKOWNER);
 887                 FREE((caddr_t)otherlop, M_NFSCLLOCK);
 888                 FREE((caddr_t)nlop, M_NFSCLLOCK);
 889                 return (error);
 890         }
 891
 892         op = NULL;
 893         if (recovery) {
 894                 ownp = rownp;
 895         } else {
 896                 nfscl_filllockowner(p, own);
 897                 ownp = own;
 898         }
 899         if (!recovery) {
 900                 NFSLOCKCLSTATE();
 901                 /*
 902                  * First, search for a delegation. If one exists for this file,
 903                  * the lock can be done locally against it, so long as there
 904                  * isn't a local lock conflict.
 905                  */
 906                 ldp = dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
 907                     np->n_fhp->nfh_len);
 908                 /* Just sanity check for correct type of delegation */
 909                 if (dp != NULL && ((dp->nfsdl_flags & NFSCLDL_RECALL) ||
 910                     (type == F_WRLCK && !(dp->nfsdl_flags & NFSCLDL_WRITE))))
 911                         dp = NULL;
 912         }
 913         if (dp != NULL) {
 914                 /* Now, find the associated open to get the correct openowner */
 915                 ret = nfscl_getopen(&dp->nfsdl_owner, np->n_fhp->nfh_fh,
 916                     np->n_fhp->nfh_len, NULL, p, mode, NULL, &op);
 917                 if (ret)
 918                         ret = nfscl_getopen(&clp->nfsc_owner,
 919                             np->n_fhp->nfh_fh, np->n_fhp->nfh_len, NULL, p,
 920                             mode, NULL, &op);
 921                 if (!ret) {
 922                         lhp = &dp->nfsdl_lock;
 923                         TAILQ_REMOVE(&clp->nfsc_deleg, dp, nfsdl_list);
 924                         TAILQ_INSERT_HEAD(&clp->nfsc_deleg, dp, nfsdl_list);
 925                         dp->nfsdl_timestamp = NFSD_MONOSEC + 120;
 926                         donelocally = 1;
 927                 } else {
 928                         dp = NULL;
 929                 }
 930         }
 931         if (!donelocally) {
 932                 /*
 933                  * Get the related Open.
 934                  */
 935                 if (recovery)
 936                         error = nfscl_getopen(&clp->nfsc_owner,
 937                             np->n_fhp->nfh_fh, np->n_fhp->nfh_len, ropenownp,
 938                             NULL, mode, NULL, &op);
 939                 else
 940                         error = nfscl_getopen(&clp->nfsc_owner,
 941                             np->n_fhp->nfh_fh, np->n_fhp->nfh_len, NULL, p,
 942                             mode, NULL, &op);
 943                 if (!error)
 944                         lhp = &op->nfso_lock;
 945         }
 946         if (!error && !recovery)
 947                 error = nfscl_localconflict(clp, nlop, ownp, ldp, NULL);
 948         if (error) {
 949                 if (!recovery) {
 950                         nfscl_clrelease(clp);
 951                         NFSUNLOCKCLSTATE();
 952                 }
 953                 FREE((caddr_t)nlp, M_NFSCLLOCKOWNER);
 954                 FREE((caddr_t)otherlop, M_NFSCLLOCK);
 955                 FREE((caddr_t)nlop, M_NFSCLLOCK);
 956                 return (error);
 957         }
 958
 959         /*
 960          * Ok, see if a lockowner exists and create one, as required.
 961          */
 962         LIST_FOREACH(lp, lhp, nfsl_list) {
 963                 if (!NFSBCMP(lp->nfsl_owner, ownp, NFSV4CL_LOCKNAMELEN))
 964                         break;
 965         }
 966         if (lp == NULL) {
 967                 NFSBCOPY(ownp, nlp->nfsl_owner, NFSV4CL_LOCKNAMELEN);
 968                 if (recovery)
 969                         NFSBCOPY(ropenownp, nlp->nfsl_openowner,
 970                             NFSV4CL_LOCKNAMELEN);
 971                 else
 972                         NFSBCOPY(op->nfso_own->nfsow_owner, nlp->nfsl_openowner,
 973                             NFSV4CL_LOCKNAMELEN);
 974                 nlp->nfsl_seqid = 0;
 975                 nlp->nfsl_defunct = 0;
 976                 nlp->nfsl_inprog = NULL;
 977                 nfscl_lockinit(&nlp->nfsl_rwlock);
 978                 LIST_INIT(&nlp->nfsl_lock);
 979                 if (donelocally) {
 980                         nlp->nfsl_open = NULL;
 981                         newnfsstats.cllocallockowners++;
 982                 } else {
 983                         nlp->nfsl_open = op;
 984                         newnfsstats.cllockowners++;
 985                 }
 986                 LIST_INSERT_HEAD(lhp, nlp, nfsl_list);
 987                 lp = nlp;
 988                 nlp = NULL;
 989                 *newonep = 1;
 990         }
 991
 992         /*
 993          * Now, update the byte ranges for locks.
 994          */
 995         ret = nfscl_updatelock(lp, &nlop, &otherlop, donelocally);
 996         if (!ret)
 997                 donelocally = 1;
 998         if (donelocally) {
 999                 *donelocallyp = 1;
1000                 if (!recovery)
1001                         nfscl_clrelease(clp);
1002         } else {
1003                 /*
1004                  * Serial modifications on the lock owner for multiple threads
1005                  * for the same process using a read/write lock.
1006                  */
1007                 if (!recovery)
1008                         nfscl_lockexcl(&lp->nfsl_rwlock, NFSCLSTATEMUTEXPTR);
1009         }
1010         if (!recovery)
1011                 NFSUNLOCKCLSTATE();
1012
1013         if (nlp)
1014                 FREE((caddr_t)nlp, M_NFSCLLOCKOWNER);
1015         if (nlop)
1016                 FREE((caddr_t)nlop, M_NFSCLLOCK);
1017         if (otherlop)
1018                 FREE((caddr_t)otherlop, M_NFSCLLOCK);
1019
1020         *lpp = lp;
1021         return (0);
1022 }
1023
1024 /*
1025  * Called to unlock a byte range, for LockU.
1026  */
1027 APPLESTATIC int
1028 nfscl_relbytelock(vnode_t vp, u_int64_t off, u_int64_t len,
1029     __unused struct ucred *cred, NFSPROC_T *p, int callcnt,
1030     struct nfsclclient *clp, struct nfscllockowner **lpp, int *dorpcp)
1031 {
1032         struct nfscllockowner *lp;
1033         struct nfsclowner *owp;
1034         struct nfsclopen *op;
1035         struct nfscllock *nlop, *other_lop = NULL;
1036         struct nfscldeleg *dp;
1037         struct nfsnode *np;
1038         u_int8_t own[NFSV4CL_LOCKNAMELEN];
1039         int ret = 0, fnd, error;
1040
1041         np = VTONFS(vp);
1042         *lpp = NULL;
1043         *dorpcp = 0;
1044
1045         /*
1046          * Might need these, so MALLOC them now, to
1047          * avoid a tsleep() in MALLOC later.
1048          */
1049         MALLOC(nlop, struct nfscllock *,
1050             sizeof (struct nfscllock), M_NFSCLLOCK, M_WAITOK);
1051         nlop->nfslo_type = F_UNLCK;
1052         nlop->nfslo_first = off;
1053         if (len == NFS64BITSSET) {
1054                 nlop->nfslo_end = NFS64BITSSET;
1055         } else {
1056                 nlop->nfslo_end = off + len;
1057                 if (nlop->nfslo_end <= nlop->nfslo_first) {
1058                         FREE((caddr_t)nlop, M_NFSCLLOCK);
1059                         return (NFSERR_INVAL);
1060                 }
1061         }
1062         if (callcnt == 0) {
1063                 MALLOC(other_lop, struct nfscllock *,
1064                     sizeof (struct nfscllock), M_NFSCLLOCK, M_WAITOK);
1065                 *other_lop = *nlop;
1066         }
1067         nfscl_filllockowner(p, own);
1068         dp = NULL;
1069         NFSLOCKCLSTATE();
1070         if (callcnt == 0)
1071                 dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
1072                     np->n_fhp->nfh_len);
1073
1074         /* Search for a local conflict. */
1075         error = nfscl_localconflict(clp, nlop, own, dp, NULL);
1076         if (error) {
1077                 NFSUNLOCKCLSTATE();
1078                 FREE((caddr_t)nlop, M_NFSCLLOCK);
1079                 if (other_lop != NULL)
1080                         FREE((caddr_t)other_lop, M_NFSCLLOCK);
1081                 return (error);
1082         }
1083
1084         /*
1085          * First, unlock any local regions on a delegation.
1086          */
1087         if (dp != NULL) {
1088                 /* Look for this lockowner. */
1089                 LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
1090                         if (!NFSBCMP(lp->nfsl_owner, own,
1091                             NFSV4CL_LOCKNAMELEN))
1092                                 break;
1093                 }
1094                 if (lp != NULL)
1095                         /* Use other_lop, so nlop is still available */
1096                         (void)nfscl_updatelock(lp, &other_lop, NULL, 1);
1097         }
1098
1099         /*
1100          * Now, find a matching open/lockowner that hasn't already been done,
1101          * as marked by nfsl_inprog.
1102          */
1103         lp = NULL;
1104         fnd = 0;
1105         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
1106             LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
1107                 if (op->nfso_fhlen == np->n_fhp->nfh_len &&
1108                     !NFSBCMP(op->nfso_fh, np->n_fhp->nfh_fh, op->nfso_fhlen)) {
1109                     LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
1110                         if (lp->nfsl_inprog == NULL &&
1111                             !NFSBCMP(lp->nfsl_owner, own,
1112                              NFSV4CL_LOCKNAMELEN)) {
1113                                 fnd = 1;
1114                                 break;
1115                         }
1116                     }
1117                     if (fnd)
1118                         break;
1119                 }
1120             }
1121             if (fnd)
1122                 break;
1123         }
1124
1125         if (lp != NULL) {
1126                 ret = nfscl_updatelock(lp, &nlop, NULL, 0);
1127                 if (ret)
1128                         *dorpcp = 1;
1129                 /*
1130                  * Serial modifications on the lock owner for multiple
1131                  * threads for the same process using a read/write lock.
1132                  */
1133                 lp->nfsl_inprog = p;
1134                 nfscl_lockexcl(&lp->nfsl_rwlock, NFSCLSTATEMUTEXPTR);
1135                 *lpp = lp;
1136         }
1137         NFSUNLOCKCLSTATE();
1138         if (nlop)
1139                 FREE((caddr_t)nlop, M_NFSCLLOCK);
1140         if (other_lop)
1141                 FREE((caddr_t)other_lop, M_NFSCLLOCK);
1142         return (0);
1143 }
1144
1145 /*
1146  * Release all lockowners marked in progess for this process and file.
1147  */
1148 APPLESTATIC void
1149 nfscl_releasealllocks(struct nfsclclient *clp, vnode_t vp, NFSPROC_T *p)
1150 {
1151         struct nfsclowner *owp;
1152         struct nfsclopen *op;
1153         struct nfscllockowner *lp;
1154         struct nfsnode *np;
1155         u_int8_t own[NFSV4CL_LOCKNAMELEN];
1156
1157         np = VTONFS(vp);
1158         nfscl_filllockowner(p, own);
1159         NFSLOCKCLSTATE();
1160         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
1161             LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
1162                 if (op->nfso_fhlen == np->n_fhp->nfh_len &&
1163                     !NFSBCMP(op->nfso_fh, np->n_fhp->nfh_fh, op->nfso_fhlen)) {
1164                     LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
1165                         if (lp->nfsl_inprog == p &&
1166                             !NFSBCMP(lp->nfsl_owner, own,
1167                             NFSV4CL_LOCKNAMELEN)) {
1168                             lp->nfsl_inprog = NULL;
1169                             nfscl_lockunlock(&lp->nfsl_rwlock);
1170                         }
1171                     }
1172                 }
1173             }
1174         }
1175         nfscl_clrelease(clp);
1176         NFSUNLOCKCLSTATE();
1177 }
1178
1179 /*
1180  * Called to find out if any bytes within the byte range specified are
1181  * write locked by the calling process. Used to determine if flushing
1182  * is required before a LockU.
1183  * If in doubt, return 1, so the flush will occur.
1184  */
1185 APPLESTATIC int
1186 nfscl_checkwritelocked(vnode_t vp, struct flock *fl,
1187     struct ucred *cred, NFSPROC_T *p)
1188 {
1189         struct nfsclowner *owp;
1190         struct nfscllockowner *lp;
1191         struct nfsclopen *op;
1192         struct nfsclclient *clp;
1193         struct nfscllock *lop;
1194         struct nfscldeleg *dp;
1195         struct nfsnode *np;
1196         u_int64_t off, end;
1197         u_int8_t own[NFSV4CL_LOCKNAMELEN];
1198         int error = 0;
1199
1200         np = VTONFS(vp);
1201         switch (fl->l_whence) {
1202         case SEEK_SET:
1203         case SEEK_CUR:
1204                 /*
1205                  * Caller is responsible for adding any necessary offset
1206                  * when SEEK_CUR is used.
1207                  */
1208                 off = fl->l_start;
1209                 break;
1210         case SEEK_END:
1211                 off = np->n_size + fl->l_start;
1212                 break;
1213         default:
1214                 return (1);
1215         };
1216         if (fl->l_len != 0) {
1217                 end = off + fl->l_len;
1218                 if (end < off)
1219                         return (1);
1220         } else {
1221                 end = NFS64BITSSET;
1222         }
1223
1224         error = nfscl_getcl(vp, cred, p, &clp);
1225         if (error)
1226                 return (1);
1227         nfscl_filllockowner(p, own);
1228         NFSLOCKCLSTATE();
1229
1230         /*
1231          * First check the delegation locks.
1232          */
1233         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
1234         if (dp != NULL) {
1235                 LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
1236                         if (!NFSBCMP(lp->nfsl_owner, own,
1237                             NFSV4CL_LOCKNAMELEN))
1238                                 break;
1239                 }
1240                 if (lp != NULL) {
1241                         LIST_FOREACH(lop, &lp->nfsl_lock, nfslo_list) {
1242                                 if (lop->nfslo_first >= end)
1243                                         break;
1244                                 if (lop->nfslo_end <= off)
1245                                         continue;
1246                                 if (lop->nfslo_type == F_WRLCK) {
1247                                         nfscl_clrelease(clp);
1248                                         NFSUNLOCKCLSTATE();
1249                                         return (1);
1250                                 }
1251                         }
1252                 }
1253         }
1254
1255         /*
1256          * Now, check state against the server.
1257          */
1258         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
1259             LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
1260                 if (op->nfso_fhlen == np->n_fhp->nfh_len &&
1261                     !NFSBCMP(op->nfso_fh, np->n_fhp->nfh_fh, op->nfso_fhlen)) {
1262                     LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
1263                         if (!NFSBCMP(lp->nfsl_owner, own,
1264                             NFSV4CL_LOCKNAMELEN))
1265                             break;
1266                     }
1267                     if (lp != NULL) {
1268                         LIST_FOREACH(lop, &lp->nfsl_lock, nfslo_list) {
1269                             if (lop->nfslo_first >= end)
1270                                 break;
1271                             if (lop->nfslo_end <= off)
1272                                 continue;
1273                             if (lop->nfslo_type == F_WRLCK) {
1274                                 nfscl_clrelease(clp);
1275                                 NFSUNLOCKCLSTATE();
1276                                 return (1);
1277                             }
1278                         }
1279                     }
1280                 }
1281             }
1282         }
1283         nfscl_clrelease(clp);
1284         NFSUNLOCKCLSTATE();
1285         return (0);
1286 }
1287
1288 /*
1289  * Release a byte range lock owner structure.
1290  */
1291 APPLESTATIC void
1292 nfscl_lockrelease(struct nfscllockowner *lp, int error, int candelete)
1293 {
1294         struct nfsclclient *clp;
1295
1296         if (lp == NULL)
1297                 return;
1298         NFSLOCKCLSTATE();
1299         clp = lp->nfsl_open->nfso_own->nfsow_clp;
1300         if (error != 0 && candelete &&
1301             (lp->nfsl_rwlock.nfslock_lock & NFSV4LOCK_WANTED) == 0)
1302                 nfscl_freelockowner(lp, 0);
1303         else
1304                 nfscl_lockunlock(&lp->nfsl_rwlock);
1305         nfscl_clrelease(clp);
1306         NFSUNLOCKCLSTATE();
1307 }
1308
1309 /*
1310  * Free up an open structure and any associated byte range lock structures.
1311  */
1312 APPLESTATIC void
1313 nfscl_freeopen(struct nfsclopen *op, int local)
1314 {
1315
1316         LIST_REMOVE(op, nfso_list);
1317         nfscl_freealllocks(&op->nfso_lock, local);
1318         FREE((caddr_t)op, M_NFSCLOPEN);
1319         if (local)
1320                 newnfsstats.cllocalopens--;
1321         else
1322                 newnfsstats.clopens--;
1323 }
1324
1325 /*
1326  * Free up all lock owners and associated locks.
1327  */
1328 static void
1329 nfscl_freealllocks(struct nfscllockownerhead *lhp, int local)
1330 {
1331         struct nfscllockowner *lp, *nlp;
1332
1333         LIST_FOREACH_SAFE(lp, lhp, nfsl_list, nlp) {
1334                 if ((lp->nfsl_rwlock.nfslock_lock & NFSV4LOCK_WANTED))
1335                         panic("nfscllckw");
1336                 nfscl_freelockowner(lp, local);
1337         }
1338 }
1339
1340 /*
1341  * Called for an Open when NFSERR_EXPIRED is received from the server.
1342  * If there are no byte range locks nor a Share Deny lost, try to do a
1343  * fresh Open. Otherwise, free the open.
1344  */
1345 static int
1346 nfscl_expireopen(struct nfsclclient *clp, struct nfsclopen *op,
1347     struct nfsmount *nmp, struct ucred *cred, NFSPROC_T *p)
1348 {
1349         struct nfscllockowner *lp;
1350         struct nfscldeleg *dp;
1351         int mustdelete = 0, error;
1352
1353         /*
1354          * Look for any byte range lock(s).
1355          */
1356         LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
1357                 if (!LIST_EMPTY(&lp->nfsl_lock)) {
1358                         mustdelete = 1;
1359                         break;
1360                 }
1361         }
1362
1363         /*
1364          * If no byte range lock(s) nor a Share deny, try to re-open.
1365          */
1366         if (!mustdelete && (op->nfso_mode & NFSLCK_DENYBITS) == 0) {
1367                 newnfs_copycred(&op->nfso_cred, cred);
1368                 dp = NULL;
1369                 error = nfsrpc_reopen(nmp, op->nfso_fh,
1370                     op->nfso_fhlen, op->nfso_mode, op, &dp, cred, p);
1371                 if (error) {
1372                         mustdelete = 1;
1373                         if (dp != NULL) {
1374                                 FREE((caddr_t)dp, M_NFSCLDELEG);
1375                                 dp = NULL;
1376                         }
1377                 }
1378                 if (dp != NULL)
1379                         nfscl_deleg(nmp->nm_mountp, clp, op->nfso_fh,
1380                             op->nfso_fhlen, cred, p, &dp);
1381         }
1382
1383         /*
1384          * If a byte range lock or Share deny or couldn't re-open, free it.
1385          */
1386         if (mustdelete)
1387                 nfscl_freeopen(op, 0);
1388         return (mustdelete);
1389 }
1390
1391 /*
1392  * Free up an open owner structure.
1393  */
1394 static void
1395 nfscl_freeopenowner(struct nfsclowner *owp, int local)
1396 {
1397
1398         LIST_REMOVE(owp, nfsow_list);
1399         FREE((caddr_t)owp, M_NFSCLOWNER);
1400         if (local)
1401                 newnfsstats.cllocalopenowners--;
1402         else
1403                 newnfsstats.clopenowners--;
1404 }
1405
1406 /*
1407  * Free up a byte range lock owner structure.
1408  */
1409 static void
1410 nfscl_freelockowner(struct nfscllockowner *lp, int local)
1411 {
1412         struct nfscllock *lop, *nlop;
1413
1414         LIST_REMOVE(lp, nfsl_list);
1415         LIST_FOREACH_SAFE(lop, &lp->nfsl_lock, nfslo_list, nlop) {
1416                 nfscl_freelock(lop, local);
1417         }
1418         FREE((caddr_t)lp, M_NFSCLLOCKOWNER);
1419         if (local)
1420                 newnfsstats.cllocallockowners--;
1421         else
1422                 newnfsstats.cllockowners--;
1423 }
1424
1425 /*
1426  * Free up a byte range lock structure.
1427  */
1428 APPLESTATIC void
1429 nfscl_freelock(struct nfscllock *lop, int local)
1430 {
1431
1432         LIST_REMOVE(lop, nfslo_list);
1433         FREE((caddr_t)lop, M_NFSCLLOCK);
1434         if (local)
1435                 newnfsstats.cllocallocks--;
1436         else
1437                 newnfsstats.cllocks--;
1438 }
1439
1440 /*
1441  * Clean out the state related to a delegation.
1442  */
1443 static void
1444 nfscl_cleandeleg(struct nfscldeleg *dp)
1445 {
1446         struct nfsclowner *owp, *nowp;
1447         struct nfsclopen *op;
1448
1449         LIST_FOREACH_SAFE(owp, &dp->nfsdl_owner, nfsow_list, nowp) {
1450                 op = LIST_FIRST(&owp->nfsow_open);
1451                 if (op != NULL) {
1452                         if (LIST_NEXT(op, nfso_list) != NULL)
1453                                 panic("nfscleandel");
1454                         nfscl_freeopen(op, 1);
1455                 }
1456                 nfscl_freeopenowner(owp, 1);
1457         }
1458         nfscl_freealllocks(&dp->nfsdl_lock, 1);
1459 }
1460
1461 /*
1462  * Free a delegation.
1463  */
1464 static void
1465 nfscl_freedeleg(struct nfscldeleghead *hdp, struct nfscldeleg *dp)
1466 {
1467
1468         TAILQ_REMOVE(hdp, dp, nfsdl_list);
1469         LIST_REMOVE(dp, nfsdl_hash);
1470         FREE((caddr_t)dp, M_NFSCLDELEG);
1471         newnfsstats.cldelegates--;
1472         nfscl_delegcnt--;
1473 }
1474
1475 /*
1476  * Free up all state related to this client structure.
1477  */
1478 static void
1479 nfscl_cleanclient(struct nfsclclient *clp)
1480 {
1481         struct nfsclowner *owp, *nowp;
1482         struct nfsclopen *op, *nop;
1483         struct nfscllockowner *lp, *nlp;
1484
1485
1486         /* get rid of defunct lockowners */
1487         LIST_FOREACH_SAFE(lp, &clp->nfsc_defunctlockowner, nfsl_list, nlp) {
1488                 nfscl_freelockowner(lp, 0);
1489         }
1490
1491         /* Now, all the OpenOwners, etc. */
1492         LIST_FOREACH_SAFE(owp, &clp->nfsc_owner, nfsow_list, nowp) {
1493                 LIST_FOREACH_SAFE(op, &owp->nfsow_open, nfso_list, nop) {
1494                         nfscl_freeopen(op, 0);
1495                 }
1496                 nfscl_freeopenowner(owp, 0);
1497         }
1498 }
1499
1500 /*
1501  * Called when an NFSERR_EXPIRED is received from the server.
1502  */
1503 static void
1504 nfscl_expireclient(struct nfsclclient *clp, struct nfsmount *nmp,
1505     struct ucred *cred, NFSPROC_T *p)
1506 {
1507         struct nfsclowner *owp, *nowp, *towp;
1508         struct nfsclopen *op, *nop, *top;
1509         struct nfscldeleg *dp, *ndp;
1510         int ret, printed = 0;
1511
1512         /*
1513          * First, merge locally issued Opens into the list for the server.
1514          */
1515         dp = TAILQ_FIRST(&clp->nfsc_deleg);
1516         while (dp != NULL) {
1517             ndp = TAILQ_NEXT(dp, nfsdl_list);
1518             owp = LIST_FIRST(&dp->nfsdl_owner);
1519             while (owp != NULL) {
1520                 nowp = LIST_NEXT(owp, nfsow_list);
1521                 op = LIST_FIRST(&owp->nfsow_open);
1522                 if (op != NULL) {
1523                     if (LIST_NEXT(op, nfso_list) != NULL)
1524                         panic("nfsclexp");
1525                     LIST_FOREACH(towp, &clp->nfsc_owner, nfsow_list) {
1526                         if (!NFSBCMP(towp->nfsow_owner, owp->nfsow_owner,
1527                             NFSV4CL_LOCKNAMELEN))
1528                             break;
1529                     }
1530                     if (towp != NULL) {
1531                         /* Merge opens in */
1532                         LIST_FOREACH(top, &towp->nfsow_open, nfso_list) {
1533                             if (top->nfso_fhlen == op->nfso_fhlen &&
1534                                 !NFSBCMP(top->nfso_fh, op->nfso_fh,
1535                                  op->nfso_fhlen)) {
1536                                 top->nfso_mode |= op->nfso_mode;
1537                                 top->nfso_opencnt += op->nfso_opencnt;
1538                                 break;
1539                             }
1540                         }
1541                         if (top == NULL) {
1542                             /* Just add the open to the owner list */
1543                             LIST_REMOVE(op, nfso_list);
1544                             op->nfso_own = towp;
1545                             LIST_INSERT_HEAD(&towp->nfsow_open, op, nfso_list);
1546                             newnfsstats.cllocalopens--;
1547                             newnfsstats.clopens++;
1548                         }
1549                     } else {
1550                         /* Just add the openowner to the client list */
1551                         LIST_REMOVE(owp, nfsow_list);
1552                         owp->nfsow_clp = clp;
1553                         LIST_INSERT_HEAD(&clp->nfsc_owner, owp, nfsow_list);
1554                         newnfsstats.cllocalopenowners--;
1555                         newnfsstats.clopenowners++;
1556                         newnfsstats.cllocalopens--;
1557                         newnfsstats.clopens++;
1558                     }
1559                 }
1560                 owp = nowp;
1561             }
1562             if (!printed && !LIST_EMPTY(&dp->nfsdl_lock)) {
1563                 printed = 1;
1564                 printf("nfsv4 expired locks lost\n");
1565             }
1566             nfscl_cleandeleg(dp);
1567             nfscl_freedeleg(&clp->nfsc_deleg, dp);
1568             dp = ndp;
1569         }
1570         if (!TAILQ_EMPTY(&clp->nfsc_deleg))
1571             panic("nfsclexp");
1572
1573         /*
1574          * Now, try and reopen against the server.
1575          */
1576         LIST_FOREACH_SAFE(owp, &clp->nfsc_owner, nfsow_list, nowp) {
1577                 owp->nfsow_seqid = 0;
1578                 LIST_FOREACH_SAFE(op, &owp->nfsow_open, nfso_list, nop) {
1579                         ret = nfscl_expireopen(clp, op, nmp, cred, p);
1580                         if (ret && !printed) {
1581                                 printed = 1;
1582                                 printf("nfsv4 expired locks lost\n");
1583                         }
1584                 }
1585                 if (LIST_EMPTY(&owp->nfsow_open))
1586                         nfscl_freeopenowner(owp, 0);
1587         }
1588 }
1589
1590 #ifndef __FreeBSD__
1591 /*
1592  * Called from exit() upon process termination.
1593  */
1594 APPLESTATIC void
1595 nfscl_cleanup(NFSPROC_T *p)
1596 {
1597         struct nfsclclient *clp;
1598         u_int8_t own[NFSV4CL_LOCKNAMELEN];
1599
1600         if (!nfscl_inited)
1601                 return;
1602         nfscl_filllockowner(p, own);
1603
1604         NFSLOCKCLSTATE();
1605         /*
1606          * Loop through all the clientids, looking for the OpenOwners.
1607          */
1608         LIST_FOREACH(clp, &nfsclhead, nfsc_list)
1609                 nfscl_cleanup_common(clp, own);
1610         NFSUNLOCKCLSTATE();
1611 }
1612 #endif  /* !__FreeBSD__ */
1613
1614 /*
1615  * Common code used by nfscl_cleanup() and nfscl_cleanupkext().
1616  * Must be called with CLSTATE lock held.
1617  */
1618 static void
1619 nfscl_cleanup_common(struct nfsclclient *clp, u_int8_t *own)
1620 {
1621         struct nfsclowner *owp, *nowp;
1622         struct nfsclopen *op;
1623         struct nfscllockowner *lp, *nlp;
1624         struct nfscldeleg *dp;
1625
1626         /* First, get rid of local locks on delegations. */
1627         TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list) {
1628                 LIST_FOREACH_SAFE(lp, &dp->nfsdl_lock, nfsl_list, nlp) {
1629                     if (!NFSBCMP(lp->nfsl_owner, own, NFSV4CL_LOCKNAMELEN)) {
1630                         if ((lp->nfsl_rwlock.nfslock_lock & NFSV4LOCK_WANTED))
1631                             panic("nfscllckw");
1632                         nfscl_freelockowner(lp, 1);
1633                     }
1634                 }
1635         }
1636         owp = LIST_FIRST(&clp->nfsc_owner);
1637         while (owp != NULL) {
1638                 nowp = LIST_NEXT(owp, nfsow_list);
1639                 if (!NFSBCMP(owp->nfsow_owner, own,
1640                     NFSV4CL_LOCKNAMELEN)) {
1641                         /*
1642                          * If there are children that haven't closed the
1643                          * file descriptors yet, the opens will still be
1644                          * here. For that case, let the renew thread clear
1645                          * out the OpenOwner later.
1646                          */
1647                         if (LIST_EMPTY(&owp->nfsow_open))
1648                                 nfscl_freeopenowner(owp, 0);
1649                         else
1650                                 owp->nfsow_defunct = 1;
1651                 } else {
1652                         /* look for lockowners on other opens */
1653                         LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
1654                                 LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
1655                                         if (!NFSBCMP(lp->nfsl_owner, own,
1656                                             NFSV4CL_LOCKNAMELEN))
1657                                                 lp->nfsl_defunct = 1;
1658                                 }
1659                         }
1660                 }
1661                 owp = nowp;
1662         }
1663
1664         /* and check the defunct list */
1665         LIST_FOREACH(lp, &clp->nfsc_defunctlockowner, nfsl_list) {
1666                 if (!NFSBCMP(lp->nfsl_owner, own, NFSV4CL_LOCKNAMELEN))
1667                     lp->nfsl_defunct = 1;
1668         }
1669 }
1670
1671 #if defined(APPLEKEXT) || defined(__FreeBSD__)
1672 /*
1673  * Simulate the call nfscl_cleanup() by looking for open owners associated
1674  * with processes that no longer exist, since a call to nfscl_cleanup()
1675  * can't be patched into exit().
1676  */
1677 static void
1678 nfscl_cleanupkext(struct nfsclclient *clp)
1679 {
1680         struct nfsclowner *owp, *nowp;
1681         struct nfscllockowner *lp;
1682
1683         NFSPROCLISTLOCK();
1684         NFSLOCKCLSTATE();
1685         LIST_FOREACH_SAFE(owp, &clp->nfsc_owner, nfsow_list, nowp) {
1686                 if (nfscl_procdoesntexist(owp->nfsow_owner))
1687                         nfscl_cleanup_common(clp, owp->nfsow_owner);
1688         }
1689
1690         /* and check the defunct list */
1691         LIST_FOREACH(lp, &clp->nfsc_defunctlockowner, nfsl_list) {
1692                 if (nfscl_procdoesntexist(lp->nfsl_owner))
1693                         lp->nfsl_defunct = 1;
1694         }
1695         NFSUNLOCKCLSTATE();
1696         NFSPROCLISTUNLOCK();
1697 }
1698 #endif  /* APPLEKEXT || __FreeBSD__ */
1699
1700 /*
1701  * Called from nfs umount to free up the clientid.
1702  */
1703 APPLESTATIC void
1704 nfscl_umount(struct nfsmount *nmp, NFSPROC_T *p)
1705 {
1706         struct nfsclclient *clp;
1707         struct ucred *cred;
1708         int igotlock;
1709
1710         clp = nmp->nm_clp;
1711         if (clp != NULL) {
1712                 if ((clp->nfsc_flags & NFSCLFLAGS_INITED) == 0)
1713                         panic("nfscl umount");
1714
1715                 /*
1716                  * First, handshake with the nfscl renew thread, to terminate
1717                  * it.
1718                  */
1719                 clp->nfsc_flags |= NFSCLFLAGS_UMOUNT;
1720                 while (clp->nfsc_flags & NFSCLFLAGS_HASTHREAD)
1721                         (void) tsleep((caddr_t)clp, PWAIT, "nfsclumnt", hz);
1722
1723                 NFSLOCKCLSTATE();
1724                 do {
1725                         igotlock = nfsv4_lock(&clp->nfsc_lock, 1, NULL,
1726                             NFSCLSTATEMUTEXPTR);
1727                 } while (!igotlock);
1728                 NFSUNLOCKCLSTATE();
1729
1730                 /*
1731                  * Free up all the state. It will expire on the server, but
1732                  * maybe we should do a SetClientId/SetClientIdConfirm so
1733                  * the server throws it away?
1734                  */
1735                 LIST_REMOVE(clp, nfsc_list);
1736                 nfscl_delegreturnall(clp, p);
1737                 cred = newnfs_getcred();
1738                 (void) nfsrpc_setclient(nmp, clp, cred, p);
1739                 nfscl_cleanclient(clp);
1740                 nmp->nm_clp = NULL;
1741                 NFSFREECRED(cred);
1742                 FREE((caddr_t)clp, M_NFSCLCLIENT);
1743         }
1744
1745 }
1746
1747 /*
1748  * This function is called when a server replies with NFSERR_STALECLIENTID
1749  * or NFSERR_STALESTATEID. It traverses the clientid lists, doing Opens
1750  * and Locks with reclaim. If these fail, it deletes the corresponding state.
1751  */
1752 static void
1753 nfscl_recover(struct nfsclclient *clp, struct ucred *cred, NFSPROC_T *p)
1754 {
1755         struct nfsclowner *owp, *nowp;
1756         struct nfsclopen *op, *nop;
1757         struct nfscllockowner *lp, *nlp;
1758         struct nfscllock *lop, *nlop;
1759         struct nfscldeleg *dp, *ndp, *tdp;
1760         struct nfsmount *nmp;
1761         struct ucred *tcred;
1762         struct nfsclopenhead extra_open;
1763         struct nfscldeleghead extra_deleg;
1764         struct nfsreq *rep;
1765         u_int64_t len;
1766         u_int32_t delegtype = NFSV4OPEN_DELEGATEWRITE, mode;
1767         int igotlock = 0, error, trycnt, firstlock, s;
1768
1769         /*
1770          * First, lock the client structure, so everyone else will
1771          * block when trying to use state.
1772          */
1773         NFSLOCKCLSTATE();
1774         do {
1775                 igotlock = nfsv4_lock(&clp->nfsc_lock, 1, NULL,
1776                     NFSCLSTATEMUTEXPTR);
1777         } while (!igotlock);
1778         NFSUNLOCKCLSTATE();
1779
1780         nmp = clp->nfsc_nmp;
1781         if (nmp == NULL)
1782                 panic("nfscl recover");
1783         trycnt = 5;
1784         do {
1785                 error = nfsrpc_setclient(nmp, clp, cred, p);
1786         } while ((error == NFSERR_STALECLIENTID ||
1787              error == NFSERR_STALEDONTRECOVER) && --trycnt > 0);
1788         if (error) {
1789                 nfscl_cleanclient(clp);
1790                 clp->nfsc_flags &= ~(NFSCLFLAGS_HASCLIENTID |
1791                     NFSCLFLAGS_RECOVER);
1792                 NFSLOCKCLSTATE();
1793                 nfsv4_unlock(&clp->nfsc_lock, 0);
1794                 NFSUNLOCKCLSTATE();
1795                 return;
1796         }
1797         clp->nfsc_flags |= NFSCLFLAGS_HASCLIENTID;
1798         clp->nfsc_flags &= ~NFSCLFLAGS_RECOVER;
1799
1800         /*
1801          * Mark requests already queued on the server, so that they don't
1802          * initiate another recovery cycle. Any requests already in the
1803          * queue that handle state information will have the old stale
1804          * clientid/stateid and will get a NFSERR_STALESTATEID or
1805          * NFSERR_STALECLIENTID reply from the server. This will be
1806          * translated to NFSERR_STALEDONTRECOVER when R_DONTRECOVER is set.
1807          */
1808         s = splsoftclock();
1809         NFSLOCKREQ();
1810         TAILQ_FOREACH(rep, &nfsd_reqq, r_chain) {
1811                 if (rep->r_nmp == nmp)
1812                         rep->r_flags |= R_DONTRECOVER;
1813         }
1814         NFSUNLOCKREQ();
1815         splx(s);
1816
1817         /* get rid of defunct lockowners */
1818         LIST_FOREACH_SAFE(lp, &clp->nfsc_defunctlockowner, nfsl_list, nlp) {
1819                 nfscl_freelockowner(lp, 0);
1820         }
1821
1822         /*
1823          * Now, mark all delegations "need reclaim".
1824          */
1825         TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list)
1826                 dp->nfsdl_flags |= NFSCLDL_NEEDRECLAIM;
1827
1828         TAILQ_INIT(&extra_deleg);
1829         LIST_INIT(&extra_open);
1830         /*
1831          * Now traverse the state lists, doing Open and Lock Reclaims.
1832          */
1833         tcred = newnfs_getcred();
1834         owp = LIST_FIRST(&clp->nfsc_owner);
1835         while (owp != NULL) {
1836             nowp = LIST_NEXT(owp, nfsow_list);
1837             owp->nfsow_seqid = 0;
1838             op = LIST_FIRST(&owp->nfsow_open);
1839             while (op != NULL) {
1840                 nop = LIST_NEXT(op, nfso_list);
1841                 if (error != NFSERR_NOGRACE) {
1842                     /* Search for a delegation to reclaim with the open */
1843                     TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list) {
1844                         if (!(dp->nfsdl_flags & NFSCLDL_NEEDRECLAIM))
1845                             continue;
1846                         if ((dp->nfsdl_flags & NFSCLDL_WRITE)) {
1847                             mode = NFSV4OPEN_ACCESSWRITE;
1848                             delegtype = NFSV4OPEN_DELEGATEWRITE;
1849                         } else {
1850                             mode = NFSV4OPEN_ACCESSREAD;
1851                             delegtype = NFSV4OPEN_DELEGATEREAD;
1852                         }
1853                         if ((op->nfso_mode & mode) == mode &&
1854                             op->nfso_fhlen == dp->nfsdl_fhlen &&
1855                             !NFSBCMP(op->nfso_fh, dp->nfsdl_fh, op->nfso_fhlen))
1856                             break;
1857                     }
1858                     ndp = dp;
1859                     if (dp == NULL)
1860                         delegtype = NFSV4OPEN_DELEGATENONE;
1861                     newnfs_copycred(&op->nfso_cred, tcred);
1862                     error = nfscl_tryopen(nmp, NULL, op->nfso_fh,
1863                         op->nfso_fhlen, op->nfso_fh, op->nfso_fhlen,
1864                         op->nfso_mode, op, NULL, 0, &ndp, 1, delegtype,
1865                         tcred, p);
1866                     if (!error) {
1867                         /* Handle any replied delegation */
1868                         if (ndp != NULL && ((ndp->nfsdl_flags & NFSCLDL_WRITE)
1869                             || NFSMNT_RDONLY(nmp->nm_mountp))) {
1870                             if ((ndp->nfsdl_flags & NFSCLDL_WRITE))
1871                                 mode = NFSV4OPEN_ACCESSWRITE;
1872                             else
1873                                 mode = NFSV4OPEN_ACCESSREAD;
1874                             TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list) {
1875                                 if (!(dp->nfsdl_flags & NFSCLDL_NEEDRECLAIM))
1876                                     continue;
1877                                 if ((op->nfso_mode & mode) == mode &&
1878                                     op->nfso_fhlen == dp->nfsdl_fhlen &&
1879                                     !NFSBCMP(op->nfso_fh, dp->nfsdl_fh,
1880                                     op->nfso_fhlen)) {
1881                                     dp->nfsdl_stateid = ndp->nfsdl_stateid;
1882                                     dp->nfsdl_sizelimit = ndp->nfsdl_sizelimit;
1883                                     dp->nfsdl_ace = ndp->nfsdl_ace;
1884                                     dp->nfsdl_change = ndp->nfsdl_change;
1885                                     dp->nfsdl_flags &= ~NFSCLDL_NEEDRECLAIM;
1886                                     if ((ndp->nfsdl_flags & NFSCLDL_RECALL))
1887                                         dp->nfsdl_flags |= NFSCLDL_RECALL;
1888                                     FREE((caddr_t)ndp, M_NFSCLDELEG);
1889                                     ndp = NULL;
1890                                     break;
1891                                 }
1892                             }
1893                         }
1894                         if (ndp != NULL)
1895                             TAILQ_INSERT_HEAD(&extra_deleg, ndp, nfsdl_list);
1896
1897                         /* and reclaim all byte range locks */
1898                         lp = LIST_FIRST(&op->nfso_lock);
1899                         while (lp != NULL) {
1900                             nlp = LIST_NEXT(lp, nfsl_list);
1901                             lp->nfsl_seqid = 0;
1902                             firstlock = 1;
1903                             lop = LIST_FIRST(&lp->nfsl_lock);
1904                             while (lop != NULL) {
1905                                 nlop = LIST_NEXT(lop, nfslo_list);
1906                                 if (lop->nfslo_end == NFS64BITSSET)
1907                                     len = NFS64BITSSET;
1908                                 else
1909                                     len = lop->nfslo_end - lop->nfslo_first;
1910                                 if (error != NFSERR_NOGRACE)
1911                                     error = nfscl_trylock(nmp, NULL,
1912                                         op->nfso_fh, op->nfso_fhlen, lp,
1913                                         firstlock, 1, lop->nfslo_first, len,
1914                                         lop->nfslo_type, tcred, p);
1915                                 if (error != 0)
1916                                     nfscl_freelock(lop, 0);
1917                                 else
1918                                     firstlock = 0;
1919                                 lop = nlop;
1920                             }
1921                             /* If no locks, but a lockowner, just delete it. */
1922                             if (LIST_EMPTY(&lp->nfsl_lock))
1923                                 nfscl_freelockowner(lp, 0);
1924                             lp = nlp;
1925                         }
1926                     } else {
1927                         nfscl_freeopen(op, 0);
1928                     }
1929                 }
1930                 op = nop;
1931             }
1932             owp = nowp;
1933         }
1934
1935         /*
1936          * Now, try and get any delegations not yet reclaimed by cobbling
1937          * to-gether an appropriate open.
1938          */
1939         nowp = NULL;
1940         dp = TAILQ_FIRST(&clp->nfsc_deleg);
1941         while (dp != NULL) {
1942             ndp = TAILQ_NEXT(dp, nfsdl_list);
1943             if ((dp->nfsdl_flags & NFSCLDL_NEEDRECLAIM)) {
1944                 if (nowp == NULL) {
1945                     MALLOC(nowp, struct nfsclowner *,
1946                         sizeof (struct nfsclowner), M_NFSCLOWNER, M_WAITOK);
1947                     /*
1948                      * Name must be as long an largest possible
1949                      * NFSV4CL_LOCKNAMELEN. 12 for now.
1950                      */
1951                     NFSBCOPY("RECLAIMDELEG", nowp->nfsow_owner,
1952                         NFSV4CL_LOCKNAMELEN);
1953                     LIST_INIT(&nowp->nfsow_open);
1954                     nowp->nfsow_clp = clp;
1955                     nowp->nfsow_seqid = 0;
1956                     nowp->nfsow_defunct = 0;
1957                     nfscl_lockinit(&nowp->nfsow_rwlock);
1958                 }
1959                 nop = NULL;
1960                 if (error != NFSERR_NOGRACE) {
1961                     MALLOC(nop, struct nfsclopen *, sizeof (struct nfsclopen) +
1962                         dp->nfsdl_fhlen - 1, M_NFSCLOPEN, M_WAITOK);
1963                     nop->nfso_own = nowp;
1964                     if ((dp->nfsdl_flags & NFSCLDL_WRITE)) {
1965                         nop->nfso_mode = NFSV4OPEN_ACCESSWRITE;
1966                         delegtype = NFSV4OPEN_DELEGATEWRITE;
1967                     } else {
1968                         nop->nfso_mode = NFSV4OPEN_ACCESSREAD;
1969                         delegtype = NFSV4OPEN_DELEGATEREAD;
1970                     }
1971                     nop->nfso_opencnt = 0;
1972                     nop->nfso_posixlock = 1;
1973                     nop->nfso_fhlen = dp->nfsdl_fhlen;
1974                     NFSBCOPY(dp->nfsdl_fh, nop->nfso_fh, dp->nfsdl_fhlen);
1975                     LIST_INIT(&nop->nfso_lock);
1976                     nop->nfso_stateid.seqid = 0;
1977                     nop->nfso_stateid.other[0] = 0;
1978                     nop->nfso_stateid.other[1] = 0;
1979                     nop->nfso_stateid.other[2] = 0;
1980                     newnfs_copycred(&dp->nfsdl_cred, tcred);
1981                     newnfs_copyincred(tcred, &nop->nfso_cred);
1982                     tdp = NULL;
1983                     error = nfscl_tryopen(nmp, NULL, nop->nfso_fh,
1984                         nop->nfso_fhlen, nop->nfso_fh, nop->nfso_fhlen,
1985                         nop->nfso_mode, nop, NULL, 0, &tdp, 1,
1986                         delegtype, tcred, p);
1987                     if (tdp != NULL) {
1988                         if ((tdp->nfsdl_flags & NFSCLDL_WRITE))
1989                             mode = NFSV4OPEN_ACCESSWRITE;
1990                         else
1991                             mode = NFSV4OPEN_ACCESSREAD;
1992                         if ((nop->nfso_mode & mode) == mode &&
1993                             nop->nfso_fhlen == tdp->nfsdl_fhlen &&
1994                             !NFSBCMP(nop->nfso_fh, tdp->nfsdl_fh,
1995                             nop->nfso_fhlen)) {
1996                             dp->nfsdl_stateid = tdp->nfsdl_stateid;
1997                             dp->nfsdl_sizelimit = tdp->nfsdl_sizelimit;
1998                             dp->nfsdl_ace = tdp->nfsdl_ace;
1999                             dp->nfsdl_change = tdp->nfsdl_change;
2000                             dp->nfsdl_flags &= ~NFSCLDL_NEEDRECLAIM;
2001                             if ((tdp->nfsdl_flags & NFSCLDL_RECALL))
2002                                 dp->nfsdl_flags |= NFSCLDL_RECALL;
2003                             FREE((caddr_t)tdp, M_NFSCLDELEG);
2004                         } else {
2005                             TAILQ_INSERT_HEAD(&extra_deleg, tdp, nfsdl_list);
2006                         }
2007                     }
2008                 }
2009                 if (error) {
2010                     if (nop != NULL)
2011                         FREE((caddr_t)nop, M_NFSCLOPEN);
2012                     /*
2013                      * Couldn't reclaim it, so throw the state
2014                      * away. Ouch!!
2015                      */
2016                     nfscl_cleandeleg(dp);
2017                     nfscl_freedeleg(&clp->nfsc_deleg, dp);
2018                 } else {
2019                     LIST_INSERT_HEAD(&extra_open, nop, nfso_list);
2020                 }
2021             }
2022             dp = ndp;
2023         }
2024
2025         /*
2026          * Now, get rid of extra Opens and Delegations.
2027          */
2028         LIST_FOREACH_SAFE(op, &extra_open, nfso_list, nop) {
2029                 do {
2030                         newnfs_copycred(&op->nfso_cred, tcred);
2031                         error = nfscl_tryclose(op, tcred, nmp, p);
2032                         if (error == NFSERR_GRACE)
2033                                 (void) nfs_catnap(PZERO, "nfsexcls");
2034                 } while (error == NFSERR_GRACE);
2035                 LIST_REMOVE(op, nfso_list);
2036                 FREE((caddr_t)op, M_NFSCLOPEN);
2037         }
2038         if (nowp != NULL)
2039                 FREE((caddr_t)nowp, M_NFSCLOWNER);
2040
2041         TAILQ_FOREACH_SAFE(dp, &extra_deleg, nfsdl_list, ndp) {
2042                 do {
2043                         newnfs_copycred(&dp->nfsdl_cred, tcred);
2044                         error = nfscl_trydelegreturn(dp, tcred, nmp, p);
2045                         if (error == NFSERR_GRACE)
2046                                 (void) nfs_catnap(PZERO, "nfsexdlg");
2047                 } while (error == NFSERR_GRACE);
2048                 TAILQ_REMOVE(&extra_deleg, dp, nfsdl_list);
2049                 FREE((caddr_t)dp, M_NFSCLDELEG);
2050         }
2051
2052         NFSLOCKCLSTATE();
2053         nfsv4_unlock(&clp->nfsc_lock, 0);
2054         NFSUNLOCKCLSTATE();
2055         NFSFREECRED(tcred);
2056 }
2057
2058 /*
2059  * This function is called when a server replies with NFSERR_EXPIRED.
2060  * It deletes all state for the client and does a fresh SetClientId/confirm.
2061  * XXX Someday it should post a signal to the process(es) that hold the
2062  * state, so they know that lock state has been lost.
2063  */
2064 APPLESTATIC int
2065 nfscl_hasexpired(struct nfsclclient *clp, u_int32_t clidrev, NFSPROC_T *p)
2066 {
2067         struct nfscllockowner *lp, *nlp;
2068         struct nfsmount *nmp;
2069         struct ucred *cred;
2070         int igotlock = 0, error, trycnt;
2071
2072         /*
2073          * If the clientid has gone away or a new SetClientid has already
2074          * been done, just return ok.
2075          */
2076         if (clp == NULL || clidrev != clp->nfsc_clientidrev)
2077                 return (0);
2078
2079         /*
2080          * First, lock the client structure, so everyone else will
2081          * block when trying to use state. Also, use NFSCLFLAGS_EXPIREIT so
2082          * that only one thread does the work.
2083          */
2084         NFSLOCKCLSTATE();
2085         clp->nfsc_flags |= NFSCLFLAGS_EXPIREIT;
2086         do {
2087                 igotlock = nfsv4_lock(&clp->nfsc_lock, 1, NULL,
2088                     NFSCLSTATEMUTEXPTR);
2089         } while (!igotlock && (clp->nfsc_flags & NFSCLFLAGS_EXPIREIT));
2090         if ((clp->nfsc_flags & NFSCLFLAGS_EXPIREIT) == 0) {
2091                 if (igotlock)
2092                         nfsv4_unlock(&clp->nfsc_lock, 0);
2093                 NFSUNLOCKCLSTATE();
2094                 return (0);
2095         }
2096         NFSUNLOCKCLSTATE();
2097
2098         nmp = clp->nfsc_nmp;
2099         if (nmp == NULL)
2100                 panic("nfscl expired");
2101         cred = newnfs_getcred();
2102         trycnt = 5;
2103         do {
2104                 error = nfsrpc_setclient(nmp, clp, cred, p);
2105         } while ((error == NFSERR_STALECLIENTID ||
2106              error == NFSERR_STALEDONTRECOVER) && --trycnt > 0);
2107         if (error) {
2108                 /*
2109                  * Clear out any state.
2110                  */
2111                 nfscl_cleanclient(clp);
2112                 clp->nfsc_flags &= ~(NFSCLFLAGS_HASCLIENTID |
2113                     NFSCLFLAGS_RECOVER);
2114         } else {
2115                 /* get rid of defunct lockowners */
2116                 LIST_FOREACH_SAFE(lp, &clp->nfsc_defunctlockowner, nfsl_list,
2117                     nlp) {
2118                         nfscl_freelockowner(lp, 0);
2119                 }
2120
2121                 /*
2122                  * Expire the state for the client.
2123                  */
2124                 nfscl_expireclient(clp, nmp, cred, p);
2125                 clp->nfsc_flags |= NFSCLFLAGS_HASCLIENTID;
2126                 clp->nfsc_flags &= ~NFSCLFLAGS_RECOVER;
2127         }
2128         NFSFREECRED(cred);
2129         clp->nfsc_flags &= ~NFSCLFLAGS_EXPIREIT;
2130         NFSLOCKCLSTATE();
2131         nfsv4_unlock(&clp->nfsc_lock, 0);
2132         NFSUNLOCKCLSTATE();
2133         return (error);
2134 }
2135
2136 /*
2137  * This function inserts a lock in the list after insert_lop.
2138  */
2139 static void
2140 nfscl_insertlock(struct nfscllockowner *lp, struct nfscllock *new_lop,
2141     struct nfscllock *insert_lop, int local)
2142 {
2143
2144         if ((struct nfscllockowner *)insert_lop == lp)
2145                 LIST_INSERT_HEAD(&lp->nfsl_lock, new_lop, nfslo_list);
2146         else
2147                 LIST_INSERT_AFTER(insert_lop, new_lop, nfslo_list);
2148         if (local)
2149                 newnfsstats.cllocallocks++;
2150         else
2151                 newnfsstats.cllocks++;
2152 }
2153
2154 /*
2155  * This function updates the locking for a lock owner and given file. It
2156  * maintains a list of lock ranges ordered on increasing file offset that
2157  * are NFSCLLOCK_READ or NFSCLLOCK_WRITE and non-overlapping (aka POSIX style).
2158  * It always adds new_lop to the list and sometimes uses the one pointed
2159  * at by other_lopp.
2160  * Returns 1 if the locks were modified, 0 otherwise.
2161  */
2162 static int
2163 nfscl_updatelock(struct nfscllockowner *lp, struct nfscllock **new_lopp,
2164     struct nfscllock **other_lopp, int local)
2165 {
2166         struct nfscllock *new_lop = *new_lopp;
2167         struct nfscllock *lop, *tlop, *ilop;
2168         struct nfscllock *other_lop;
2169         int unlock = 0, modified = 0;
2170         u_int64_t tmp;
2171
2172         /*
2173          * Work down the list until the lock is merged.
2174          */
2175         if (new_lop->nfslo_type == F_UNLCK)
2176                 unlock = 1;
2177         ilop = (struct nfscllock *)lp;
2178         lop = LIST_FIRST(&lp->nfsl_lock);
2179         while (lop != NULL) {
2180             /*
2181              * Only check locks for this file that aren't before the start of
2182              * new lock's range.
2183              */
2184             if (lop->nfslo_end >= new_lop->nfslo_first) {
2185                 if (new_lop->nfslo_end < lop->nfslo_first) {
2186                     /*
2187                      * If the new lock ends before the start of the
2188                      * current lock's range, no merge, just insert
2189                      * the new lock.
2190                      */
2191                     break;
2192                 }
2193                 if (new_lop->nfslo_type == lop->nfslo_type ||
2194                     (new_lop->nfslo_first <= lop->nfslo_first &&
2195                      new_lop->nfslo_end >= lop->nfslo_end)) {
2196                     /*
2197                      * This lock can be absorbed by the new lock/unlock.
2198                      * This happens when it covers the entire range
2199                      * of the old lock or is contiguous
2200                      * with the old lock and is of the same type or an
2201                      * unlock.
2202                      */
2203                     if (new_lop->nfslo_type != lop->nfslo_type ||
2204                         new_lop->nfslo_first != lop->nfslo_first ||
2205                         new_lop->nfslo_end != lop->nfslo_end)
2206                         modified = 1;
2207                     if (lop->nfslo_first < new_lop->nfslo_first)
2208                         new_lop->nfslo_first = lop->nfslo_first;
2209                     if (lop->nfslo_end > new_lop->nfslo_end)
2210                         new_lop->nfslo_end = lop->nfslo_end;
2211                     tlop = lop;
2212                     lop = LIST_NEXT(lop, nfslo_list);
2213                     nfscl_freelock(tlop, local);
2214                     continue;
2215                 }
2216
2217                 /*
2218                  * All these cases are for contiguous locks that are not the
2219                  * same type, so they can't be merged.
2220                  */
2221                 if (new_lop->nfslo_first <= lop->nfslo_first) {
2222                     /*
2223                      * This case is where the new lock overlaps with the
2224                      * first part of the old lock. Move the start of the
2225                      * old lock to just past the end of the new lock. The
2226                      * new lock will be inserted in front of the old, since
2227                      * ilop hasn't been updated. (We are done now.)
2228                      */
2229                     if (lop->nfslo_first != new_lop->nfslo_end) {
2230                         lop->nfslo_first = new_lop->nfslo_end;
2231                         modified = 1;
2232                     }
2233                     break;
2234                 }
2235                 if (new_lop->nfslo_end >= lop->nfslo_end) {
2236                     /*
2237                      * This case is where the new lock overlaps with the
2238                      * end of the old lock's range. Move the old lock's
2239                      * end to just before the new lock's first and insert
2240                      * the new lock after the old lock.
2241                      * Might not be done yet, since the new lock could
2242                      * overlap further locks with higher ranges.
2243                      */
2244                     if (lop->nfslo_end != new_lop->nfslo_first) {
2245                         lop->nfslo_end = new_lop->nfslo_first;
2246                         modified = 1;
2247                     }
2248                     ilop = lop;
2249                     lop = LIST_NEXT(lop, nfslo_list);
2250                     continue;
2251                 }
2252                 /*
2253                  * The final case is where the new lock's range is in the
2254                  * middle of the current lock's and splits the current lock
2255                  * up. Use *other_lopp to handle the second part of the
2256                  * split old lock range. (We are done now.)
2257                  * For unlock, we use new_lop as other_lop and tmp, since
2258                  * other_lop and new_lop are the same for this case.
2259                  * We noted the unlock case above, so we don't need
2260                  * new_lop->nfslo_type any longer.
2261                  */
2262                 tmp = new_lop->nfslo_first;
2263                 if (unlock) {
2264                     other_lop = new_lop;
2265                     *new_lopp = NULL;
2266                 } else {
2267                     other_lop = *other_lopp;
2268                     *other_lopp = NULL;
2269                 }
2270                 other_lop->nfslo_first = new_lop->nfslo_end;
2271                 other_lop->nfslo_end = lop->nfslo_end;
2272                 other_lop->nfslo_type = lop->nfslo_type;
2273                 lop->nfslo_end = tmp;
2274                 nfscl_insertlock(lp, other_lop, lop, local);
2275                 ilop = lop;
2276                 modified = 1;
2277                 break;
2278             }
2279             ilop = lop;
2280             lop = LIST_NEXT(lop, nfslo_list);
2281             if (lop == NULL)
2282                 break;
2283         }
2284
2285         /*
2286          * Insert the new lock in the list at the appropriate place.
2287          */
2288         if (!unlock) {
2289                 nfscl_insertlock(lp, new_lop, ilop, local);
2290                 *new_lopp = NULL;
2291                 modified = 1;
2292         }
2293         return (modified);
2294 }
2295
2296 /*
2297  * This function must be run as a kernel thread.
2298  * It does Renew Ops and recovery, when required.
2299  */
2300 APPLESTATIC void
2301 nfscl_renewthread(struct nfsclclient *clp, NFSPROC_T *p)
2302 {
2303         struct nfsclowner *owp, *nowp;
2304         struct nfsclopen *op;
2305         struct nfscllockowner *lp, *nlp, *olp;
2306         struct nfscldeleghead dh;
2307         struct nfscllockownerhead lh;
2308         struct nfscldeleg *dp, *ndp;
2309         struct ucred *cred;
2310         u_int32_t clidrev;
2311         int error, cbpathdown, islept, igotlock, ret, clearok;
2312
2313         cred = newnfs_getcred();
2314         clp->nfsc_flags |= NFSCLFLAGS_HASTHREAD;
2315         for(;;) {
2316                 newnfs_setroot(cred);
2317                 cbpathdown = 0;
2318                 if (clp->nfsc_flags & NFSCLFLAGS_RECOVER)
2319                         nfscl_recover(clp, cred, p);
2320                 if (clp->nfsc_expire <= NFSD_MONOSEC &&
2321                     (clp->nfsc_flags & NFSCLFLAGS_HASCLIENTID)) {
2322                         clp->nfsc_expire = NFSD_MONOSEC + clp->nfsc_renew;
2323                         clidrev = clp->nfsc_clientidrev;
2324                         error = nfsrpc_renew(clp, cred, p);
2325                         if (error == NFSERR_CBPATHDOWN)
2326                             cbpathdown = 1;
2327                         else if (error == NFSERR_STALECLIENTID)
2328                             clp->nfsc_flags |= NFSCLFLAGS_RECOVER;
2329                         else if (error == NFSERR_EXPIRED)
2330                             (void) nfscl_hasexpired(clp, clidrev, p);
2331                 }
2332
2333                 LIST_INIT(&lh);
2334                 TAILQ_INIT(&dh);
2335                 NFSLOCKCLSTATE();
2336                 if (cbpathdown)
2337                         /* It's a Total Recall! */
2338                         nfscl_totalrecall(clp);
2339
2340                 /*
2341                  * Now, handle defunct owners.
2342                  */
2343                 owp = LIST_FIRST(&clp->nfsc_owner);
2344                 while (owp != NULL) {
2345                     nowp = LIST_NEXT(owp, nfsow_list);
2346                     if (LIST_EMPTY(&owp->nfsow_open)) {
2347                         if (owp->nfsow_defunct)
2348                             nfscl_freeopenowner(owp, 0);
2349                     } else {
2350                         LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
2351                             lp = LIST_FIRST(&op->nfso_lock);
2352                             while (lp != NULL) {
2353                                 nlp = LIST_NEXT(lp, nfsl_list);
2354                                 if (lp->nfsl_defunct &&
2355                                     LIST_EMPTY(&lp->nfsl_lock)) {
2356                                     LIST_FOREACH(olp, &lh, nfsl_list) {
2357                                         if (!NFSBCMP(olp->nfsl_owner,
2358                                             lp->nfsl_owner,NFSV4CL_LOCKNAMELEN))
2359                                             break;
2360                                     }
2361                                     if (olp == NULL) {
2362                                         LIST_REMOVE(lp, nfsl_list);
2363                                         LIST_INSERT_HEAD(&lh, lp, nfsl_list);
2364                                     } else {
2365                                         nfscl_freelockowner(lp, 0);
2366                                     }
2367                                 }
2368                                 lp = nlp;
2369                             }
2370                         }
2371                     }
2372                     owp = nowp;
2373                 }
2374
2375                 /* also search the defunct list */
2376                 lp = LIST_FIRST(&clp->nfsc_defunctlockowner);
2377                 while (lp != NULL) {
2378                     nlp = LIST_NEXT(lp, nfsl_list);
2379                     if (lp->nfsl_defunct) {
2380                         LIST_FOREACH(olp, &lh, nfsl_list) {
2381                             if (!NFSBCMP(olp->nfsl_owner, lp->nfsl_owner,
2382                                 NFSV4CL_LOCKNAMELEN))
2383                                 break;
2384                         }
2385                         if (olp == NULL) {
2386                             LIST_REMOVE(lp, nfsl_list);
2387                             LIST_INSERT_HEAD(&lh, lp, nfsl_list);
2388                         } else {
2389                             nfscl_freelockowner(lp, 0);
2390                         }
2391                     }
2392                     lp = nlp;
2393                 }
2394                 /* and release defunct lock owners */
2395                 LIST_FOREACH_SAFE(lp, &lh, nfsl_list, nlp) {
2396                     nfscl_freelockowner(lp, 0);
2397                 }
2398
2399                 /*
2400                  * Do the recall on any delegations. To avoid trouble, always
2401                  * come back up here after having slept.
2402                  */
2403                 igotlock = 0;
2404 tryagain:
2405                 dp = TAILQ_FIRST(&clp->nfsc_deleg);
2406                 while (dp != NULL) {
2407                         ndp = TAILQ_NEXT(dp, nfsdl_list);
2408                         if ((dp->nfsdl_flags & NFSCLDL_RECALL)) {
2409                                 /*
2410                                  * Wait for outstanding I/O ops to be done.
2411                                  */
2412                                 if (dp->nfsdl_rwlock.nfslock_usecnt > 0) {
2413                                     if (igotlock) {
2414                                         nfsv4_unlock(&clp->nfsc_lock, 0);
2415                                         igotlock = 0;
2416                                     }
2417                                     dp->nfsdl_rwlock.nfslock_lock |=
2418                                         NFSV4LOCK_WANTED;
2419                                     (void) nfsmsleep(&dp->nfsdl_rwlock,
2420                                         NFSCLSTATEMUTEXPTR, PZERO, "nfscld",
2421                                         NULL);
2422                                     goto tryagain;
2423                                 }
2424                                 while (!igotlock) {
2425                                     igotlock = nfsv4_lock(&clp->nfsc_lock, 1,
2426                                         &islept, NFSCLSTATEMUTEXPTR);
2427                                     if (islept)
2428                                         goto tryagain;
2429                                 }
2430                                 NFSUNLOCKCLSTATE();
2431                                 newnfs_copycred(&dp->nfsdl_cred, cred);
2432                                 ret = nfscl_recalldeleg(clp, clp->nfsc_nmp, dp,
2433                                     NULL, cred, p);
2434                                 if (!ret) {
2435                                     nfscl_cleandeleg(dp);
2436                                     TAILQ_REMOVE(&clp->nfsc_deleg, dp,
2437                                         nfsdl_list);
2438                                     LIST_REMOVE(dp, nfsdl_hash);
2439                                     TAILQ_INSERT_HEAD(&dh, dp, nfsdl_list);
2440                                     nfscl_delegcnt--;
2441                                     newnfsstats.cldelegates--;
2442                                 }
2443                                 NFSLOCKCLSTATE();
2444                         }
2445                         dp = ndp;
2446                 }
2447
2448                 /*
2449                  * Clear out old delegations, if we are above the high water
2450                  * mark. Only clear out ones with no state related to them.
2451                  * The tailq list is in LRU order.
2452                  */
2453                 dp = TAILQ_LAST(&clp->nfsc_deleg, nfscldeleghead);
2454                 while (nfscl_delegcnt > nfscl_deleghighwater && dp != NULL) {
2455                     ndp = TAILQ_PREV(dp, nfscldeleghead, nfsdl_list);
2456                     if (dp->nfsdl_rwlock.nfslock_usecnt == 0 &&
2457                         dp->nfsdl_rwlock.nfslock_lock == 0 &&
2458                         dp->nfsdl_timestamp < NFSD_MONOSEC &&
2459                         !(dp->nfsdl_flags & (NFSCLDL_RECALL | NFSCLDL_ZAPPED |
2460                           NFSCLDL_NEEDRECLAIM))) {
2461                         clearok = 1;
2462                         LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
2463                             op = LIST_FIRST(&owp->nfsow_open);
2464                             if (op != NULL && op->nfso_opencnt > 0) {
2465                                 clearok = 0;
2466                                 break;
2467                             }
2468                         }
2469                         if (clearok) {
2470                             LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
2471                                 if (!LIST_EMPTY(&lp->nfsl_lock)) {
2472                                     clearok = 0;
2473                                     break;
2474                                 }
2475                             }
2476                         }
2477                         if (clearok) {
2478                             TAILQ_REMOVE(&clp->nfsc_deleg, dp, nfsdl_list);
2479                             LIST_REMOVE(dp, nfsdl_hash);
2480                             TAILQ_INSERT_HEAD(&dh, dp, nfsdl_list);
2481                             nfscl_delegcnt--;
2482                             newnfsstats.cldelegates--;
2483                         }
2484                     }
2485                     dp = ndp;
2486                 }
2487                 if (igotlock)
2488                         nfsv4_unlock(&clp->nfsc_lock, 0);
2489                 NFSUNLOCKCLSTATE();
2490
2491                 /*
2492                  * Delegreturn any delegations cleaned out or recalled.
2493                  */
2494                 TAILQ_FOREACH_SAFE(dp, &dh, nfsdl_list, ndp) {
2495                         newnfs_copycred(&dp->nfsdl_cred, cred);
2496                         (void) nfscl_trydelegreturn(dp, cred, clp->nfsc_nmp, p);
2497                         TAILQ_REMOVE(&dh, dp, nfsdl_list);
2498                         FREE((caddr_t)dp, M_NFSCLDELEG);
2499                 }
2500
2501 #if defined(APPLEKEXT) || defined(__FreeBSD__)
2502                 /*
2503                  * Simulate the calls to nfscl_cleanup() when a process
2504                  * exits, since the call can't be patched into exit().
2505                  */
2506                 {
2507                         struct timespec mytime;
2508                         static time_t prevsec = 0;
2509
2510                         NFSGETNANOTIME(&mytime);
2511                         if (prevsec != mytime.tv_sec) {
2512                                 prevsec = mytime.tv_sec;
2513                                 nfscl_cleanupkext(clp);
2514                         }
2515                 }
2516 #endif  /* APPLEKEXT || __FreeBSD__ */
2517
2518                 if ((clp->nfsc_flags & NFSCLFLAGS_RECOVER) == 0)
2519                     (void) tsleep((caddr_t)clp, PWAIT, "nfscl", hz);
2520                 if (clp->nfsc_flags & NFSCLFLAGS_UMOUNT) {
2521                         NFSFREECRED(cred);
2522                         clp->nfsc_flags &= ~NFSCLFLAGS_HASTHREAD;
2523                         wakeup((caddr_t)clp);
2524                         return;
2525                 }
2526         }
2527 }
2528
2529 /*
2530  * Initiate state recovery. Called when NFSERR_STALECLIENTID or
2531  * NFSERR_STALESTATEID is received.
2532  */
2533 APPLESTATIC void
2534 nfscl_initiate_recovery(struct nfsclclient *clp)
2535 {
2536
2537         if (clp == NULL)
2538                 return;
2539         NFSLOCKCLSTATE();
2540         clp->nfsc_flags |= NFSCLFLAGS_RECOVER;
2541         NFSUNLOCKCLSTATE();
2542         wakeup((caddr_t)clp);
2543 }
2544
2545 /*
2546  * Dump out the state stuff for debugging.
2547  */
2548 APPLESTATIC void
2549 nfscl_dumpstate(struct nfsmount *nmp, int openowner, int opens,
2550     int lockowner, int locks)
2551 {
2552         struct nfsclclient *clp;
2553         struct nfsclowner *owp;
2554         struct nfsclopen *op;
2555         struct nfscllockowner *lp;
2556         struct nfscllock *lop;
2557         struct nfscldeleg *dp;
2558
2559         clp = nmp->nm_clp;
2560         if (clp == NULL) {
2561                 printf("nfscl dumpstate NULL clp\n");
2562                 return;
2563         }
2564         NFSLOCKCLSTATE();
2565         TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list) {
2566           LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
2567             if (openowner && !LIST_EMPTY(&owp->nfsow_open))
2568                 printf("owner=0x%x 0x%x 0x%x 0x%x seqid=%d\n",
2569                     owp->nfsow_owner[0], owp->nfsow_owner[1],
2570                     owp->nfsow_owner[2], owp->nfsow_owner[3],
2571                     owp->nfsow_seqid);
2572             LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
2573                 if (opens)
2574                     printf("open st=0x%x 0x%x 0x%x cnt=%d fh12=0x%x\n",
2575                         op->nfso_stateid.other[0], op->nfso_stateid.other[1],
2576                         op->nfso_stateid.other[2], op->nfso_opencnt,
2577                         op->nfso_fh[12]);
2578                 LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
2579                     if (lockowner)
2580                         printf("lckown=0x%x 0x%x 0x%x 0x%x seqid=%d st=0x%x 0x%x 0x%x\n",
2581                             lp->nfsl_owner[0], lp->nfsl_owner[1],
2582                             lp->nfsl_owner[2], lp->nfsl_owner[3],
2583                             lp->nfsl_seqid,
2584                             lp->nfsl_stateid.other[0], lp->nfsl_stateid.other[1],
2585                             lp->nfsl_stateid.other[2]);
2586                     LIST_FOREACH(lop, &lp->nfsl_lock, nfslo_list) {
2587                         if (locks)
2588 #ifdef __FreeBSD__
2589                             printf("lck typ=%d fst=%ju end=%ju\n",
2590                                 lop->nfslo_type, (intmax_t)lop->nfslo_first,
2591                                 (intmax_t)lop->nfslo_end);
2592 #else
2593                             printf("lck typ=%d fst=%qd end=%qd\n",
2594                                 lop->nfslo_type, lop->nfslo_first,
2595                                 lop->nfslo_end);
2596 #endif
2597                     }
2598                 }
2599             }
2600           }
2601         }
2602         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
2603             if (openowner && !LIST_EMPTY(&owp->nfsow_open))
2604                 printf("owner=0x%x 0x%x 0x%x 0x%x seqid=%d\n",
2605                     owp->nfsow_owner[0], owp->nfsow_owner[1],
2606                     owp->nfsow_owner[2], owp->nfsow_owner[3],
2607                     owp->nfsow_seqid);
2608             LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
2609                 if (opens)
2610                     printf("open st=0x%x 0x%x 0x%x cnt=%d fh12=0x%x\n",
2611                         op->nfso_stateid.other[0], op->nfso_stateid.other[1],
2612                         op->nfso_stateid.other[2], op->nfso_opencnt,
2613                         op->nfso_fh[12]);
2614                 LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
2615                     if (lockowner)
2616                         printf("lckown=0x%x 0x%x 0x%x 0x%x seqid=%d st=0x%x 0x%x 0x%x\n",
2617                             lp->nfsl_owner[0], lp->nfsl_owner[1],
2618                             lp->nfsl_owner[2], lp->nfsl_owner[3],
2619                             lp->nfsl_seqid,
2620                             lp->nfsl_stateid.other[0], lp->nfsl_stateid.other[1],
2621                             lp->nfsl_stateid.other[2]);
2622                     LIST_FOREACH(lop, &lp->nfsl_lock, nfslo_list) {
2623                         if (locks)
2624 #ifdef __FreeBSD__
2625                             printf("lck typ=%d fst=%ju end=%ju\n",
2626                                 lop->nfslo_type, (intmax_t)lop->nfslo_first,
2627                                 (intmax_t)lop->nfslo_end);
2628 #else
2629                             printf("lck typ=%d fst=%qd end=%qd\n",
2630                                 lop->nfslo_type, lop->nfslo_first,
2631                                 lop->nfslo_end);
2632 #endif
2633                     }
2634                 }
2635             }
2636         }
2637         NFSUNLOCKCLSTATE();
2638 }
2639
2640 /*
2641  * Check for duplicate open owners and opens.
2642  * (Only used as a diagnostic aid.)
2643  */
2644 APPLESTATIC void
2645 nfscl_dupopen(vnode_t vp, int dupopens)
2646 {
2647         struct nfsclclient *clp;
2648         struct nfsclowner *owp, *owp2;
2649         struct nfsclopen *op, *op2;
2650         struct nfsfh *nfhp;
2651
2652         clp = VFSTONFS(vnode_mount(vp))->nm_clp;
2653         if (clp == NULL) {
2654                 printf("nfscl dupopen NULL clp\n");
2655                 return;
2656         }
2657         nfhp = VTONFS(vp)->n_fhp;
2658         NFSLOCKCLSTATE();
2659
2660         /*
2661          * First, search for duplicate owners.
2662          * These should never happen!
2663          */
2664         LIST_FOREACH(owp2, &clp->nfsc_owner, nfsow_list) {
2665             LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
2666                 if (owp != owp2 &&
2667                     !NFSBCMP(owp->nfsow_owner, owp2->nfsow_owner,
2668                     NFSV4CL_LOCKNAMELEN)) {
2669                         NFSUNLOCKCLSTATE();
2670                         printf("DUP OWNER\n");
2671                         nfscl_dumpstate(VFSTONFS(vnode_mount(vp)), 1, 1, 0, 0);
2672                         return;
2673                 }
2674             }
2675         }
2676
2677         /*
2678          * Now, search for duplicate stateids.
2679          * These shouldn't happen, either.
2680          */
2681         LIST_FOREACH(owp2, &clp->nfsc_owner, nfsow_list) {
2682             LIST_FOREACH(op2, &owp2->nfsow_open, nfso_list) {
2683                 LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
2684                     LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
2685                         if (op != op2 &&
2686                             (op->nfso_stateid.other[0] != 0 ||
2687                              op->nfso_stateid.other[1] != 0 ||
2688                              op->nfso_stateid.other[2] != 0) &&
2689                             op->nfso_stateid.other[0] == op2->nfso_stateid.other[0] &&
2690                             op->nfso_stateid.other[1] == op2->nfso_stateid.other[1] &&
2691                             op->nfso_stateid.other[2] == op2->nfso_stateid.other[2]) {
2692                             NFSUNLOCKCLSTATE();
2693                             printf("DUP STATEID\n");
2694                             nfscl_dumpstate(VFSTONFS(vnode_mount(vp)), 1, 1, 0,
2695                                 0);
2696                             return;
2697                         }
2698                     }
2699                 }
2700             }
2701         }
2702
2703         /*
2704          * Now search for duplicate opens.
2705          * Duplicate opens for the same owner
2706          * should never occur. Other duplicates are
2707          * possible and are checked for if "dupopens"
2708          * is true.
2709          */
2710         LIST_FOREACH(owp2, &clp->nfsc_owner, nfsow_list) {
2711             LIST_FOREACH(op2, &owp2->nfsow_open, nfso_list) {
2712                 if (nfhp->nfh_len == op2->nfso_fhlen &&
2713                     !NFSBCMP(nfhp->nfh_fh, op2->nfso_fh, nfhp->nfh_len)) {
2714                     LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
2715                         LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
2716                             if (op != op2 && nfhp->nfh_len == op->nfso_fhlen &&
2717                                 !NFSBCMP(nfhp->nfh_fh, op->nfso_fh, nfhp->nfh_len) &&
2718                                 (!NFSBCMP(op->nfso_own->nfsow_owner,
2719                                  op2->nfso_own->nfsow_owner, NFSV4CL_LOCKNAMELEN) ||
2720                                  dupopens)) {
2721                                 if (!NFSBCMP(op->nfso_own->nfsow_owner,
2722                                     op2->nfso_own->nfsow_owner, NFSV4CL_LOCKNAMELEN)) {
2723                                     NFSUNLOCKCLSTATE();
2724                                     printf("BADDUP OPEN\n");
2725                                 } else {
2726                                     NFSUNLOCKCLSTATE();
2727                                     printf("DUP OPEN\n");
2728                                 }
2729                                 nfscl_dumpstate(VFSTONFS(vnode_mount(vp)), 1, 1,
2730                                     0, 0);
2731                                 return;
2732                             }
2733                         }
2734                     }
2735                 }
2736             }
2737         }
2738         NFSUNLOCKCLSTATE();
2739 }
2740
2741 /*
2742  * During close, find an open that needs to be dereferenced and
2743  * dereference it. If there are no more opens for this file,
2744  * return the list of opens, so they can be closed on the
2745  * server. As such, opens aren't closed on the server until
2746  * all the opens for the file are closed off.
2747  * This is the safe way, since it is difficult to identify
2748  * which open the close is for.
2749  * If it returns 0 for success, there will be a referenced
2750  * clp returned via clpp and a list of opens to close/free
2751  * on ohp.
2752  */
2753 APPLESTATIC int
2754 nfscl_getclose(vnode_t vp, struct nfsclclient **clpp,
2755     struct nfsclopenhead *ohp)
2756 {
2757         struct nfsclclient *clp;
2758         struct nfsclowner *owp, *nowp;
2759         struct nfsclopen *op, *nop;
2760         struct nfscldeleg *dp;
2761         struct nfsfh *nfhp;
2762         int error, notdecr, candelete;
2763
2764         error = nfscl_getcl(vp, NULL, NULL, &clp);
2765         if (error)
2766                 return (error);
2767         *clpp = clp;
2768
2769         if (ohp != NULL)
2770                 LIST_INIT(ohp);
2771         nfhp = VTONFS(vp)->n_fhp;
2772         notdecr = 1;
2773         NFSLOCKCLSTATE();
2774         /*
2775          * First, look for one under a delegation that was locally issued
2776          * and just decrement the opencnt for it. Since all my Opens against
2777          * the server are DENY_NONE, I don't see a problem with hanging
2778          * onto them. (It is much easier to use one of the extant Opens
2779          * that I already have on the server when a Delegation is recalled
2780          * than to do fresh Opens.) Someday, I might need to rethink this, but..
2781          */
2782         dp = nfscl_finddeleg(clp, nfhp->nfh_fh, nfhp->nfh_len);
2783         if (dp != NULL) {
2784                 LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
2785                         op = LIST_FIRST(&owp->nfsow_open);
2786                         if (op != NULL) {
2787                                 /*
2788                                  * Since a delegation is for a file, there
2789                                  * should never be more than one open for
2790                                  * each openowner.
2791                                  */
2792                                 if (LIST_NEXT(op, nfso_list) != NULL)
2793                                         panic("nfscdeleg opens");
2794                                 if (notdecr && op->nfso_opencnt > 0) {
2795                                         notdecr = 0;
2796                                         op->nfso_opencnt--;
2797                                         break;
2798                                 }
2799                         }
2800                 }
2801         }
2802
2803         /* Now process the opens against the server. */
2804         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
2805                 op = LIST_FIRST(&owp->nfsow_open);
2806                 while (op != NULL) {
2807                         nop = LIST_NEXT(op, nfso_list);
2808                         if (op->nfso_fhlen == nfhp->nfh_len &&
2809                             !NFSBCMP(op->nfso_fh, nfhp->nfh_fh,
2810                             nfhp->nfh_len)) {
2811                                 /* Found an open, decrement cnt if possible */
2812                                 if (notdecr && op->nfso_opencnt > 0) {
2813                                         notdecr = 0;
2814                                         op->nfso_opencnt--;
2815                                 }
2816                                 /*
2817                                  * There are more opens, so just return after
2818                                  * putting any opens already found back in the
2819                                  * state list.
2820                                  */
2821                                 if (op->nfso_opencnt > 0) {
2822                                         if (ohp != NULL) {
2823                                             /* Reattach open until later */
2824                                             op = LIST_FIRST(ohp);
2825                                             while (op != NULL) {
2826                                                 nop = LIST_NEXT(op, nfso_list);
2827                                                 LIST_REMOVE(op, nfso_list);
2828                                                 LIST_INSERT_HEAD(
2829                                                     &op->nfso_own->nfsow_open,
2830                                                     op, nfso_list);
2831                                                 op = nop;
2832                                             }
2833                                             LIST_INIT(ohp);
2834                                         }
2835                                         NFSUNLOCKCLSTATE();
2836                                         return (0);
2837                                 }
2838
2839                                 /*
2840                                  * Move this entry to the list of opens to be
2841                                  * returned. (If we find other open(s) still in
2842                                  * use, it will be put back in the state list
2843                                  * in the code just above.)
2844                                  */
2845                                 if (ohp != NULL) {
2846                                         LIST_REMOVE(op, nfso_list);
2847                                         LIST_INSERT_HEAD(ohp, op, nfso_list);
2848                                 }
2849                         }
2850                         op = nop;
2851                 }
2852         }
2853
2854         if (dp != NULL && ohp != NULL) {
2855                 /*
2856                  * If we are flushing all writes against the server for this
2857                  * file upon close, we do not need to keep the local opens
2858                  * (against the delegation) if they all have an opencnt == 0,
2859                  * since there are now no opens on the file and no dirty blocks.
2860                  * If the writes aren't being flushed upon close,
2861                  * a test for "no dirty blocks to write back" would have to
2862                  * be added to this code.
2863                  */
2864                 candelete = 1;
2865                 LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
2866                         op = LIST_FIRST(&owp->nfsow_open);
2867                         if (op != NULL && op->nfso_opencnt > 0) {
2868                                 candelete = 0;
2869                                 break;
2870                         }
2871                 }
2872                 if (candelete) {
2873                         LIST_FOREACH_SAFE(owp, &dp->nfsdl_owner, nfsow_list,
2874                             nowp) {
2875                                 op = LIST_FIRST(&owp->nfsow_open);
2876                                 if (op != NULL)
2877                                         nfscl_freeopen(op, 1);
2878                                 nfscl_freeopenowner(owp, 1);
2879                         }
2880                 }
2881         }
2882         NFSUNLOCKCLSTATE();
2883         if (notdecr && ohp == NULL)
2884                 printf("nfscl: never fnd open\n");
2885         return (0);
2886 }
2887
2888 /*
2889  * Return all delegations on this client.
2890  * (Must be called with client sleep lock.)
2891  */
2892 static void
2893 nfscl_delegreturnall(struct nfsclclient *clp, NFSPROC_T *p)
2894 {
2895         struct nfscldeleg *dp, *ndp;
2896         struct ucred *cred;
2897
2898         cred = newnfs_getcred();
2899         TAILQ_FOREACH_SAFE(dp, &clp->nfsc_deleg, nfsdl_list, ndp) {
2900                 nfscl_cleandeleg(dp);
2901                 (void) nfscl_trydelegreturn(dp, cred, clp->nfsc_nmp, p);
2902                 nfscl_freedeleg(&clp->nfsc_deleg, dp);
2903         }
2904         NFSFREECRED(cred);
2905 }
2906
2907 /*
2908  * Do a callback RPC.
2909  */
2910 APPLESTATIC void
2911 nfscl_docb(struct nfsrv_descript *nd, NFSPROC_T *p)
2912 {
2913         int i, op;
2914         u_int32_t *tl;
2915         struct nfsclclient *clp;
2916         struct nfscldeleg *dp = NULL;
2917         int numops, taglen = -1, error = 0, trunc, ret = 0;
2918         u_int32_t minorvers, retops = 0, *retopsp = NULL, *repp, cbident;
2919         u_char tag[NFSV4_SMALLSTR + 1], *tagstr;
2920         vnode_t vp = NULL;
2921         struct nfsnode *np;
2922         struct vattr va;
2923         struct nfsfh *nfhp;
2924         mount_t mp;
2925         nfsattrbit_t attrbits, rattrbits;
2926         nfsv4stateid_t stateid;
2927
2928         nfsrvd_rephead(nd);
2929         NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
2930         taglen = fxdr_unsigned(int, *tl);
2931         if (taglen < 0) {
2932                 error = EBADRPC;
2933                 goto nfsmout;
2934         }
2935         if (taglen <= NFSV4_SMALLSTR)
2936                 tagstr = tag;
2937         else
2938                 tagstr = malloc(taglen + 1, M_TEMP, M_WAITOK);
2939         error = nfsrv_mtostr(nd, tagstr, taglen);
2940         if (error) {
2941                 if (taglen > NFSV4_SMALLSTR)
2942                         free(tagstr, M_TEMP);
2943                 taglen = -1;
2944                 goto nfsmout;
2945         }
2946         (void) nfsm_strtom(nd, tag, taglen);
2947         if (taglen > NFSV4_SMALLSTR) {
2948                 free(tagstr, M_TEMP);
2949         }
2950         NFSM_BUILD(retopsp, u_int32_t *, NFSX_UNSIGNED);
2951         NFSM_DISSECT(tl, u_int32_t *, 3 * NFSX_UNSIGNED);
2952         minorvers = fxdr_unsigned(u_int32_t, *tl++);
2953         if (minorvers != NFSV4_MINORVERSION)
2954                 nd->nd_repstat = NFSERR_MINORVERMISMATCH;
2955         cbident = fxdr_unsigned(u_int32_t, *tl++);
2956         if (nd->nd_repstat)
2957                 numops = 0;
2958         else
2959                 numops = fxdr_unsigned(int, *tl);
2960         /*
2961          * Loop around doing the sub ops.
2962          */
2963         for (i = 0; i < numops; i++) {
2964                 NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
2965                 NFSM_BUILD(repp, u_int32_t *, 2 * NFSX_UNSIGNED);
2966                 *repp++ = *tl;
2967                 op = fxdr_unsigned(int, *tl);
2968                 if (op < NFSV4OP_CBGETATTR || op > NFSV4OP_CBRECALL) {
2969                     nd->nd_repstat = NFSERR_OPILLEGAL;
2970                     *repp = nfscl_errmap(nd);
2971                     retops++;
2972                     break;
2973                 }
2974                 nd->nd_procnum = op;
2975                 newnfsstats.cbrpccnt[nd->nd_procnum]++;
2976                 switch (op) {
2977                 case NFSV4OP_CBGETATTR:
2978                         clp = NULL;
2979                         error = nfsm_getfh(nd, &nfhp);
2980                         if (!error)
2981                                 error = nfsrv_getattrbits(nd, &attrbits,
2982                                     NULL, NULL);
2983                         if (!error) {
2984                                 mp = nfscl_getmnt(cbident);
2985                                 if (mp == NULL)
2986                                         error = NFSERR_SERVERFAULT;
2987                         }
2988                         if (!error) {
2989                                 dp = NULL;
2990                                 NFSLOCKCLSTATE();
2991                                 clp = nfscl_findcl(VFSTONFS(mp));
2992                                 if (clp != NULL)
2993                                         dp = nfscl_finddeleg(clp, nfhp->nfh_fh,
2994                                             nfhp->nfh_len);
2995                                 NFSUNLOCKCLSTATE();
2996                                 if (dp == NULL)
2997                                         error = NFSERR_SERVERFAULT;
2998                         }
2999                         if (!error) {
3000                                 ret = nfscl_ngetreopen(mp, nfhp->nfh_fh,
3001                                     nfhp->nfh_len, p, &np);
3002                                 if (!ret)
3003                                         vp = NFSTOV(np);
3004                         }
3005                         if (nfhp != NULL)
3006                                 FREE((caddr_t)nfhp, M_NFSFH);
3007                         if (!error) {
3008                                 NFSZERO_ATTRBIT(&rattrbits);
3009                                 if (NFSISSET_ATTRBIT(&attrbits,
3010                                     NFSATTRBIT_SIZE)) {
3011                                         if (!ret)
3012                                                 va.va_size = np->n_size;
3013                                         else
3014                                                 va.va_size = dp->nfsdl_size;
3015                                         NFSSETBIT_ATTRBIT(&rattrbits,
3016                                             NFSATTRBIT_SIZE);
3017                                 }
3018                                 if (NFSISSET_ATTRBIT(&attrbits,
3019                                     NFSATTRBIT_CHANGE)) {
3020                                         va.va_filerev = dp->nfsdl_change;
3021                                         if (ret || (np->n_flag & NDELEGMOD))
3022                                                 va.va_filerev++;
3023                                         NFSSETBIT_ATTRBIT(&rattrbits,
3024                                             NFSATTRBIT_CHANGE);
3025                                 }
3026                                 (void) nfsv4_fillattr(nd, NULL, NULL, &va,
3027                                     NULL, 0, &rattrbits, NULL, NULL, 0, 0);
3028                                 if (!ret)
3029                                         vrele(vp);
3030                         }
3031                         break;
3032                 case NFSV4OP_CBRECALL:
3033                         clp = NULL;
3034                         NFSM_DISSECT(tl, u_int32_t *, NFSX_STATEID +
3035                             NFSX_UNSIGNED);
3036                         stateid.seqid = *tl++;
3037                         NFSBCOPY((caddr_t)tl, (caddr_t)stateid.other,
3038                             NFSX_STATEIDOTHER);
3039                         tl += (NFSX_STATEIDOTHER / NFSX_UNSIGNED);
3040                         trunc = fxdr_unsigned(int, *tl);
3041                         error = nfsm_getfh(nd, &nfhp);
3042                         if (!error) {
3043                                 mp = nfscl_getmnt(cbident);
3044                                 if (mp == NULL)
3045                                         error = NFSERR_SERVERFAULT;
3046                         }
3047                         if (!error) {
3048                                 NFSLOCKCLSTATE();
3049                                 clp = nfscl_findcl(VFSTONFS(mp));
3050                                 if (clp != NULL) {
3051                                         dp = nfscl_finddeleg(clp, nfhp->nfh_fh,
3052                                             nfhp->nfh_len);
3053                                         if (dp != NULL) {
3054                                                 dp->nfsdl_flags |=
3055                                                     NFSCLDL_RECALL;
3056                                                 wakeup((caddr_t)clp);
3057                                         }
3058                                 } else {
3059                                         error = NFSERR_SERVERFAULT;
3060                                 }
3061                                 NFSUNLOCKCLSTATE();
3062                         }
3063                         if (nfhp != NULL)
3064                                 FREE((caddr_t)nfhp, M_NFSFH);
3065                         break;
3066                 };
3067                 if (error) {
3068                         if (error == EBADRPC || error == NFSERR_BADXDR) {
3069                                 nd->nd_repstat = NFSERR_BADXDR;
3070                         } else {
3071                                 nd->nd_repstat = error;
3072                         }
3073                         error = 0;
3074                 }
3075                 retops++;
3076                 if (nd->nd_repstat) {
3077                         *repp = nfscl_errmap(nd);
3078                         break;
3079                 } else
3080                         *repp = 0;      /* NFS4_OK */
3081         }
3082 nfsmout:
3083         if (error) {
3084                 if (error == EBADRPC || error == NFSERR_BADXDR)
3085                         nd->nd_repstat = NFSERR_BADXDR;
3086                 else
3087                         printf("nfsv4 comperr1=%d\n", error);
3088         }
3089         if (taglen == -1) {
3090                 NFSM_BUILD(tl, u_int32_t *, 2 * NFSX_UNSIGNED);
3091                 *tl++ = 0;
3092                 *tl = 0;
3093         } else {
3094                 *retopsp = txdr_unsigned(retops);
3095         }
3096         *nd->nd_errp = nfscl_errmap(nd);
3097 }
3098
3099 /*
3100  * Generate the next cbident value. Basically just increment a static value
3101  * and then check that it isn't already in the list, if it has wrapped around.
3102  */
3103 static u_int32_t
3104 nfscl_nextcbident(void)
3105 {
3106         struct nfsclclient *clp;
3107         int matched;
3108         static u_int32_t nextcbident = 0;
3109         static int haswrapped = 0;
3110
3111         nextcbident++;
3112         if (nextcbident == 0)
3113                 haswrapped = 1;
3114         if (haswrapped) {
3115                 /*
3116                  * Search the clientid list for one already using this cbident.
3117                  */
3118                 do {
3119                         matched = 0;
3120                         NFSLOCKCLSTATE();
3121                         LIST_FOREACH(clp, &nfsclhead, nfsc_list) {
3122                                 if (clp->nfsc_cbident == nextcbident) {
3123                                         matched = 1;
3124                                         break;
3125                                 }
3126                         }
3127                         NFSUNLOCKCLSTATE();
3128                         if (matched == 1)
3129                                 nextcbident++;
3130                 } while (matched);
3131         }
3132         return (nextcbident);
3133 }
3134
3135 /*
3136  * Get the mount point related to a given cbident.
3137  */
3138 static mount_t
3139 nfscl_getmnt(u_int32_t cbident)
3140 {
3141         struct nfsclclient *clp;
3142         struct nfsmount *nmp;
3143
3144         NFSLOCKCLSTATE();
3145         LIST_FOREACH(clp, &nfsclhead, nfsc_list) {
3146                 if (clp->nfsc_cbident == cbident)
3147                         break;
3148         }
3149         if (clp == NULL) {
3150                 NFSUNLOCKCLSTATE();
3151                 return (NULL);
3152         }
3153         nmp = clp->nfsc_nmp;
3154         NFSUNLOCKCLSTATE();
3155         return (nmp->nm_mountp);
3156 }
3157
3158 /*
3159  * Search for a lock conflict locally on the client. A conflict occurs if
3160  * - not same owner and overlapping byte range and at least one of them is
3161  *   a write lock or this is an unlock.
3162  */
3163 static int
3164 nfscl_localconflict(struct nfsclclient *clp, struct nfscllock *nlop,
3165     u_int8_t *own, struct nfscldeleg *dp, struct nfscllock **lopp)
3166 {
3167         struct nfsclowner *owp;
3168         struct nfsclopen *op;
3169         int ret;
3170
3171         if (dp != NULL) {
3172                 ret = nfscl_checkconflict(&dp->nfsdl_lock, nlop, own, lopp);
3173                 if (ret)
3174                         return (ret);
3175         }
3176         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
3177                 LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
3178                         ret = nfscl_checkconflict(&op->nfso_lock, nlop, own,
3179                             lopp);
3180                         if (ret)
3181                                 return (ret);
3182                 }
3183         }
3184         return (0);
3185 }
3186
3187 static int
3188 nfscl_checkconflict(struct nfscllockownerhead *lhp, struct nfscllock *nlop,
3189     u_int8_t *own, struct nfscllock **lopp)
3190 {
3191         struct nfscllockowner *lp;
3192         struct nfscllock *lop;
3193
3194         LIST_FOREACH(lp, lhp, nfsl_list) {
3195                 if (NFSBCMP(lp->nfsl_owner, own, NFSV4CL_LOCKNAMELEN)) {
3196                         LIST_FOREACH(lop, &lp->nfsl_lock, nfslo_list) {
3197                                 if (lop->nfslo_first >= nlop->nfslo_end)
3198                                         break;
3199                                 if (lop->nfslo_end <= nlop->nfslo_first)
3200                                         continue;
3201                                 if (lop->nfslo_type == F_WRLCK ||
3202                                     nlop->nfslo_type == F_WRLCK ||
3203                                     nlop->nfslo_type == F_UNLCK) {
3204                                         if (lopp != NULL)
3205                                                 *lopp = lop;
3206                                         return (NFSERR_DENIED);
3207                                 }
3208                         }
3209                 }
3210         }
3211         return (0);
3212 }
3213
3214 /*
3215  * Check for a local conflicting lock.
3216  */
3217 APPLESTATIC int
3218 nfscl_lockt(vnode_t vp, struct nfsclclient *clp, u_int64_t off,
3219     u_int64_t len, struct flock *fl, NFSPROC_T *p)
3220 {
3221         struct nfscllock *lop, nlck;
3222         struct nfscldeleg *dp;
3223         struct nfsnode *np;
3224         u_int8_t own[NFSV4CL_LOCKNAMELEN];
3225         int error;
3226
3227         nlck.nfslo_type = fl->l_type;
3228         nlck.nfslo_first = off;
3229         if (len == NFS64BITSSET) {
3230                 nlck.nfslo_end = NFS64BITSSET;
3231         } else {
3232                 nlck.nfslo_end = off + len;
3233                 if (nlck.nfslo_end <= nlck.nfslo_first)
3234                         return (NFSERR_INVAL);
3235         }
3236         np = VTONFS(vp);
3237         nfscl_filllockowner(p, own);
3238         NFSLOCKCLSTATE();
3239         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
3240         error = nfscl_localconflict(clp, &nlck, own, dp, &lop);
3241         if (error == NFSERR_DENIED)
3242                 error = EACCES;
3243         if (error) {
3244                 fl->l_whence = SEEK_SET;
3245                 fl->l_start = lop->nfslo_first;
3246                 if (lop->nfslo_end == NFS64BITSSET)
3247                         fl->l_len = 0;
3248                 else
3249                         fl->l_len = lop->nfslo_end - lop->nfslo_first;
3250                 fl->l_pid = (pid_t)0;
3251                 fl->l_type = lop->nfslo_type;
3252         } else if (dp != NULL && ((dp->nfsdl_flags & NFSCLDL_WRITE) ||
3253             fl->l_type == F_RDLCK)) {
3254                 /*
3255                  * The delegation ensures that there isn't a conflicting
3256                  * lock on the server, so return -1 to indicate an RPC
3257                  * isn't required.
3258                  */
3259                 fl->l_type = F_UNLCK;
3260                 error = -1;
3261         }
3262         NFSUNLOCKCLSTATE();
3263         return (error);
3264 }
3265
3266 /*
3267  * Handle Recall of a delegation.
3268  * The clp must be exclusive locked when this is called.
3269  */
3270 static int
3271 nfscl_recalldeleg(struct nfsclclient *clp, struct nfsmount *nmp,
3272     struct nfscldeleg *dp, vnode_t vp, struct ucred *cred, NFSPROC_T *p)
3273 {
3274         struct nfsclowner *owp, *lowp, *nowp;
3275         struct nfsclopen *op, *lop;
3276         struct nfscllockowner *lp;
3277         struct nfscllock *lckp;
3278         struct nfsnode *np;
3279         int error = 0, ret, gotvp = 0;
3280
3281         if (vp == NULL) {
3282                 /*
3283                  * First, get a vnode for the file. This is needed to do RPCs.
3284                  */
3285                 ret = nfscl_ngetreopen(nmp->nm_mountp, dp->nfsdl_fh,
3286                     dp->nfsdl_fhlen, p, &np);
3287                 if (ret) {
3288                         /*
3289                          * File isn't open, so nothing to move over to the
3290                          * server.
3291                          */
3292                         return (0);
3293                 }
3294                 vp = NFSTOV(np);
3295                 gotvp = 1;
3296         } else {
3297                 np = VTONFS(vp);
3298         }
3299         dp->nfsdl_flags &= ~NFSCLDL_MODTIMESET;
3300         NFSINVALATTRCACHE(np);
3301
3302         /*
3303          * Ok, if it's a write delegation, flush data to the server, so
3304          * that close/open consistency is retained.
3305          */
3306         NFSLOCKNODE(np);
3307         if ((dp->nfsdl_flags & NFSCLDL_WRITE) && (np->n_flag & NMODIFIED)) {
3308 #ifdef APPLE
3309                 OSBitOrAtomic((u_int32_t)NDELEGRECALL, (UInt32 *)&np->n_flag);
3310 #else
3311                 np->n_flag |= NDELEGRECALL;
3312 #endif
3313                 NFSUNLOCKNODE(np);
3314                 (void) ncl_flush(vp, MNT_WAIT, cred, p, 1);
3315                 NFSLOCKNODE(np);
3316 #ifdef APPLE
3317                 OSBitAndAtomic((int32_t)~(NMODIFIED | NDELEGRECALL), (UInt32 *)&np->n_flag);
3318 #else
3319                 np->n_flag &= ~(NMODIFIED | NDELEGRECALL);
3320 #endif
3321         }
3322         NFSUNLOCKNODE(np);
3323
3324         /*
3325          * Now, for each openowner with opens issued locally, move them
3326          * over to state against the server.
3327          */
3328         LIST_FOREACH(lowp, &dp->nfsdl_owner, nfsow_list) {
3329                 lop = LIST_FIRST(&lowp->nfsow_open);
3330                 if (lop != NULL) {
3331                         if (LIST_NEXT(lop, nfso_list) != NULL)
3332                                 panic("nfsdlg mult opens");
3333                         /*
3334                          * Look for the same openowner against the server.
3335                          */
3336                         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
3337                                 if (!NFSBCMP(lowp->nfsow_owner,
3338                                     owp->nfsow_owner, NFSV4CL_LOCKNAMELEN)) {
3339                                         newnfs_copycred(&dp->nfsdl_cred, cred);
3340                                         ret = nfscl_moveopen(vp, clp, nmp, lop,
3341                                             owp, dp, cred, p);
3342                                         if (ret == NFSERR_STALECLIENTID ||
3343                                             ret == NFSERR_STALEDONTRECOVER) {
3344                                                 if (gotvp)
3345                                                         vrele(vp);
3346                                                 return (ret);
3347                                         }
3348                                         if (ret) {
3349                                                 nfscl_freeopen(lop, 1);
3350                                                 if (!error)
3351                                                         error = ret;
3352                                         }
3353                                         break;
3354                                 }
3355                         }
3356
3357                         /*
3358                          * If no openowner found, create one and get an open
3359                          * for it.
3360                          */
3361                         if (owp == NULL) {
3362                                 MALLOC(nowp, struct nfsclowner *,
3363                                     sizeof (struct nfsclowner), M_NFSCLOWNER,
3364                                     M_WAITOK);
3365                                 nfscl_newopen(clp, NULL, &owp, &nowp, &op,
3366                                     NULL, lowp->nfsow_owner, dp->nfsdl_fh,
3367                                     dp->nfsdl_fhlen, NULL);
3368                                 newnfs_copycred(&dp->nfsdl_cred, cred);
3369                                 ret = nfscl_moveopen(vp, clp, nmp, lop,
3370                                     owp, dp, cred, p);
3371                                 if (ret) {
3372                                         nfscl_freeopenowner(owp, 0);
3373                                         if (ret == NFSERR_STALECLIENTID ||
3374                                             ret == NFSERR_STALEDONTRECOVER) {
3375                                                 if (gotvp)
3376                                                         vrele(vp);
3377                                                 return (ret);
3378                                         }
3379                                         if (ret) {
3380                                                 nfscl_freeopen(lop, 1);
3381                                                 if (!error)
3382                                                         error = ret;
3383                                         }
3384                                 }
3385                         }
3386                 }
3387         }
3388
3389         /*
3390          * Now, get byte range locks for any locks done locally.
3391          */
3392         LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
3393                 LIST_FOREACH(lckp, &lp->nfsl_lock, nfslo_list) {
3394                         newnfs_copycred(&dp->nfsdl_cred, cred);
3395                         ret = nfscl_relock(vp, clp, nmp, lp, lckp, cred, p);
3396                         if (ret == NFSERR_STALESTATEID ||
3397                             ret == NFSERR_STALEDONTRECOVER ||
3398                             ret == NFSERR_STALECLIENTID) {
3399                                 if (gotvp)
3400                                         vrele(vp);
3401                                 return (ret);
3402                         }
3403                         if (ret && !error)
3404                                 error = ret;
3405                 }
3406         }
3407         if (gotvp)
3408                 vrele(vp);
3409         return (error);
3410 }
3411
3412 /*
3413  * Move a locally issued open over to an owner on the state list.
3414  * SIDE EFFECT: If it needs to sleep (do an rpc), it unlocks clstate and
3415  * returns with it unlocked.
3416  */
3417 static int
3418 nfscl_moveopen(vnode_t vp, struct nfsclclient *clp, struct nfsmount *nmp,
3419     struct nfsclopen *lop, struct nfsclowner *owp, struct nfscldeleg *dp,
3420     struct ucred *cred, NFSPROC_T *p)
3421 {
3422         struct nfsclopen *op, *nop;
3423         struct nfscldeleg *ndp;
3424         struct nfsnode *np;
3425         int error = 0, newone;
3426
3427         /*
3428          * First, look for an appropriate open, If found, just increment the
3429          * opencnt in it.
3430          */
3431         LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
3432                 if ((op->nfso_mode & lop->nfso_mode) == lop->nfso_mode &&
3433                     op->nfso_fhlen == lop->nfso_fhlen &&
3434                     !NFSBCMP(op->nfso_fh, lop->nfso_fh, op->nfso_fhlen)) {
3435                         op->nfso_opencnt += lop->nfso_opencnt;
3436                         nfscl_freeopen(lop, 1);
3437                         return (0);
3438                 }
3439         }
3440
3441         /* No appropriate open, so we have to do one against the server. */
3442         np = VTONFS(vp);
3443         MALLOC(nop, struct nfsclopen *, sizeof (struct nfsclopen) +
3444             lop->nfso_fhlen - 1, M_NFSCLOPEN, M_WAITOK);
3445         newone = 0;
3446         nfscl_newopen(clp, NULL, &owp, NULL, &op, &nop, owp->nfsow_owner,
3447             lop->nfso_fh, lop->nfso_fhlen, &newone);
3448         ndp = dp;
3449         error = nfscl_tryopen(nmp, vp, np->n_v4->n4_data, np->n_v4->n4_fhlen,
3450             lop->nfso_fh, lop->nfso_fhlen, lop->nfso_mode, op,
3451             NFS4NODENAME(np->n_v4), np->n_v4->n4_namelen, &ndp, 0, 0, cred, p);
3452         if (error) {
3453                 if (newone)
3454                         nfscl_freeopen(op, 0);
3455         } else {
3456                 if (newone)
3457                         newnfs_copyincred(cred, &op->nfso_cred);
3458                 op->nfso_mode |= lop->nfso_mode;
3459                 op->nfso_opencnt += lop->nfso_opencnt;
3460                 nfscl_freeopen(lop, 1);
3461         }
3462         if (nop != NULL)
3463                 FREE((caddr_t)nop, M_NFSCLOPEN);
3464         if (ndp != NULL) {
3465                 /*
3466                  * What should I do with the returned delegation, since the
3467                  * delegation is being recalled? For now, just printf and
3468                  * through it away.
3469                  */
3470                 printf("Moveopen returned deleg\n");
3471                 FREE((caddr_t)ndp, M_NFSCLDELEG);
3472         }
3473         return (error);
3474 }
3475
3476 /*
3477  * Recall all delegations on this client.
3478  */
3479 static void
3480 nfscl_totalrecall(struct nfsclclient *clp)
3481 {
3482         struct nfscldeleg *dp;
3483
3484         TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list)
3485                 dp->nfsdl_flags |= NFSCLDL_RECALL;
3486 }
3487
3488 /*
3489  * Relock byte ranges. Called for delegation recall and state expiry.
3490  */
3491 static int
3492 nfscl_relock(vnode_t vp, struct nfsclclient *clp, struct nfsmount *nmp,
3493     struct nfscllockowner *lp, struct nfscllock *lop, struct ucred *cred,
3494     NFSPROC_T *p)
3495 {
3496         struct nfscllockowner *nlp;
3497         struct nfsfh *nfhp;
3498         u_int64_t off, len;
3499         u_int32_t clidrev = 0;
3500         int error, newone, donelocally;
3501
3502         off = lop->nfslo_first;
3503         len = lop->nfslo_end - lop->nfslo_first;
3504         error = nfscl_getbytelock(vp, off, len, lop->nfslo_type, cred, p,
3505             clp, 1, lp->nfsl_owner, lp->nfsl_openowner, &nlp, &newone,
3506             &donelocally);
3507         if (error || donelocally)
3508                 return (error);
3509         if (nmp->nm_clp != NULL)
3510                 clidrev = nmp->nm_clp->nfsc_clientidrev;
3511         else
3512                 clidrev = 0;
3513         nfhp = VTONFS(vp)->n_fhp;
3514         error = nfscl_trylock(nmp, vp, nfhp->nfh_fh,
3515             nfhp->nfh_len, nlp, newone, 0, off,
3516             len, lop->nfslo_type, cred, p);
3517         if (error)
3518                 nfscl_freelockowner(nlp, 0);
3519         return (error);
3520 }
3521
3522 /*
3523  * Called to re-open a file. Basically get a vnode for the file handle
3524  * and then call nfsrpc_openrpc() to do the rest.
3525  */
3526 static int
3527 nfsrpc_reopen(struct nfsmount *nmp, u_int8_t *fhp, int fhlen,
3528     u_int32_t mode, struct nfsclopen *op, struct nfscldeleg **dpp,
3529     struct ucred *cred, NFSPROC_T *p)
3530 {
3531         struct nfsnode *np;
3532         vnode_t vp;
3533         int error;
3534
3535         error = nfscl_ngetreopen(nmp->nm_mountp, fhp, fhlen, p, &np);
3536         if (error)
3537                 return (error);
3538         vp = NFSTOV(np);
3539         if (np->n_v4 != NULL) {
3540                 error = nfscl_tryopen(nmp, vp, np->n_v4->n4_data,
3541                     np->n_v4->n4_fhlen, fhp, fhlen, mode, op,
3542                     NFS4NODENAME(np->n_v4), np->n_v4->n4_namelen, dpp, 0, 0,
3543                     cred, p);
3544         } else {
3545                 error = EINVAL;
3546         }
3547         vrele(vp);
3548         return (error);
3549 }
3550
3551 /*
3552  * Try an open against the server. Just call nfsrpc_openrpc(), retrying while
3553  * NFSERR_DELAY. Also, try system credentials, if the passed in credentials
3554  * fail.
3555  */
3556 static int
3557 nfscl_tryopen(struct nfsmount *nmp, vnode_t vp, u_int8_t *fhp, int fhlen,
3558     u_int8_t *newfhp, int newfhlen, u_int32_t mode, struct nfsclopen *op,
3559     u_int8_t *name, int namelen, struct nfscldeleg **ndpp,
3560     int reclaim, u_int32_t delegtype, struct ucred *cred, NFSPROC_T *p)
3561 {
3562         int error;
3563
3564         do {
3565                 error = nfsrpc_openrpc(nmp, vp, fhp, fhlen, newfhp, newfhlen,
3566                     mode, op, name, namelen, ndpp, reclaim, delegtype, cred, p,
3567                     0, 0);
3568                 if (error == NFSERR_DELAY)
3569                         (void) nfs_catnap(PZERO, "nfstryop");
3570         } while (error == NFSERR_DELAY);
3571         if (error == EAUTH || error == EACCES) {
3572                 /* Try again using system credentials */
3573                 newnfs_setroot(cred);
3574                 do {
3575                     error = nfsrpc_openrpc(nmp, vp, fhp, fhlen, newfhp,
3576                         newfhlen, mode, op, name, namelen, ndpp, reclaim,
3577                         delegtype, cred, p, 1, 0);
3578                     if (error == NFSERR_DELAY)
3579                         (void) nfs_catnap(PZERO, "nfstryop");
3580                 } while (error == NFSERR_DELAY);
3581         }
3582         return (error);
3583 }
3584
3585 /*
3586  * Try a byte range lock. Just loop on nfsrpc_lock() while it returns
3587  * NFSERR_DELAY. Also, retry with system credentials, if the provided
3588  * cred don't work.
3589  */
3590 static int
3591 nfscl_trylock(struct nfsmount *nmp, vnode_t vp, u_int8_t *fhp,
3592     int fhlen, struct nfscllockowner *nlp, int newone, int reclaim,
3593     u_int64_t off, u_int64_t len, short type, struct ucred *cred, NFSPROC_T *p)
3594 {
3595         struct nfsrv_descript nfsd, *nd = &nfsd;
3596         int error;
3597
3598         do {
3599                 error = nfsrpc_lock(nd, nmp, vp, fhp, fhlen, nlp, newone,
3600                     reclaim, off, len, type, cred, p, 0);
3601                 if (!error && nd->nd_repstat == NFSERR_DELAY)
3602                         (void) nfs_catnap(PZERO, "nfstrylck");
3603         } while (!error && nd->nd_repstat == NFSERR_DELAY);
3604         if (!error)
3605                 error = nd->nd_repstat;
3606         if (error == EAUTH || error == EACCES) {
3607                 /* Try again using root credentials */
3608                 newnfs_setroot(cred);
3609                 do {
3610                         error = nfsrpc_lock(nd, nmp, vp, fhp, fhlen, nlp,
3611                             newone, reclaim, off, len, type, cred, p, 1);
3612                         if (!error && nd->nd_repstat == NFSERR_DELAY)
3613                                 (void) nfs_catnap(PZERO, "nfstrylck");
3614                 } while (!error && nd->nd_repstat == NFSERR_DELAY);
3615                 if (!error)
3616                         error = nd->nd_repstat;
3617         }
3618         return (error);
3619 }
3620
3621 /*
3622  * Try a delegreturn against the server. Just call nfsrpc_delegreturn(),
3623  * retrying while NFSERR_DELAY. Also, try system credentials, if the passed in
3624  * credentials fail.
3625  */
3626 static int
3627 nfscl_trydelegreturn(struct nfscldeleg *dp, struct ucred *cred,
3628     struct nfsmount *nmp, NFSPROC_T *p)
3629 {
3630         int error;
3631
3632         do {
3633                 error = nfsrpc_delegreturn(dp, cred, nmp, p, 0);
3634                 if (error == NFSERR_DELAY)
3635                         (void) nfs_catnap(PZERO, "nfstrydp");
3636         } while (error == NFSERR_DELAY);
3637         if (error == EAUTH || error == EACCES) {
3638                 /* Try again using system credentials */
3639                 newnfs_setroot(cred);
3640                 do {
3641                         error = nfsrpc_delegreturn(dp, cred, nmp, p, 1);
3642                         if (error == NFSERR_DELAY)
3643                                 (void) nfs_catnap(PZERO, "nfstrydp");
3644                 } while (error == NFSERR_DELAY);
3645         }
3646         return (error);
3647 }
3648
3649 /*
3650  * Try a close against the server. Just call nfsrpc_closerpc(),
3651  * retrying while NFSERR_DELAY. Also, try system credentials, if the passed in
3652  * credentials fail.
3653  */
3654 APPLESTATIC int
3655 nfscl_tryclose(struct nfsclopen *op, struct ucred *cred,
3656     struct nfsmount *nmp, NFSPROC_T *p)
3657 {
3658         struct nfsrv_descript nfsd, *nd = &nfsd;
3659         int error;
3660
3661         do {
3662                 error = nfsrpc_closerpc(nd, nmp, op, cred, p, 0);
3663                 if (error == NFSERR_DELAY)
3664                         (void) nfs_catnap(PZERO, "nfstrycl");
3665         } while (error == NFSERR_DELAY);
3666         if (error == EAUTH || error == EACCES) {
3667                 /* Try again using system credentials */
3668                 newnfs_setroot(cred);
3669                 do {
3670                         error = nfsrpc_closerpc(nd, nmp, op, cred, p, 1);
3671                         if (error == NFSERR_DELAY)
3672                                 (void) nfs_catnap(PZERO, "nfstrycl");
3673                 } while (error == NFSERR_DELAY);
3674         }
3675         return (error);
3676 }
3677
3678 /*
3679  * Decide if a delegation on a file permits close without flushing writes
3680  * to the server. This might be a big performance win in some environments.
3681  * (Not useful until the client does caching on local stable storage.)
3682  */
3683 APPLESTATIC int
3684 nfscl_mustflush(vnode_t vp)
3685 {
3686         struct nfsclclient *clp;
3687         struct nfscldeleg *dp;
3688         struct nfsnode *np;
3689         struct nfsmount *nmp;
3690
3691         np = VTONFS(vp);
3692         nmp = VFSTONFS(vnode_mount(vp));
3693         if (!NFSHASNFSV4(nmp))
3694                 return (1);
3695         NFSLOCKCLSTATE();
3696         clp = nfscl_findcl(nmp);
3697         if (clp == NULL) {
3698                 NFSUNLOCKCLSTATE();
3699                 return (1);
3700         }
3701         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
3702         if (dp != NULL && (dp->nfsdl_flags & (NFSCLDL_WRITE | NFSCLDL_RECALL))
3703              == NFSCLDL_WRITE &&
3704             (dp->nfsdl_sizelimit >= np->n_size ||
3705              !NFSHASSTRICT3530(nmp))) {
3706                 NFSUNLOCKCLSTATE();
3707                 return (0);
3708         }
3709         NFSUNLOCKCLSTATE();
3710         return (1);
3711 }
3712
3713 /*
3714  * See if a (write) delegation exists for this file.
3715  */
3716 APPLESTATIC int
3717 nfscl_nodeleg(vnode_t vp, int writedeleg)
3718 {
3719         struct nfsclclient *clp;
3720         struct nfscldeleg *dp;
3721         struct nfsnode *np;
3722         struct nfsmount *nmp;
3723
3724         np = VTONFS(vp);
3725         nmp = VFSTONFS(vnode_mount(vp));
3726         if (!NFSHASNFSV4(nmp))
3727                 return (1);
3728         NFSLOCKCLSTATE();
3729         clp = nfscl_findcl(nmp);
3730         if (clp == NULL) {
3731                 NFSUNLOCKCLSTATE();
3732                 return (1);
3733         }
3734         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
3735         if (dp != NULL && (dp->nfsdl_flags & NFSCLDL_RECALL) == 0 &&
3736             (writedeleg == 0 || (dp->nfsdl_flags & NFSCLDL_WRITE)
3737              == NFSCLDL_WRITE)) {
3738                 NFSUNLOCKCLSTATE();
3739                 return (0);
3740         }
3741         NFSUNLOCKCLSTATE();
3742         return (1);
3743 }
3744
3745 /*
3746  * Look for an associated delegation that should be DelegReturned.
3747  */
3748 APPLESTATIC int
3749 nfscl_removedeleg(vnode_t vp, NFSPROC_T *p, nfsv4stateid_t *stp)
3750 {
3751         struct nfsclclient *clp;
3752         struct nfscldeleg *dp;
3753         struct nfsclowner *owp;
3754         struct nfscllockowner *lp;
3755         struct nfsmount *nmp;
3756         struct ucred *cred;
3757         struct nfsnode *np;
3758         int igotlock = 0, triedrecall = 0, needsrecall, retcnt = 0, islept;
3759
3760         nmp = VFSTONFS(vnode_mount(vp));
3761         np = VTONFS(vp);
3762         NFSLOCKCLSTATE();
3763         /*
3764          * Loop around waiting for:
3765          * - outstanding I/O operations on delegations to complete
3766          * - for a delegation on vp that has state, lock the client and
3767          *   do a recall
3768          * - return delegation with no state
3769          */
3770         while (1) {
3771                 clp = nfscl_findcl(nmp);
3772                 if (clp == NULL) {
3773                         NFSUNLOCKCLSTATE();
3774                         return (retcnt);
3775                 }
3776                 dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
3777                     np->n_fhp->nfh_len);
3778                 if (dp != NULL) {
3779                     /*
3780                      * Wait for outstanding I/O ops to be done.
3781                      */
3782                     if (dp->nfsdl_rwlock.nfslock_usecnt > 0) {
3783                         if (igotlock) {
3784                             nfsv4_unlock(&clp->nfsc_lock, 0);
3785                             igotlock = 0;
3786                         }
3787                         dp->nfsdl_rwlock.nfslock_lock |= NFSV4LOCK_WANTED;
3788                         (void) nfsmsleep(&dp->nfsdl_rwlock,
3789                             NFSCLSTATEMUTEXPTR, PZERO, "nfscld", NULL);
3790                         continue;
3791                     }
3792                     needsrecall = 0;
3793                     LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
3794                         if (!LIST_EMPTY(&owp->nfsow_open)) {
3795                             needsrecall = 1;
3796                             break;
3797                         }
3798                     }
3799                     if (!needsrecall) {
3800                         LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
3801                             if (!LIST_EMPTY(&lp->nfsl_lock)) {
3802                                 needsrecall = 1;
3803                                 break;
3804                             }
3805                         }
3806                     }
3807                     if (needsrecall && !triedrecall) {
3808                         islept = 0;
3809                         while (!igotlock) {
3810                             igotlock = nfsv4_lock(&clp->nfsc_lock, 1,
3811                                 &islept, NFSCLSTATEMUTEXPTR);
3812                             if (islept)
3813                                 break;
3814                         }
3815                         if (islept)
3816                             continue;
3817                         NFSUNLOCKCLSTATE();
3818                         cred = newnfs_getcred();
3819                         newnfs_copycred(&dp->nfsdl_cred, cred);
3820                         (void) nfscl_recalldeleg(clp, nmp, dp, vp, cred, p);
3821                         NFSFREECRED(cred);
3822                         triedrecall = 1;
3823                         NFSLOCKCLSTATE();
3824                         nfsv4_unlock(&clp->nfsc_lock, 0);
3825                         igotlock = 0;
3826                         continue;
3827                     }
3828                     *stp = dp->nfsdl_stateid;
3829                     retcnt = 1;
3830                     nfscl_cleandeleg(dp);
3831                     nfscl_freedeleg(&clp->nfsc_deleg, dp);
3832                 }
3833                 if (igotlock)
3834                     nfsv4_unlock(&clp->nfsc_lock, 0);
3835                 NFSUNLOCKCLSTATE();
3836                 return (retcnt);
3837         }
3838 }
3839
3840 /*
3841  * Look for associated delegation(s) that should be DelegReturned.
3842  */
3843 APPLESTATIC int
3844 nfscl_renamedeleg(vnode_t fvp, nfsv4stateid_t *fstp, int *gotfdp, vnode_t tvp,
3845     nfsv4stateid_t *tstp, int *gottdp, NFSPROC_T *p)
3846 {
3847         struct nfsclclient *clp;
3848         struct nfscldeleg *dp;
3849         struct nfsclowner *owp;
3850         struct nfscllockowner *lp;
3851         struct nfsmount *nmp;
3852         struct ucred *cred;
3853         struct nfsnode *np;
3854         int igotlock = 0, triedrecall = 0, needsrecall, retcnt = 0, islept;
3855
3856         nmp = VFSTONFS(vnode_mount(fvp));
3857         *gotfdp = 0;
3858         *gottdp = 0;
3859         NFSLOCKCLSTATE();
3860         /*
3861          * Loop around waiting for:
3862          * - outstanding I/O operations on delegations to complete
3863          * - for a delegation on fvp that has state, lock the client and
3864          *   do a recall
3865          * - return delegation(s) with no state.
3866          */
3867         while (1) {
3868                 clp = nfscl_findcl(nmp);
3869                 if (clp == NULL) {
3870                         NFSUNLOCKCLSTATE();
3871                         return (retcnt);
3872                 }
3873                 np = VTONFS(fvp);
3874                 dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
3875                     np->n_fhp->nfh_len);
3876                 if (dp != NULL && *gotfdp == 0) {
3877                     /*
3878                      * Wait for outstanding I/O ops to be done.
3879                      */
3880                     if (dp->nfsdl_rwlock.nfslock_usecnt > 0) {
3881                         if (igotlock) {
3882                             nfsv4_unlock(&clp->nfsc_lock, 0);
3883                             igotlock = 0;
3884                         }
3885                         dp->nfsdl_rwlock.nfslock_lock |= NFSV4LOCK_WANTED;
3886                         (void) nfsmsleep(&dp->nfsdl_rwlock,
3887                             NFSCLSTATEMUTEXPTR, PZERO, "nfscld", NULL);
3888                         continue;
3889                     }
3890                     needsrecall = 0;
3891                     LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
3892                         if (!LIST_EMPTY(&owp->nfsow_open)) {
3893                             needsrecall = 1;
3894                             break;
3895                         }
3896                     }
3897                     if (!needsrecall) {
3898                         LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
3899                             if (!LIST_EMPTY(&lp->nfsl_lock)) {
3900                                 needsrecall = 1;
3901                                 break;
3902                             }
3903                         }
3904                     }
3905                     if (needsrecall && !triedrecall) {
3906                         islept = 0;
3907                         while (!igotlock) {
3908                             igotlock = nfsv4_lock(&clp->nfsc_lock, 1,
3909                                 &islept, NFSCLSTATEMUTEXPTR);
3910                             if (islept)
3911                                 break;
3912                         }
3913                         if (islept)
3914                             continue;
3915                         NFSUNLOCKCLSTATE();
3916                         cred = newnfs_getcred();
3917                         newnfs_copycred(&dp->nfsdl_cred, cred);
3918                         (void) nfscl_recalldeleg(clp, nmp, dp, fvp, cred, p);
3919                         NFSFREECRED(cred);
3920                         triedrecall = 1;
3921                         NFSLOCKCLSTATE();
3922                         nfsv4_unlock(&clp->nfsc_lock, 0);
3923                         igotlock = 0;
3924                         continue;
3925                     }
3926                     *fstp = dp->nfsdl_stateid;
3927                     retcnt++;
3928                     *gotfdp = 1;
3929                     nfscl_cleandeleg(dp);
3930                     nfscl_freedeleg(&clp->nfsc_deleg, dp);
3931                 }
3932                 if (igotlock) {
3933                     nfsv4_unlock(&clp->nfsc_lock, 0);
3934                     igotlock = 0;
3935                 }
3936                 if (tvp != NULL) {
3937                     np = VTONFS(tvp);
3938                     dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
3939                         np->n_fhp->nfh_len);
3940                     if (dp != NULL && *gottdp == 0) {
3941                         /*
3942                          * Wait for outstanding I/O ops to be done.
3943                          */
3944                         if (dp->nfsdl_rwlock.nfslock_usecnt > 0) {
3945                             dp->nfsdl_rwlock.nfslock_lock |= NFSV4LOCK_WANTED;
3946                             (void) nfsmsleep(&dp->nfsdl_rwlock,
3947                                 NFSCLSTATEMUTEXPTR, PZERO, "nfscld", NULL);
3948                             continue;
3949                         }
3950                         LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
3951                             if (!LIST_EMPTY(&owp->nfsow_open)) {
3952                                 NFSUNLOCKCLSTATE();
3953                                 return (retcnt);
3954                             }
3955                         }
3956                         LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
3957                             if (!LIST_EMPTY(&lp->nfsl_lock)) {
3958                                 NFSUNLOCKCLSTATE();
3959                                 return (retcnt);
3960                             }
3961                         }
3962                         *tstp = dp->nfsdl_stateid;
3963                         retcnt++;
3964                         *gottdp = 1;
3965                         nfscl_cleandeleg(dp);
3966                         nfscl_freedeleg(&clp->nfsc_deleg, dp);
3967                     }
3968                 }
3969                 NFSUNLOCKCLSTATE();
3970                 return (retcnt);
3971         }
3972 }
3973
3974 /*
3975  * Get a reference on the clientid associated with the mount point.
3976  * Return 1 if success, 0 otherwise.
3977  */
3978 APPLESTATIC int
3979 nfscl_getref(struct nfsmount *nmp)
3980 {
3981         struct nfsclclient *clp;
3982
3983         NFSLOCKCLSTATE();
3984         clp = nfscl_findcl(nmp);
3985         if (clp == NULL) {
3986                 NFSUNLOCKCLSTATE();
3987                 return (0);
3988         }
3989         nfsv4_getref(&clp->nfsc_lock, NULL, NFSCLSTATEMUTEXPTR);
3990         NFSUNLOCKCLSTATE();
3991         return (1);
3992 }
3993
3994 /*
3995  * Release a reference on a clientid acquired with the above call.
3996  */
3997 APPLESTATIC void
3998 nfscl_relref(struct nfsmount *nmp)
3999 {
4000         struct nfsclclient *clp;
4001
4002         NFSLOCKCLSTATE();
4003         clp = nfscl_findcl(nmp);
4004         if (clp == NULL) {
4005                 NFSUNLOCKCLSTATE();
4006                 return;
4007         }
4008         nfsv4_relref(&clp->nfsc_lock);
4009         NFSUNLOCKCLSTATE();
4010 }
4011
4012 /*
4013  * Save the size attribute in the delegation, since the nfsnode
4014  * is going away.
4015  */
4016 APPLESTATIC void
4017 nfscl_reclaimnode(vnode_t vp)
4018 {
4019         struct nfsclclient *clp;
4020         struct nfscldeleg *dp;
4021         struct nfsnode *np = VTONFS(vp);
4022         struct nfsmount *nmp;
4023
4024         nmp = VFSTONFS(vnode_mount(vp));
4025         if (!NFSHASNFSV4(nmp))
4026                 return;
4027         NFSLOCKCLSTATE();
4028         clp = nfscl_findcl(nmp);
4029         if (clp == NULL) {
4030                 NFSUNLOCKCLSTATE();
4031                 return;
4032         }
4033         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
4034         if (dp != NULL && (dp->nfsdl_flags & NFSCLDL_WRITE))
4035                 dp->nfsdl_size = np->n_size;
4036         NFSUNLOCKCLSTATE();
4037 }
4038
4039 /*
4040  * Get the saved size attribute in the delegation, since it is a
4041  * newly allocated nfsnode.
4042  */
4043 APPLESTATIC void
4044 nfscl_newnode(vnode_t vp)
4045 {
4046         struct nfsclclient *clp;
4047         struct nfscldeleg *dp;
4048         struct nfsnode *np = VTONFS(vp);
4049         struct nfsmount *nmp;
4050
4051         nmp = VFSTONFS(vnode_mount(vp));
4052         if (!NFSHASNFSV4(nmp))
4053                 return;
4054         NFSLOCKCLSTATE();
4055         clp = nfscl_findcl(nmp);
4056         if (clp == NULL) {
4057                 NFSUNLOCKCLSTATE();
4058                 return;
4059         }
4060         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
4061         if (dp != NULL && (dp->nfsdl_flags & NFSCLDL_WRITE))
4062                 np->n_size = dp->nfsdl_size;
4063         NFSUNLOCKCLSTATE();
4064 }
4065
4066 /*
4067  * If there is a valid write delegation for this file, set the modtime
4068  * to the local clock time.
4069  */
4070 APPLESTATIC void
4071 nfscl_delegmodtime(vnode_t vp)
4072 {
4073         struct nfsclclient *clp;
4074         struct nfscldeleg *dp;
4075         struct nfsnode *np = VTONFS(vp);
4076         struct nfsmount *nmp;
4077
4078         nmp = VFSTONFS(vnode_mount(vp));
4079         if (!NFSHASNFSV4(nmp))
4080                 return;
4081         NFSLOCKCLSTATE();
4082         clp = nfscl_findcl(nmp);
4083         if (clp == NULL) {
4084                 NFSUNLOCKCLSTATE();
4085                 return;
4086         }
4087         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
4088         if (dp != NULL && (dp->nfsdl_flags & NFSCLDL_WRITE)) {
4089                 NFSGETNANOTIME(&dp->nfsdl_modtime);
4090                 dp->nfsdl_flags |= NFSCLDL_MODTIMESET;
4091         }
4092         NFSUNLOCKCLSTATE();
4093 }
4094
4095 /*
4096  * If there is a valid write delegation for this file with a modtime set,
4097  * put that modtime in mtime.
4098  */
4099 APPLESTATIC void
4100 nfscl_deleggetmodtime(vnode_t vp, struct timespec *mtime)
4101 {
4102         struct nfsclclient *clp;
4103         struct nfscldeleg *dp;
4104         struct nfsnode *np = VTONFS(vp);
4105         struct nfsmount *nmp;
4106
4107         nmp = VFSTONFS(vnode_mount(vp));
4108         if (!NFSHASNFSV4(nmp))
4109                 return;
4110         NFSLOCKCLSTATE();
4111         clp = nfscl_findcl(nmp);
4112         if (clp == NULL) {
4113                 NFSUNLOCKCLSTATE();
4114                 return;
4115         }
4116         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
4117         if (dp != NULL &&
4118             (dp->nfsdl_flags & (NFSCLDL_WRITE | NFSCLDL_MODTIMESET)) ==
4119             (NFSCLDL_WRITE | NFSCLDL_MODTIMESET))
4120                 *mtime = dp->nfsdl_modtime;
4121         NFSUNLOCKCLSTATE();
4122 }
4123
4124 static int
4125 nfscl_errmap(struct nfsrv_descript *nd)
4126 {
4127         short *defaulterrp, *errp;
4128
4129         if (!nd->nd_repstat)
4130                 return (0);
4131         if (nd->nd_procnum == NFSPROC_NOOP)
4132                 return (txdr_unsigned(nd->nd_repstat & 0xffff));
4133         if (nd->nd_repstat == EBADRPC)
4134                 return (txdr_unsigned(NFSERR_BADXDR));
4135         if (nd->nd_repstat == NFSERR_MINORVERMISMATCH ||
4136             nd->nd_repstat == NFSERR_OPILLEGAL)
4137                 return (txdr_unsigned(nd->nd_repstat));
4138         errp = defaulterrp = nfscl_cberrmap[nd->nd_procnum];
4139         while (*++errp)
4140                 if (*errp == (short)nd->nd_repstat)
4141                         return (txdr_unsigned(nd->nd_repstat));
4142         return (txdr_unsigned(*defaulterrp));
4143 }
4144