sys/geom/geom_fox.c

   1 /*-
   2  * Copyright (c) 2003 Poul-Henning Kamp
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. The names of the authors may not be used to endorse or promote
  14  *    products derived from this software without specific prior written
  15  *    permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 /* This is a GEOM module for handling path selection for multi-path
  33  * storage devices.  It is named "fox" because it, like they, prefer
  34  * to have multiple exits to choose from.
  35  *
  36  */
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/kernel.h>
  41 #include <sys/conf.h>
  42 #include <sys/bio.h>
  43 #include <sys/malloc.h>
  44 #include <sys/lock.h>
  45 #include <sys/mutex.h>
  46 #include <sys/libkern.h>
  47 #include <sys/endian.h>
  48 #include <sys/md5.h>
  49 #include <sys/errno.h>
  50 #include <geom/geom.h>
  51
  52 #define FOX_CLASS_NAME "FOX"
  53 #define FOX_MAGIC       "GEOM::FOX"
  54
  55 struct g_fox_softc {
  56         off_t                   mediasize;
  57         u_int                   sectorsize;
  58         TAILQ_HEAD(, bio)       queue;
  59         struct mtx              lock;
  60         u_char                  magic[16];
  61         struct g_consumer       *path;
  62         struct g_consumer       *opath;
  63         int                     waiting;
  64         int                     cr, cw, ce;
  65 };
  66
  67 /*
  68  * This function is called whenever we need to select a new path.
  69  */
  70 static void
  71 g_fox_select_path(void *arg, int flag)
  72 {
  73         struct g_geom *gp;
  74         struct g_fox_softc *sc;
  75         struct g_consumer *cp1;
  76         struct bio *bp;
  77         int error;
  78
  79         g_topology_assert();
  80         if (flag == EV_CANCEL)
  81                 return;
  82         gp = arg;
  83         sc = gp->softc;
  84
  85         if (sc->opath != NULL) {
  86                 /*
  87                  * First, close the old path entirely.
  88                  */
  89                 printf("Closing old path (%s) on fox (%s)\n",
  90                         sc->opath->provider->name, gp->name);
  91
  92                 cp1 = LIST_NEXT(sc->opath, consumer);
  93
  94                 g_access(sc->opath, -sc->cr, -sc->cw, -(sc->ce + 1));
  95
  96                 /*
  97                  * The attempt to reopen it with a exclusive count
  98                  */
  99                 error = g_access(sc->opath, 0, 0, 1);
 100                 if (error) {
 101                         /*
 102                          * Ok, ditch this consumer, we can't use it.
 103                          */
 104                         printf("Drop old path (%s) on fox (%s)\n",
 105                                 sc->opath->provider->name, gp->name);
 106                         g_detach(sc->opath);
 107                         g_destroy_consumer(sc->opath);
 108                         if (LIST_EMPTY(&gp->consumer)) {
 109                                 /* No consumers left */
 110                                 g_wither_geom(gp, ENXIO);
 111                                 for (;;) {
 112                                         bp = TAILQ_FIRST(&sc->queue);
 113                                         if (bp == NULL)
 114                                                 break;
 115                                         TAILQ_REMOVE(&sc->queue, bp, bio_queue);
 116                                         bp->bio_error = ENXIO;
 117                                         g_std_done(bp);
 118                                 }
 119                                 return;
 120                         }
 121                 } else {
 122                         printf("Got e-bit on old path (%s) on fox (%s)\n",
 123                                 sc->opath->provider->name, gp->name);
 124                 }
 125                 sc->opath = NULL;
 126         } else {
 127                 cp1 = LIST_FIRST(&gp->consumer);
 128         }
 129         if (cp1 == NULL)
 130                 cp1 = LIST_FIRST(&gp->consumer);
 131         printf("Open new path (%s) on fox (%s)\n",
 132                 cp1->provider->name, gp->name);
 133         error = g_access(cp1, sc->cr, sc->cw, sc->ce);
 134         if (error) {
 135                 /*
 136                  * If we failed, we take another trip through here
 137                  */
 138                 printf("Open new path (%s) on fox (%s) failed, reselect.\n",
 139                         cp1->provider->name, gp->name);
 140                 sc->opath = cp1;
 141                 g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL);
 142         } else {
 143                 printf("Open new path (%s) on fox (%s) succeeded\n",
 144                         cp1->provider->name, gp->name);
 145                 mtx_lock(&sc->lock);
 146                 sc->path = cp1;
 147                 sc->waiting = 0;
 148                 for (;;) {
 149                         bp = TAILQ_FIRST(&sc->queue);
 150                         if (bp == NULL)
 151                                 break;
 152                         TAILQ_REMOVE(&sc->queue, bp, bio_queue);
 153                         g_io_request(bp, sc->path);
 154                 }
 155                 mtx_unlock(&sc->lock);
 156         }
 157 }
 158
 159 static void
 160 g_fox_orphan(struct g_consumer *cp)
 161 {
 162         struct g_geom *gp;
 163         struct g_fox_softc *sc;
 164         int error, mark;
 165
 166         g_topology_assert();
 167         gp = cp->geom;
 168         sc = gp->softc;
 169         printf("Removing path (%s) from fox (%s)\n",
 170             cp->provider->name, gp->name);
 171         mtx_lock(&sc->lock);
 172         if (cp == sc->path) {
 173                 sc->opath = NULL;
 174                 sc->path = NULL;
 175                 sc->waiting = 1;
 176                 mark = 1;
 177         } else {
 178                 mark = 0;
 179         }
 180         mtx_unlock(&sc->lock);
 181
 182         g_access(cp, -cp->acr, -cp->acw, -cp->ace);
 183         error = cp->provider->error;
 184         g_detach(cp);
 185         g_destroy_consumer(cp);
 186         if (!LIST_EMPTY(&gp->consumer)) {
 187                 if (mark)
 188                         g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL);
 189                 return;
 190         }
 191
 192         mtx_destroy(&sc->lock);
 193         g_free(gp->softc);
 194         gp->softc = NULL;
 195         g_wither_geom(gp, ENXIO);
 196 }
 197
 198 static void
 199 g_fox_done(struct bio *bp)
 200 {
 201         struct g_geom *gp;
 202         struct g_fox_softc *sc;
 203         int error;
 204
 205         if (bp->bio_error == 0) {
 206                 g_std_done(bp);
 207                 return;
 208         }
 209         gp = bp->bio_from->geom;
 210         sc = gp->softc;
 211         if (bp->bio_from != sc->path) {
 212                 g_io_request(bp, sc->path);
 213                 return;
 214         }
 215         mtx_lock(&sc->lock);
 216         sc->opath = sc->path;
 217         sc->path = NULL;
 218         error = g_post_event(g_fox_select_path, gp, M_NOWAIT, gp, NULL);
 219         if (error) {
 220                 bp->bio_error = ENOMEM;
 221                 g_std_done(bp);
 222         } else {
 223                 sc->waiting = 1;
 224                 TAILQ_INSERT_TAIL(&sc->queue, bp, bio_queue);
 225         }
 226         mtx_unlock(&sc->lock);
 227 }
 228
 229 static void
 230 g_fox_start(struct bio *bp)
 231 {
 232         struct g_geom *gp;
 233         struct bio *bp2;
 234         struct g_fox_softc *sc;
 235         int error;
 236
 237         gp = bp->bio_to->geom;
 238         sc = gp->softc;
 239         if (sc == NULL) {
 240                 g_io_deliver(bp, ENXIO);
 241                 return;
 242         }
 243         switch(bp->bio_cmd) {
 244         case BIO_READ:
 245         case BIO_WRITE:
 246         case BIO_DELETE:
 247                 bp2 = g_clone_bio(bp);
 248                 if (bp2 == NULL) {
 249                         g_io_deliver(bp, ENOMEM);
 250                         break;
 251                 }
 252                 bp2->bio_offset += sc->sectorsize;
 253                 bp2->bio_done = g_fox_done;
 254                 mtx_lock(&sc->lock);
 255                 if (sc->path == NULL || !TAILQ_EMPTY(&sc->queue)) {
 256                         if (sc->waiting == 0) {
 257                                 error = g_post_event(g_fox_select_path, gp,
 258                                     M_NOWAIT, gp, NULL);
 259                                 if (error) {
 260                                         g_destroy_bio(bp2);
 261                                         bp2 = NULL;
 262                                         g_io_deliver(bp, error);
 263                                 } else {
 264                                         sc->waiting = 1;
 265                                 }
 266                         }
 267                         if (bp2 != NULL)
 268                                 TAILQ_INSERT_TAIL(&sc->queue, bp2,
 269                                     bio_queue);
 270                 } else {
 271                         g_io_request(bp2, sc->path);
 272                 }
 273                 mtx_unlock(&sc->lock);
 274                 break;
 275         default:
 276                 g_io_deliver(bp, EOPNOTSUPP);
 277                 break;
 278         }
 279         return;
 280 }
 281
 282 static int
 283 g_fox_access(struct g_provider *pp, int dr, int dw, int de)
 284 {
 285         struct g_geom *gp;
 286         struct g_fox_softc *sc;
 287         struct g_consumer *cp1;
 288         int error;
 289
 290         g_topology_assert();
 291         gp = pp->geom;
 292         sc = gp->softc;
 293         if (sc == NULL) {
 294                 if (dr <= 0 && dw <= 0 && de <= 0)
 295                         return (0);
 296                 else
 297                         return (ENXIO);
 298         }
 299
 300         if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) {
 301                 /*
 302                  * First open, open all consumers with an exclusive bit
 303                  */
 304                 error = 0;
 305                 LIST_FOREACH(cp1, &gp->consumer, consumer) {
 306                         error = g_access(cp1, 0, 0, 1);
 307                         if (error) {
 308                                 printf("FOX: access(%s,0,0,1) = %d\n",
 309                                     cp1->provider->name, error);
 310                                 break;
 311                         }
 312                 }
 313                 if (error) {
 314                         LIST_FOREACH(cp1, &gp->consumer, consumer) {
 315                                 if (cp1->ace)
 316                                         g_access(cp1, 0, 0, -1);
 317                         }
 318                         return (error);
 319                 }
 320         }
 321         if (sc->path == NULL)
 322                 g_fox_select_path(gp, 0);
 323         if (sc->path == NULL)
 324                 error = ENXIO;
 325         else
 326                 error = g_access(sc->path, dr, dw, de);
 327         if (error == 0) {
 328                 sc->cr += dr;
 329                 sc->cw += dw;
 330                 sc->ce += de;
 331                 if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) {
 332                         /*
 333                          * Last close, remove e-bit on all consumers
 334                          */
 335                         LIST_FOREACH(cp1, &gp->consumer, consumer)
 336                                 g_access(cp1, 0, 0, -1);
 337                 }
 338         }
 339         return (error);
 340 }
 341
 342 static struct g_geom *
 343 g_fox_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 344 {
 345         struct g_geom *gp, *gp2;
 346         struct g_provider *pp2;
 347         struct g_consumer *cp, *cp2;
 348         struct g_fox_softc *sc, *sc2;
 349         int error;
 350         u_int sectorsize;
 351         u_char *buf;
 352
 353         g_trace(G_T_TOPOLOGY, "fox_taste(%s, %s)", mp->name, pp->name);
 354         g_topology_assert();
 355         if (!strcmp(pp->geom->class->name, mp->name))
 356                 return (NULL);
 357         gp = g_new_geomf(mp, "%s.fox", pp->name);
 358         gp->softc = g_malloc(sizeof(struct g_fox_softc), M_WAITOK | M_ZERO);
 359         sc = gp->softc;
 360
 361         cp = g_new_consumer(gp);
 362         g_attach(cp, pp);
 363         error = g_access(cp, 1, 0, 0);
 364         if (error) {
 365                 g_free(sc);
 366                 g_detach(cp);
 367                 g_destroy_consumer(cp);
 368                 g_destroy_geom(gp);
 369                 return(NULL);
 370         }
 371         do {
 372                 sectorsize = cp->provider->sectorsize;
 373                 g_topology_unlock();
 374                 buf = g_read_data(cp, 0, sectorsize, NULL);
 375                 g_topology_lock();
 376                 if (buf == NULL)
 377                         break;
 378                 if (memcmp(buf, FOX_MAGIC, strlen(FOX_MAGIC)))
 379                         break;
 380
 381                 /*
 382                  * First we need to see if this a new path for an existing fox.
 383                  */
 384                 LIST_FOREACH(gp2, &mp->geom, geom) {
 385                         sc2 = gp2->softc;
 386                         if (sc2 == NULL)
 387                                 continue;
 388                         if (memcmp(buf + 16, sc2->magic, sizeof sc2->magic))
 389                                 continue;
 390                         break;
 391                 }
 392                 if (gp2 != NULL) {
 393                         /*
 394                          * It was.  Create a new consumer for that fox,
 395                          * attach it, and if the fox is open, open this
 396                          * path with an exclusive count of one.
 397                          */
 398                         printf("Adding path (%s) to fox (%s)\n",
 399                             pp->name, gp2->name);
 400                         cp2 = g_new_consumer(gp2);
 401                         g_attach(cp2, pp);
 402                         pp2 = LIST_FIRST(&gp2->provider);
 403                         if (pp2->acr > 0 || pp2->acw > 0 || pp2->ace > 0) {
 404                                 error = g_access(cp2, 0, 0, 1);
 405                                 if (error) {
 406                                         /*
 407                                          * This is bad, or more likely,
 408                                          * the user is doing something stupid
 409                                          */
 410                                         printf(
 411         "WARNING: New path (%s) to fox(%s) not added: %s\n%s",
 412                                             cp2->provider->name, gp2->name,
 413         "Could not get exclusive bit.",
 414         "WARNING: This indicates a risk of data inconsistency."
 415                                         );
 416                                         g_detach(cp2);
 417                                         g_destroy_consumer(cp2);
 418                                 }
 419                         }
 420                         break;
 421                 }
 422                 printf("Creating new fox (%s)\n", pp->name);
 423                 sc->path = cp;
 424                 memcpy(sc->magic, buf + 16, sizeof sc->magic);
 425                 pp2 = g_new_providerf(gp, "%s", gp->name);
 426                 pp2->mediasize = sc->mediasize = pp->mediasize - pp->sectorsize;
 427                 pp2->sectorsize = sc->sectorsize = pp->sectorsize;
 428 printf("fox %s lock %p\n", gp->name, &sc->lock);
 429
 430                 mtx_init(&sc->lock, "fox queue", NULL, MTX_DEF);
 431                 TAILQ_INIT(&sc->queue);
 432                 g_error_provider(pp2, 0);
 433         } while (0);
 434         if (buf != NULL)
 435                 g_free(buf);
 436         g_access(cp, -1, 0, 0);
 437
 438         if (!LIST_EMPTY(&gp->provider))
 439                 return (gp);
 440
 441         g_free(gp->softc);
 442         g_detach(cp);
 443         g_destroy_consumer(cp);
 444         g_destroy_geom(gp);
 445         return (NULL);
 446 }
 447
 448 static int
 449 g_fox_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp)
 450 {
 451         struct g_fox_softc *sc;
 452
 453         g_topology_assert();
 454         sc = gp->softc;
 455         mtx_destroy(&sc->lock);
 456         g_free(gp->softc);
 457         gp->softc = NULL;
 458         g_wither_geom(gp, ENXIO);
 459         return (0);
 460 }
 461
 462 static struct g_class g_fox_class       = {
 463         .name = FOX_CLASS_NAME,
 464         .version = G_VERSION,
 465         .taste = g_fox_taste,
 466         .destroy_geom = g_fox_destroy_geom,
 467         .start = g_fox_start,
 468         .spoiled = g_fox_orphan,
 469         .orphan = g_fox_orphan,
 470         .access= g_fox_access,
 471 };
 472
 473 DECLARE_GEOM_CLASS(g_fox_class, g_fox);