sys/geom/geom_fox.c

   1 /*-
   2  * Copyright (c) 2003 Poul-Henning Kamp
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. The names of the authors may not be used to endorse or promote
  14  *    products derived from this software without specific prior written
  15  *    permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 /* This is a GEOM module for handling path selection for multi-path
  33  * storage devices.  It is named "fox" because it, like they, prefer
  34  * to have multiple exits to choose from.
  35  *
  36  */
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/kernel.h>
  42 #include <sys/conf.h>
  43 #include <sys/bio.h>
  44 #include <sys/malloc.h>
  45 #include <sys/lock.h>
  46 #include <sys/mutex.h>
  47 #include <sys/libkern.h>
  48 #include <sys/endian.h>
  49 #include <sys/md5.h>
  50 #include <sys/errno.h>
  51 #include <geom/geom.h>
  52
  53 #define FOX_CLASS_NAME "FOX"
  54 #define FOX_MAGIC       "GEOM::FOX"
  55
  56 static int g_fox_once;
  57
  58 FEATURE(geom_fox, "GEOM FOX redundant path mitigation support");
  59
  60 struct g_fox_softc {
  61         off_t                   mediasize;
  62         u_int                   sectorsize;
  63         TAILQ_HEAD(, bio)       queue;
  64         struct mtx              lock;
  65         u_char                  magic[16];
  66         struct g_consumer       *path;
  67         struct g_consumer       *opath;
  68         int                     waiting;
  69         int                     cr, cw, ce;
  70 };
  71
  72 /*
  73  * This function is called whenever we need to select a new path.
  74  */
  75 static void
  76 g_fox_select_path(void *arg, int flag)
  77 {
  78         struct g_geom *gp;
  79         struct g_fox_softc *sc;
  80         struct g_consumer *cp1;
  81         struct bio *bp;
  82         int error;
  83
  84         g_topology_assert();
  85         if (flag == EV_CANCEL)
  86                 return;
  87         gp = arg;
  88         sc = gp->softc;
  89
  90         if (sc->opath != NULL) {
  91                 /*
  92                  * First, close the old path entirely.
  93                  */
  94                 printf("Closing old path (%s) on fox (%s)\n",
  95                         sc->opath->provider->name, gp->name);
  96
  97                 cp1 = LIST_NEXT(sc->opath, consumer);
  98
  99                 g_access(sc->opath, -sc->cr, -sc->cw, -(sc->ce + 1));
 100
 101                 /*
 102                  * The attempt to reopen it with a exclusive count
 103                  */
 104                 error = g_access(sc->opath, 0, 0, 1);
 105                 if (error) {
 106                         /*
 107                          * Ok, ditch this consumer, we can't use it.
 108                          */
 109                         printf("Drop old path (%s) on fox (%s)\n",
 110                                 sc->opath->provider->name, gp->name);
 111                         g_detach(sc->opath);
 112                         g_destroy_consumer(sc->opath);
 113                         if (LIST_EMPTY(&gp->consumer)) {
 114                                 /* No consumers left */
 115                                 g_wither_geom(gp, ENXIO);
 116                                 for (;;) {
 117                                         bp = TAILQ_FIRST(&sc->queue);
 118                                         if (bp == NULL)
 119                                                 break;
 120                                         TAILQ_REMOVE(&sc->queue, bp, bio_queue);
 121                                         bp->bio_error = ENXIO;
 122                                         g_std_done(bp);
 123                                 }
 124                                 return;
 125                         }
 126                 } else {
 127                         printf("Got e-bit on old path (%s) on fox (%s)\n",
 128                                 sc->opath->provider->name, gp->name);
 129                 }
 130                 sc->opath = NULL;
 131         } else {
 132                 cp1 = LIST_FIRST(&gp->consumer);
 133         }
 134         if (cp1 == NULL)
 135                 cp1 = LIST_FIRST(&gp->consumer);
 136         printf("Open new path (%s) on fox (%s)\n",
 137                 cp1->provider->name, gp->name);
 138         error = g_access(cp1, sc->cr, sc->cw, sc->ce);
 139         if (error) {
 140                 /*
 141                  * If we failed, we take another trip through here
 142                  */
 143                 printf("Open new path (%s) on fox (%s) failed, reselect.\n",
 144                         cp1->provider->name, gp->name);
 145                 sc->opath = cp1;
 146                 g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL);
 147         } else {
 148                 printf("Open new path (%s) on fox (%s) succeeded\n",
 149                         cp1->provider->name, gp->name);
 150                 mtx_lock(&sc->lock);
 151                 sc->path = cp1;
 152                 sc->waiting = 0;
 153                 for (;;) {
 154                         bp = TAILQ_FIRST(&sc->queue);
 155                         if (bp == NULL)
 156                                 break;
 157                         TAILQ_REMOVE(&sc->queue, bp, bio_queue);
 158                         g_io_request(bp, sc->path);
 159                 }
 160                 mtx_unlock(&sc->lock);
 161         }
 162 }
 163
 164 static void
 165 g_fox_orphan(struct g_consumer *cp)
 166 {
 167         struct g_geom *gp;
 168         struct g_fox_softc *sc;
 169         int error, mark;
 170
 171         g_topology_assert();
 172         gp = cp->geom;
 173         sc = gp->softc;
 174         printf("Removing path (%s) from fox (%s)\n",
 175             cp->provider->name, gp->name);
 176         mtx_lock(&sc->lock);
 177         if (cp == sc->path) {
 178                 sc->opath = NULL;
 179                 sc->path = NULL;
 180                 sc->waiting = 1;
 181                 mark = 1;
 182         } else {
 183                 mark = 0;
 184         }
 185         mtx_unlock(&sc->lock);
 186
 187         g_access(cp, -cp->acr, -cp->acw, -cp->ace);
 188         error = cp->provider->error;
 189         g_detach(cp);
 190         g_destroy_consumer(cp);
 191         if (!LIST_EMPTY(&gp->consumer)) {
 192                 if (mark)
 193                         g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL);
 194                 return;
 195         }
 196
 197         mtx_destroy(&sc->lock);
 198         g_free(gp->softc);
 199         gp->softc = NULL;
 200         g_wither_geom(gp, ENXIO);
 201 }
 202
 203 static void
 204 g_fox_done(struct bio *bp)
 205 {
 206         struct g_geom *gp;
 207         struct g_fox_softc *sc;
 208         int error;
 209
 210         if (bp->bio_error == 0) {
 211                 g_std_done(bp);
 212                 return;
 213         }
 214         gp = bp->bio_from->geom;
 215         sc = gp->softc;
 216         if (bp->bio_from != sc->path) {
 217                 g_io_request(bp, sc->path);
 218                 return;
 219         }
 220         mtx_lock(&sc->lock);
 221         sc->opath = sc->path;
 222         sc->path = NULL;
 223         error = g_post_event(g_fox_select_path, gp, M_NOWAIT, gp, NULL);
 224         if (error) {
 225                 bp->bio_error = ENOMEM;
 226                 g_std_done(bp);
 227         } else {
 228                 sc->waiting = 1;
 229                 TAILQ_INSERT_TAIL(&sc->queue, bp, bio_queue);
 230         }
 231         mtx_unlock(&sc->lock);
 232 }
 233
 234 static void
 235 g_fox_start(struct bio *bp)
 236 {
 237         struct g_geom *gp;
 238         struct bio *bp2;
 239         struct g_fox_softc *sc;
 240         int error;
 241
 242         gp = bp->bio_to->geom;
 243         sc = gp->softc;
 244         if (sc == NULL) {
 245                 g_io_deliver(bp, ENXIO);
 246                 return;
 247         }
 248         switch(bp->bio_cmd) {
 249         case BIO_READ:
 250         case BIO_WRITE:
 251         case BIO_DELETE:
 252                 bp2 = g_clone_bio(bp);
 253                 if (bp2 == NULL) {
 254                         g_io_deliver(bp, ENOMEM);
 255                         break;
 256                 }
 257                 bp2->bio_offset += sc->sectorsize;
 258                 bp2->bio_done = g_fox_done;
 259                 mtx_lock(&sc->lock);
 260                 if (sc->path == NULL || !TAILQ_EMPTY(&sc->queue)) {
 261                         if (sc->waiting == 0) {
 262                                 error = g_post_event(g_fox_select_path, gp,
 263                                     M_NOWAIT, gp, NULL);
 264                                 if (error) {
 265                                         g_destroy_bio(bp2);
 266                                         bp2 = NULL;
 267                                         g_io_deliver(bp, error);
 268                                 } else {
 269                                         sc->waiting = 1;
 270                                 }
 271                         }
 272                         if (bp2 != NULL)
 273                                 TAILQ_INSERT_TAIL(&sc->queue, bp2,
 274                                     bio_queue);
 275                 } else {
 276                         g_io_request(bp2, sc->path);
 277                 }
 278                 mtx_unlock(&sc->lock);
 279                 break;
 280         default:
 281                 g_io_deliver(bp, EOPNOTSUPP);
 282                 break;
 283         }
 284         return;
 285 }
 286
 287 static int
 288 g_fox_access(struct g_provider *pp, int dr, int dw, int de)
 289 {
 290         struct g_geom *gp;
 291         struct g_fox_softc *sc;
 292         struct g_consumer *cp1;
 293         int error;
 294
 295         g_topology_assert();
 296         gp = pp->geom;
 297         sc = gp->softc;
 298         if (sc == NULL) {
 299                 if (dr <= 0 && dw <= 0 && de <= 0)
 300                         return (0);
 301                 else
 302                         return (ENXIO);
 303         }
 304
 305         if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) {
 306                 /*
 307                  * First open, open all consumers with an exclusive bit
 308                  */
 309                 error = 0;
 310                 LIST_FOREACH(cp1, &gp->consumer, consumer) {
 311                         error = g_access(cp1, 0, 0, 1);
 312                         if (error) {
 313                                 printf("FOX: access(%s,0,0,1) = %d\n",
 314                                     cp1->provider->name, error);
 315                                 break;
 316                         }
 317                 }
 318                 if (error) {
 319                         LIST_FOREACH(cp1, &gp->consumer, consumer) {
 320                                 if (cp1->ace)
 321                                         g_access(cp1, 0, 0, -1);
 322                         }
 323                         return (error);
 324                 }
 325         }
 326         if (sc->path == NULL)
 327                 g_fox_select_path(gp, 0);
 328         if (sc->path == NULL)
 329                 error = ENXIO;
 330         else
 331                 error = g_access(sc->path, dr, dw, de);
 332         if (error == 0) {
 333                 sc->cr += dr;
 334                 sc->cw += dw;
 335                 sc->ce += de;
 336                 if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) {
 337                         /*
 338                          * Last close, remove e-bit on all consumers
 339                          */
 340                         LIST_FOREACH(cp1, &gp->consumer, consumer)
 341                                 g_access(cp1, 0, 0, -1);
 342                 }
 343         }
 344         return (error);
 345 }
 346
 347 static struct g_geom *
 348 g_fox_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 349 {
 350         struct g_geom *gp, *gp2;
 351         struct g_provider *pp2;
 352         struct g_consumer *cp, *cp2;
 353         struct g_fox_softc *sc, *sc2;
 354         int error;
 355         u_int sectorsize;
 356         u_char *buf;
 357
 358         g_trace(G_T_TOPOLOGY, "fox_taste(%s, %s)", mp->name, pp->name);
 359         g_topology_assert();
 360         if (!strcmp(pp->geom->class->name, mp->name))
 361                 return (NULL);
 362         gp = g_new_geomf(mp, "%s.fox", pp->name);
 363         gp->softc = g_malloc(sizeof(struct g_fox_softc), M_WAITOK | M_ZERO);
 364         sc = gp->softc;
 365
 366         cp = g_new_consumer(gp);
 367         g_attach(cp, pp);
 368         error = g_access(cp, 1, 0, 0);
 369         if (error) {
 370                 g_free(sc);
 371                 g_detach(cp);
 372                 g_destroy_consumer(cp);
 373                 g_destroy_geom(gp);
 374                 return(NULL);
 375         }
 376         do {
 377                 sectorsize = cp->provider->sectorsize;
 378                 g_topology_unlock();
 379                 buf = g_read_data(cp, 0, sectorsize, NULL);
 380                 g_topology_lock();
 381                 if (buf == NULL)
 382                         break;
 383                 if (memcmp(buf, FOX_MAGIC, strlen(FOX_MAGIC)))
 384                         break;
 385
 386                 /*
 387                  * First we need to see if this a new path for an existing fox.
 388                  */
 389                 LIST_FOREACH(gp2, &mp->geom, geom) {
 390                         sc2 = gp2->softc;
 391                         if (sc2 == NULL)
 392                                 continue;
 393                         if (memcmp(buf + 16, sc2->magic, sizeof sc2->magic))
 394                                 continue;
 395                         break;
 396                 }
 397                 if (gp2 != NULL) {
 398                         /*
 399                          * It was.  Create a new consumer for that fox,
 400                          * attach it, and if the fox is open, open this
 401                          * path with an exclusive count of one.
 402                          */
 403                         printf("Adding path (%s) to fox (%s)\n",
 404                             pp->name, gp2->name);
 405                         cp2 = g_new_consumer(gp2);
 406                         g_attach(cp2, pp);
 407                         pp2 = LIST_FIRST(&gp2->provider);
 408                         if (pp2->acr > 0 || pp2->acw > 0 || pp2->ace > 0) {
 409                                 error = g_access(cp2, 0, 0, 1);
 410                                 if (error) {
 411                                         /*
 412                                          * This is bad, or more likely,
 413                                          * the user is doing something stupid
 414                                          */
 415                                         printf(
 416         "WARNING: New path (%s) to fox(%s) not added: %s\n%s",
 417                                             cp2->provider->name, gp2->name,
 418         "Could not get exclusive bit.",
 419         "WARNING: This indicates a risk of data inconsistency."
 420                                         );
 421                                         g_detach(cp2);
 422                                         g_destroy_consumer(cp2);
 423                                 }
 424                         }
 425                         break;
 426                 }
 427                 printf("Creating new fox (%s)\n", pp->name);
 428                 sc->path = cp;
 429                 memcpy(sc->magic, buf + 16, sizeof sc->magic);
 430                 pp2 = g_new_providerf(gp, "%s", gp->name);
 431                 pp2->mediasize = sc->mediasize = pp->mediasize - pp->sectorsize;
 432                 pp2->sectorsize = sc->sectorsize = pp->sectorsize;
 433 printf("fox %s lock %p\n", gp->name, &sc->lock);
 434
 435                 mtx_init(&sc->lock, "fox queue", NULL, MTX_DEF);
 436                 TAILQ_INIT(&sc->queue);
 437                 g_error_provider(pp2, 0);
 438         } while (0);
 439         if (buf != NULL)
 440                 g_free(buf);
 441         g_access(cp, -1, 0, 0);
 442
 443         if (!LIST_EMPTY(&gp->provider)) {
 444                 if (!g_fox_once) {
 445                         g_fox_once = 1;
 446                         printf(
 447                             "WARNING: geom_fox (geom %s) is deprecated, "
 448                             "use gmultipath instead.\n", gp->name);
 449                 }
 450                 return (gp);
 451         }
 452
 453         g_free(gp->softc);
 454         g_detach(cp);
 455         g_destroy_consumer(cp);
 456         g_destroy_geom(gp);
 457         return (NULL);
 458 }
 459
 460 static int
 461 g_fox_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp)
 462 {
 463         struct g_fox_softc *sc;
 464
 465         g_topology_assert();
 466         sc = gp->softc;
 467         mtx_destroy(&sc->lock);
 468         g_free(gp->softc);
 469         gp->softc = NULL;
 470         g_wither_geom(gp, ENXIO);
 471         return (0);
 472 }
 473
 474 static struct g_class g_fox_class       = {
 475         .name = FOX_CLASS_NAME,
 476         .version = G_VERSION,
 477         .taste = g_fox_taste,
 478         .destroy_geom = g_fox_destroy_geom,
 479         .start = g_fox_start,
 480         .spoiled = g_fox_orphan,
 481         .orphan = g_fox_orphan,
 482         .access= g_fox_access,
 483 };
 484
 485 DECLARE_GEOM_CLASS(g_fox_class, g_fox);