sys/geom/geom_fox.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 2003 Poul-Henning Kamp
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. The names of the authors may not be used to endorse or promote
  16  *    products derived from this software without specific prior written
  17  *    permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31  * $FreeBSD$
  32  */
  33
  34 /* This is a GEOM module for handling path selection for multi-path
  35  * storage devices.  It is named "fox" because it, like they, prefer
  36  * to have multiple exits to choose from.
  37  *
  38  */
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #include <sys/sysctl.h>
  43 #include <sys/kernel.h>
  44 #include <sys/conf.h>
  45 #include <sys/bio.h>
  46 #include <sys/malloc.h>
  47 #include <sys/lock.h>
  48 #include <sys/mutex.h>
  49 #include <sys/libkern.h>
  50 #include <sys/endian.h>
  51 #include <sys/md5.h>
  52 #include <sys/errno.h>
  53 #include <geom/geom.h>
  54
  55 #define FOX_CLASS_NAME "FOX"
  56 #define FOX_MAGIC       "GEOM::FOX"
  57
  58 static int g_fox_once;
  59
  60 FEATURE(geom_fox, "GEOM FOX redundant path mitigation support");
  61
  62 struct g_fox_softc {
  63         off_t                   mediasize;
  64         u_int                   sectorsize;
  65         TAILQ_HEAD(, bio)       queue;
  66         struct mtx              lock;
  67         u_char                  magic[16];
  68         struct g_consumer       *path;
  69         struct g_consumer       *opath;
  70         int                     waiting;
  71         int                     cr, cw, ce;
  72 };
  73
  74 /*
  75  * This function is called whenever we need to select a new path.
  76  */
  77 static void
  78 g_fox_select_path(void *arg, int flag)
  79 {
  80         struct g_geom *gp;
  81         struct g_fox_softc *sc;
  82         struct g_consumer *cp1;
  83         struct bio *bp;
  84         int error;
  85
  86         g_topology_assert();
  87         if (flag == EV_CANCEL)
  88                 return;
  89         gp = arg;
  90         sc = gp->softc;
  91
  92         if (sc->opath != NULL) {
  93                 /*
  94                  * First, close the old path entirely.
  95                  */
  96                 printf("Closing old path (%s) on fox (%s)\n",
  97                         sc->opath->provider->name, gp->name);
  98
  99                 cp1 = LIST_NEXT(sc->opath, consumer);
 100
 101                 g_access(sc->opath, -sc->cr, -sc->cw, -(sc->ce + 1));
 102
 103                 /*
 104                  * The attempt to reopen it with a exclusive count
 105                  */
 106                 error = g_access(sc->opath, 0, 0, 1);
 107                 if (error) {
 108                         /*
 109                          * Ok, ditch this consumer, we can't use it.
 110                          */
 111                         printf("Drop old path (%s) on fox (%s)\n",
 112                                 sc->opath->provider->name, gp->name);
 113                         g_detach(sc->opath);
 114                         g_destroy_consumer(sc->opath);
 115                         if (LIST_EMPTY(&gp->consumer)) {
 116                                 /* No consumers left */
 117                                 g_wither_geom(gp, ENXIO);
 118                                 for (;;) {
 119                                         bp = TAILQ_FIRST(&sc->queue);
 120                                         if (bp == NULL)
 121                                                 break;
 122                                         TAILQ_REMOVE(&sc->queue, bp, bio_queue);
 123                                         bp->bio_error = ENXIO;
 124                                         g_std_done(bp);
 125                                 }
 126                                 return;
 127                         }
 128                 } else {
 129                         printf("Got e-bit on old path (%s) on fox (%s)\n",
 130                                 sc->opath->provider->name, gp->name);
 131                 }
 132                 sc->opath = NULL;
 133         } else {
 134                 cp1 = LIST_FIRST(&gp->consumer);
 135         }
 136         if (cp1 == NULL)
 137                 cp1 = LIST_FIRST(&gp->consumer);
 138         printf("Open new path (%s) on fox (%s)\n",
 139                 cp1->provider->name, gp->name);
 140         error = g_access(cp1, sc->cr, sc->cw, sc->ce);
 141         if (error) {
 142                 /*
 143                  * If we failed, we take another trip through here
 144                  */
 145                 printf("Open new path (%s) on fox (%s) failed, reselect.\n",
 146                         cp1->provider->name, gp->name);
 147                 sc->opath = cp1;
 148                 g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL);
 149         } else {
 150                 printf("Open new path (%s) on fox (%s) succeeded\n",
 151                         cp1->provider->name, gp->name);
 152                 mtx_lock(&sc->lock);
 153                 sc->path = cp1;
 154                 sc->waiting = 0;
 155                 for (;;) {
 156                         bp = TAILQ_FIRST(&sc->queue);
 157                         if (bp == NULL)
 158                                 break;
 159                         TAILQ_REMOVE(&sc->queue, bp, bio_queue);
 160                         g_io_request(bp, sc->path);
 161                 }
 162                 mtx_unlock(&sc->lock);
 163         }
 164 }
 165
 166 static void
 167 g_fox_orphan(struct g_consumer *cp)
 168 {
 169         struct g_geom *gp;
 170         struct g_fox_softc *sc;
 171         int error, mark;
 172
 173         g_topology_assert();
 174         gp = cp->geom;
 175         sc = gp->softc;
 176         printf("Removing path (%s) from fox (%s)\n",
 177             cp->provider->name, gp->name);
 178         mtx_lock(&sc->lock);
 179         if (cp == sc->path) {
 180                 sc->opath = NULL;
 181                 sc->path = NULL;
 182                 sc->waiting = 1;
 183                 mark = 1;
 184         } else {
 185                 mark = 0;
 186         }
 187         mtx_unlock(&sc->lock);
 188
 189         g_access(cp, -cp->acr, -cp->acw, -cp->ace);
 190         error = cp->provider->error;
 191         g_detach(cp);
 192         g_destroy_consumer(cp);
 193         if (!LIST_EMPTY(&gp->consumer)) {
 194                 if (mark)
 195                         g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL);
 196                 return;
 197         }
 198
 199         mtx_destroy(&sc->lock);
 200         g_free(gp->softc);
 201         gp->softc = NULL;
 202         g_wither_geom(gp, ENXIO);
 203 }
 204
 205 static void
 206 g_fox_done(struct bio *bp)
 207 {
 208         struct g_geom *gp;
 209         struct g_fox_softc *sc;
 210         int error;
 211
 212         if (bp->bio_error == 0) {
 213                 g_std_done(bp);
 214                 return;
 215         }
 216         gp = bp->bio_from->geom;
 217         sc = gp->softc;
 218         if (bp->bio_from != sc->path) {
 219                 g_io_request(bp, sc->path);
 220                 return;
 221         }
 222         mtx_lock(&sc->lock);
 223         sc->opath = sc->path;
 224         sc->path = NULL;
 225         error = g_post_event(g_fox_select_path, gp, M_NOWAIT, gp, NULL);
 226         if (error) {
 227                 bp->bio_error = ENOMEM;
 228                 g_std_done(bp);
 229         } else {
 230                 sc->waiting = 1;
 231                 TAILQ_INSERT_TAIL(&sc->queue, bp, bio_queue);
 232         }
 233         mtx_unlock(&sc->lock);
 234 }
 235
 236 static void
 237 g_fox_start(struct bio *bp)
 238 {
 239         struct g_geom *gp;
 240         struct bio *bp2;
 241         struct g_fox_softc *sc;
 242         int error;
 243
 244         gp = bp->bio_to->geom;
 245         sc = gp->softc;
 246         if (sc == NULL) {
 247                 g_io_deliver(bp, ENXIO);
 248                 return;
 249         }
 250         switch(bp->bio_cmd) {
 251         case BIO_READ:
 252         case BIO_WRITE:
 253         case BIO_DELETE:
 254                 bp2 = g_clone_bio(bp);
 255                 if (bp2 == NULL) {
 256                         g_io_deliver(bp, ENOMEM);
 257                         break;
 258                 }
 259                 bp2->bio_offset += sc->sectorsize;
 260                 bp2->bio_done = g_fox_done;
 261                 mtx_lock(&sc->lock);
 262                 if (sc->path == NULL || !TAILQ_EMPTY(&sc->queue)) {
 263                         if (sc->waiting == 0) {
 264                                 error = g_post_event(g_fox_select_path, gp,
 265                                     M_NOWAIT, gp, NULL);
 266                                 if (error) {
 267                                         g_destroy_bio(bp2);
 268                                         bp2 = NULL;
 269                                         g_io_deliver(bp, error);
 270                                 } else {
 271                                         sc->waiting = 1;
 272                                 }
 273                         }
 274                         if (bp2 != NULL)
 275                                 TAILQ_INSERT_TAIL(&sc->queue, bp2,
 276                                     bio_queue);
 277                 } else {
 278                         g_io_request(bp2, sc->path);
 279                 }
 280                 mtx_unlock(&sc->lock);
 281                 break;
 282         default:
 283                 g_io_deliver(bp, EOPNOTSUPP);
 284                 break;
 285         }
 286         return;
 287 }
 288
 289 static int
 290 g_fox_access(struct g_provider *pp, int dr, int dw, int de)
 291 {
 292         struct g_geom *gp;
 293         struct g_fox_softc *sc;
 294         struct g_consumer *cp1;
 295         int error;
 296
 297         g_topology_assert();
 298         gp = pp->geom;
 299         sc = gp->softc;
 300         if (sc == NULL) {
 301                 if (dr <= 0 && dw <= 0 && de <= 0)
 302                         return (0);
 303                 else
 304                         return (ENXIO);
 305         }
 306
 307         if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) {
 308                 /*
 309                  * First open, open all consumers with an exclusive bit
 310                  */
 311                 error = 0;
 312                 LIST_FOREACH(cp1, &gp->consumer, consumer) {
 313                         error = g_access(cp1, 0, 0, 1);
 314                         if (error) {
 315                                 printf("FOX: access(%s,0,0,1) = %d\n",
 316                                     cp1->provider->name, error);
 317                                 break;
 318                         }
 319                 }
 320                 if (error) {
 321                         LIST_FOREACH(cp1, &gp->consumer, consumer) {
 322                                 if (cp1->ace)
 323                                         g_access(cp1, 0, 0, -1);
 324                         }
 325                         return (error);
 326                 }
 327         }
 328         if (sc->path == NULL)
 329                 g_fox_select_path(gp, 0);
 330         if (sc->path == NULL)
 331                 error = ENXIO;
 332         else
 333                 error = g_access(sc->path, dr, dw, de);
 334         if (error == 0) {
 335                 sc->cr += dr;
 336                 sc->cw += dw;
 337                 sc->ce += de;
 338                 if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) {
 339                         /*
 340                          * Last close, remove e-bit on all consumers
 341                          */
 342                         LIST_FOREACH(cp1, &gp->consumer, consumer)
 343                                 g_access(cp1, 0, 0, -1);
 344                 }
 345         }
 346         return (error);
 347 }
 348
 349 static struct g_geom *
 350 g_fox_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 351 {
 352         struct g_geom *gp, *gp2;
 353         struct g_provider *pp2;
 354         struct g_consumer *cp, *cp2;
 355         struct g_fox_softc *sc, *sc2;
 356         int error;
 357         u_int sectorsize;
 358         u_char *buf;
 359
 360         g_trace(G_T_TOPOLOGY, "fox_taste(%s, %s)", mp->name, pp->name);
 361         g_topology_assert();
 362         if (!strcmp(pp->geom->class->name, mp->name))
 363                 return (NULL);
 364         gp = g_new_geomf(mp, "%s.fox", pp->name);
 365         gp->softc = g_malloc(sizeof(struct g_fox_softc), M_WAITOK | M_ZERO);
 366         sc = gp->softc;
 367
 368         cp = g_new_consumer(gp);
 369         g_attach(cp, pp);
 370         error = g_access(cp, 1, 0, 0);
 371         if (error) {
 372                 g_free(sc);
 373                 g_detach(cp);
 374                 g_destroy_consumer(cp);
 375                 g_destroy_geom(gp);
 376                 return(NULL);
 377         }
 378         do {
 379                 sectorsize = cp->provider->sectorsize;
 380                 g_topology_unlock();
 381                 buf = g_read_data(cp, 0, sectorsize, NULL);
 382                 g_topology_lock();
 383                 if (buf == NULL)
 384                         break;
 385                 if (memcmp(buf, FOX_MAGIC, strlen(FOX_MAGIC)))
 386                         break;
 387
 388                 /*
 389                  * First we need to see if this a new path for an existing fox.
 390                  */
 391                 LIST_FOREACH(gp2, &mp->geom, geom) {
 392                         sc2 = gp2->softc;
 393                         if (sc2 == NULL)
 394                                 continue;
 395                         if (memcmp(buf + 16, sc2->magic, sizeof sc2->magic))
 396                                 continue;
 397                         break;
 398                 }
 399                 if (gp2 != NULL) {
 400                         /*
 401                          * It was.  Create a new consumer for that fox,
 402                          * attach it, and if the fox is open, open this
 403                          * path with an exclusive count of one.
 404                          */
 405                         printf("Adding path (%s) to fox (%s)\n",
 406                             pp->name, gp2->name);
 407                         cp2 = g_new_consumer(gp2);
 408                         g_attach(cp2, pp);
 409                         pp2 = LIST_FIRST(&gp2->provider);
 410                         if (pp2->acr > 0 || pp2->acw > 0 || pp2->ace > 0) {
 411                                 error = g_access(cp2, 0, 0, 1);
 412                                 if (error) {
 413                                         /*
 414                                          * This is bad, or more likely,
 415                                          * the user is doing something stupid
 416                                          */
 417                                         printf(
 418         "WARNING: New path (%s) to fox(%s) not added: %s\n%s",
 419                                             cp2->provider->name, gp2->name,
 420         "Could not get exclusive bit.",
 421         "WARNING: This indicates a risk of data inconsistency."
 422                                         );
 423                                         g_detach(cp2);
 424                                         g_destroy_consumer(cp2);
 425                                 }
 426                         }
 427                         break;
 428                 }
 429                 printf("Creating new fox (%s)\n", pp->name);
 430                 sc->path = cp;
 431                 memcpy(sc->magic, buf + 16, sizeof sc->magic);
 432                 pp2 = g_new_providerf(gp, "%s", gp->name);
 433                 pp2->mediasize = sc->mediasize = pp->mediasize - pp->sectorsize;
 434                 pp2->sectorsize = sc->sectorsize = pp->sectorsize;
 435 printf("fox %s lock %p\n", gp->name, &sc->lock);
 436
 437                 mtx_init(&sc->lock, "fox queue", NULL, MTX_DEF);
 438                 TAILQ_INIT(&sc->queue);
 439                 g_error_provider(pp2, 0);
 440         } while (0);
 441         if (buf != NULL)
 442                 g_free(buf);
 443         g_access(cp, -1, 0, 0);
 444
 445         if (!LIST_EMPTY(&gp->provider)) {
 446                 if (!g_fox_once) {
 447                         g_fox_once = 1;
 448                         printf(
 449                             "WARNING: geom_fox (geom %s) is deprecated, "
 450                             "use gmultipath instead.\n", gp->name);
 451                 }
 452                 return (gp);
 453         }
 454
 455         g_free(gp->softc);
 456         g_detach(cp);
 457         g_destroy_consumer(cp);
 458         g_destroy_geom(gp);
 459         return (NULL);
 460 }
 461
 462 static int
 463 g_fox_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp)
 464 {
 465         struct g_fox_softc *sc;
 466
 467         g_topology_assert();
 468         sc = gp->softc;
 469         mtx_destroy(&sc->lock);
 470         g_free(gp->softc);
 471         gp->softc = NULL;
 472         g_wither_geom(gp, ENXIO);
 473         return (0);
 474 }
 475
 476 static struct g_class g_fox_class       = {
 477         .name = FOX_CLASS_NAME,
 478         .version = G_VERSION,
 479         .taste = g_fox_taste,
 480         .destroy_geom = g_fox_destroy_geom,
 481         .start = g_fox_start,
 482         .spoiled = g_fox_orphan,
 483         .orphan = g_fox_orphan,
 484         .access= g_fox_access,
 485 };
 486
 487 DECLARE_GEOM_CLASS(g_fox_class, g_fox);
 488 MODULE_VERSION(geom_fox, 0);