sbin/hastd/hastd.c

   1 /*-
   2  * Copyright (c) 2009-2010 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Pawel Jakub Dawidek under sponsorship from
   6  * the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/linker.h>
  35 #include <sys/module.h>
  36 #include <sys/wait.h>
  37
  38 #include <assert.h>
  39 #include <err.h>
  40 #include <errno.h>
  41 #include <libutil.h>
  42 #include <signal.h>
  43 #include <stdbool.h>
  44 #include <stdio.h>
  45 #include <stdlib.h>
  46 #include <string.h>
  47 #include <sysexits.h>
  48 #include <unistd.h>
  49
  50 #include <activemap.h>
  51 #include <pjdlog.h>
  52
  53 #include "control.h"
  54 #include "hast.h"
  55 #include "hast_proto.h"
  56 #include "hastd.h"
  57 #include "subr.h"
  58
  59 /* Path to configuration file. */
  60 static const char *cfgpath = HAST_CONFIG;
  61 /* Hastd configuration. */
  62 static struct hastd_config *cfg;
  63 /* Was SIGCHLD signal received? */
  64 static bool sigchld_received = false;
  65 /* Was SIGHUP signal received? */
  66 static bool sighup_received = false;
  67 /* Was SIGINT or SIGTERM signal received? */
  68 bool sigexit_received = false;
  69 /* PID file handle. */
  70 struct pidfh *pfh;
  71
  72 static void
  73 usage(void)
  74 {
  75
  76         errx(EX_USAGE, "[-dFh] [-c config] [-P pidfile]");
  77 }
  78
  79 static void
  80 sighandler(int sig)
  81 {
  82
  83         switch (sig) {
  84         case SIGCHLD:
  85                 sigchld_received = true;
  86                 break;
  87         case SIGHUP:
  88                 sighup_received = true;
  89                 break;
  90         default:
  91                 assert(!"invalid condition");
  92         }
  93 }
  94
  95 static void
  96 g_gate_load(void)
  97 {
  98
  99         if (modfind("g_gate") == -1) {
 100                 /* Not present in kernel, try loading it. */
 101                 if (kldload("geom_gate") == -1 || modfind("g_gate") == -1) {
 102                         if (errno != EEXIST) {
 103                                 pjdlog_exit(EX_OSERR,
 104                                     "Unable to load geom_gate module");
 105                         }
 106                 }
 107         }
 108 }
 109
 110 static void
 111 child_exit_log(unsigned int pid, int status)
 112 {
 113
 114         if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
 115                 pjdlog_debug(1, "Worker process exited gracefully (pid=%u).",
 116                     pid);
 117         } else if (WIFSIGNALED(status)) {
 118                 pjdlog_error("Worker process killed (pid=%u, signal=%d).",
 119                     pid, WTERMSIG(status));
 120         } else {
 121                 pjdlog_error("Worker process exited ungracefully (pid=%u, exitcode=%d).",
 122                     pid, WIFEXITED(status) ? WEXITSTATUS(status) : -1);
 123         }
 124 }
 125
 126 static void
 127 child_exit(void)
 128 {
 129         struct hast_resource *res;
 130         int status;
 131         pid_t pid;
 132
 133         while ((pid = wait3(&status, WNOHANG, NULL)) > 0) {
 134                 /* Find resource related to the process that just exited. */
 135                 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
 136                         if (pid == res->hr_workerpid)
 137                                 break;
 138                 }
 139                 if (res == NULL) {
 140                         /*
 141                          * This can happen when new connection arrives and we
 142                          * cancel child responsible for the old one.
 143                          */
 144                         continue;
 145                 }
 146                 pjdlog_prefix_set("[%s] (%s) ", res->hr_name,
 147                     role2str(res->hr_role));
 148                 child_exit_log(pid, status);
 149                 proto_close(res->hr_ctrl);
 150                 res->hr_workerpid = 0;
 151                 if (res->hr_role == HAST_ROLE_PRIMARY) {
 152                         /*
 153                          * Restart child process if it was killed by signal
 154                          * or exited because of temporary problem.
 155                          */
 156                         if (WIFSIGNALED(status) ||
 157                             (WIFEXITED(status) &&
 158                              WEXITSTATUS(status) == EX_TEMPFAIL)) {
 159                                 sleep(1);
 160                                 pjdlog_info("Restarting worker process.");
 161                                 hastd_primary(res);
 162                         } else {
 163                                 res->hr_role = HAST_ROLE_INIT;
 164                                 pjdlog_info("Changing resource role back to %s.",
 165                                     role2str(res->hr_role));
 166                         }
 167                 }
 168                 pjdlog_prefix_set("%s", "");
 169         }
 170 }
 171
 172 static void
 173 hastd_reload(void)
 174 {
 175
 176         /* TODO */
 177         pjdlog_warning("Configuration reload is not implemented.");
 178 }
 179
 180 static void
 181 listen_accept(void)
 182 {
 183         struct hast_resource *res;
 184         struct proto_conn *conn;
 185         struct nv *nvin, *nvout, *nverr;
 186         const char *resname;
 187         const unsigned char *token;
 188         char laddr[256], raddr[256];
 189         size_t size;
 190         pid_t pid;
 191         int status;
 192
 193         proto_local_address(cfg->hc_listenconn, laddr, sizeof(laddr));
 194         pjdlog_debug(1, "Accepting connection to %s.", laddr);
 195
 196         if (proto_accept(cfg->hc_listenconn, &conn) < 0) {
 197                 pjdlog_errno(LOG_ERR, "Unable to accept connection %s", laddr);
 198                 return;
 199         }
 200
 201         proto_local_address(conn, laddr, sizeof(laddr));
 202         proto_remote_address(conn, raddr, sizeof(raddr));
 203         pjdlog_info("Connection from %s to %s.", raddr, laddr);
 204
 205         /* Error in setting timeout is not critical, but why should it fail? */
 206         if (proto_timeout(conn, HAST_TIMEOUT) < 0)
 207                 pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
 208
 209         nvin = nvout = nverr = NULL;
 210
 211         /*
 212          * Before receiving any data see if remote host have access to any
 213          * resource.
 214          */
 215         TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
 216                 if (proto_address_match(conn, res->hr_remoteaddr))
 217                         break;
 218         }
 219         if (res == NULL) {
 220                 pjdlog_error("Client %s isn't known.", raddr);
 221                 goto close;
 222         }
 223         /* Ok, remote host can access at least one resource. */
 224
 225         if (hast_proto_recv_hdr(conn, &nvin) < 0) {
 226                 pjdlog_errno(LOG_ERR, "Unable to receive header from %s",
 227                     raddr);
 228                 goto close;
 229         }
 230
 231         resname = nv_get_string(nvin, "resource");
 232         if (resname == NULL) {
 233                 pjdlog_error("No 'resource' field in the header received from %s.",
 234                     raddr);
 235                 goto close;
 236         }
 237         pjdlog_debug(2, "%s: resource=%s", raddr, resname);
 238         token = nv_get_uint8_array(nvin, &size, "token");
 239         /*
 240          * NULL token means that this is first conection.
 241          */
 242         if (token != NULL && size != sizeof(res->hr_token)) {
 243                 pjdlog_error("Received token of invalid size from %s (expected %zu, got %zu).",
 244                     raddr, sizeof(res->hr_token), size);
 245                 goto close;
 246         }
 247
 248         /*
 249          * From now on we want to send errors to the remote node.
 250          */
 251         nverr = nv_alloc();
 252
 253         /* Find resource related to this connection. */
 254         TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
 255                 if (strcmp(resname, res->hr_name) == 0)
 256                         break;
 257         }
 258         /* Have we found the resource? */
 259         if (res == NULL) {
 260                 pjdlog_error("No resource '%s' as requested by %s.",
 261                     resname, raddr);
 262                 nv_add_stringf(nverr, "errmsg", "Resource not configured.");
 263                 goto fail;
 264         }
 265
 266         /* Now that we know resource name setup log prefix. */
 267         pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
 268
 269         /* Does the remote host have access to this resource? */
 270         if (!proto_address_match(conn, res->hr_remoteaddr)) {
 271                 pjdlog_error("Client %s has no access to the resource.", raddr);
 272                 nv_add_stringf(nverr, "errmsg", "No access to the resource.");
 273                 goto fail;
 274         }
 275         /* Is the resource marked as secondary? */
 276         if (res->hr_role != HAST_ROLE_SECONDARY) {
 277                 pjdlog_error("We act as %s for the resource and not as %s as requested by %s.",
 278                     role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY),
 279                     raddr);
 280                 nv_add_stringf(nverr, "errmsg",
 281                     "Remote node acts as %s for the resource and not as %s.",
 282                     role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY));
 283                 goto fail;
 284         }
 285         /* Does token (if exists) match? */
 286         if (token != NULL && memcmp(token, res->hr_token,
 287             sizeof(res->hr_token)) != 0) {
 288                 pjdlog_error("Token received from %s doesn't match.", raddr);
 289                 nv_add_stringf(nverr, "errmsg", "Token doesn't match.");
 290                 goto fail;
 291         }
 292         /*
 293          * If there is no token, but we have half-open connection
 294          * (only remotein) or full connection (worker process is running)
 295          * we have to cancel those and accept the new connection.
 296          */
 297         if (token == NULL) {
 298                 assert(res->hr_remoteout == NULL);
 299                 pjdlog_debug(1, "Initial connection from %s.", raddr);
 300                 if (res->hr_workerpid != 0) {
 301                         assert(res->hr_remotein == NULL);
 302                         pjdlog_debug(1,
 303                             "Worker process exists (pid=%u), stopping it.",
 304                             (unsigned int)res->hr_workerpid);
 305                         /* Stop child process. */
 306                         if (kill(res->hr_workerpid, SIGINT) < 0) {
 307                                 pjdlog_errno(LOG_ERR,
 308                                     "Unable to stop worker process (pid=%u)",
 309                                     (unsigned int)res->hr_workerpid);
 310                                 /*
 311                                  * Other than logging the problem we
 312                                  * ignore it - nothing smart to do.
 313                                  */
 314                         }
 315                         /* Wait for it to exit. */
 316                         else if ((pid = waitpid(res->hr_workerpid,
 317                             &status, 0)) != res->hr_workerpid) {
 318                                 /* We can only log the problem. */
 319                                 pjdlog_errno(LOG_ERR,
 320                                     "Waiting for worker process (pid=%u) failed",
 321                                     (unsigned int)res->hr_workerpid);
 322                         } else {
 323                                 child_exit_log(res->hr_workerpid, status);
 324                         }
 325                         res->hr_workerpid = 0;
 326                 } else if (res->hr_remotein != NULL) {
 327                         char oaddr[256];
 328
 329                         proto_remote_address(conn, oaddr, sizeof(oaddr));
 330                         pjdlog_debug(1,
 331                             "Canceling half-open connection from %s on connection from %s.",
 332                             oaddr, raddr);
 333                         proto_close(res->hr_remotein);
 334                         res->hr_remotein = NULL;
 335                 }
 336         }
 337
 338         /*
 339          * Checks and cleanups are done.
 340          */
 341
 342         if (token == NULL) {
 343                 arc4random_buf(res->hr_token, sizeof(res->hr_token));
 344                 nvout = nv_alloc();
 345                 nv_add_uint8_array(nvout, res->hr_token,
 346                     sizeof(res->hr_token), "token");
 347                 if (nv_error(nvout) != 0) {
 348                         pjdlog_common(LOG_ERR, 0, nv_error(nvout),
 349                             "Unable to prepare return header for %s", raddr);
 350                         nv_add_stringf(nverr, "errmsg",
 351                             "Remote node was unable to prepare return header: %s.",
 352                             strerror(nv_error(nvout)));
 353                         goto fail;
 354                 }
 355                 if (hast_proto_send(NULL, conn, nvout, NULL, 0) < 0) {
 356                         int error = errno;
 357
 358                         pjdlog_errno(LOG_ERR, "Unable to send response to %s",
 359                             raddr);
 360                         nv_add_stringf(nverr, "errmsg",
 361                             "Remote node was unable to send response: %s.",
 362                             strerror(error));
 363                         goto fail;
 364                 }
 365                 res->hr_remotein = conn;
 366                 pjdlog_debug(1, "Incoming connection from %s configured.",
 367                     raddr);
 368         } else {
 369                 res->hr_remoteout = conn;
 370                 pjdlog_debug(1, "Outgoing connection to %s configured.", raddr);
 371                 hastd_secondary(res, nvin);
 372         }
 373         nv_free(nvin);
 374         nv_free(nvout);
 375         nv_free(nverr);
 376         pjdlog_prefix_set("%s", "");
 377         return;
 378 fail:
 379         if (nv_error(nverr) != 0) {
 380                 pjdlog_common(LOG_ERR, 0, nv_error(nverr),
 381                     "Unable to prepare error header for %s", raddr);
 382                 goto close;
 383         }
 384         if (hast_proto_send(NULL, conn, nverr, NULL, 0) < 0) {
 385                 pjdlog_errno(LOG_ERR, "Unable to send error to %s", raddr);
 386                 goto close;
 387         }
 388 close:
 389         if (nvin != NULL)
 390                 nv_free(nvin);
 391         if (nvout != NULL)
 392                 nv_free(nvout);
 393         if (nverr != NULL)
 394                 nv_free(nverr);
 395         proto_close(conn);
 396         pjdlog_prefix_set("%s", "");
 397 }
 398
 399 static void
 400 main_loop(void)
 401 {
 402         fd_set rfds, wfds;
 403         int cfd, lfd, maxfd, ret;
 404
 405         cfd = proto_descriptor(cfg->hc_controlconn);
 406         lfd = proto_descriptor(cfg->hc_listenconn);
 407         maxfd = cfd > lfd ? cfd : lfd;
 408
 409         for (;;) {
 410                 if (sigchld_received) {
 411                         sigchld_received = false;
 412                         child_exit();
 413                 }
 414                 if (sighup_received) {
 415                         sighup_received = false;
 416                         hastd_reload();
 417                 }
 418
 419                 /* Setup descriptors for select(2). */
 420                 FD_ZERO(&rfds);
 421                 FD_SET(cfd, &rfds);
 422                 FD_SET(lfd, &rfds);
 423                 FD_ZERO(&wfds);
 424                 FD_SET(cfd, &wfds);
 425                 FD_SET(lfd, &wfds);
 426
 427                 ret = select(maxfd + 1, &rfds, &wfds, NULL, NULL);
 428                 if (ret == -1) {
 429                         if (errno == EINTR)
 430                                 continue;
 431                         KEEP_ERRNO((void)pidfile_remove(pfh));
 432                         pjdlog_exit(EX_OSERR, "select() failed");
 433                 }
 434
 435                 if (FD_ISSET(cfd, &rfds) || FD_ISSET(cfd, &wfds))
 436                         control_handle(cfg);
 437                 if (FD_ISSET(lfd, &rfds) || FD_ISSET(lfd, &wfds))
 438                         listen_accept();
 439         }
 440 }
 441
 442 int
 443 main(int argc, char *argv[])
 444 {
 445         const char *pidfile;
 446         pid_t otherpid;
 447         bool foreground;
 448         int debuglevel;
 449
 450         g_gate_load();
 451
 452         foreground = false;
 453         debuglevel = 0;
 454         pidfile = HASTD_PIDFILE;
 455
 456         for (;;) {
 457                 int ch;
 458
 459                 ch = getopt(argc, argv, "c:dFhP:");
 460                 if (ch == -1)
 461                         break;
 462                 switch (ch) {
 463                 case 'c':
 464                         cfgpath = optarg;
 465                         break;
 466                 case 'd':
 467                         debuglevel++;
 468                         break;
 469                 case 'F':
 470                         foreground = true;
 471                         break;
 472                 case 'P':
 473                         pidfile = optarg;
 474                         break;
 475                 case 'h':
 476                 default:
 477                         usage();
 478                 }
 479         }
 480         argc -= optind;
 481         argv += optind;
 482
 483         pjdlog_debug_set(debuglevel);
 484
 485         pfh = pidfile_open(pidfile, 0600, &otherpid);
 486         if (pfh == NULL) {
 487                 if (errno == EEXIST) {
 488                         pjdlog_exitx(EX_TEMPFAIL,
 489                             "Another hastd is already running, pid: %jd.",
 490                             (intmax_t)otherpid);
 491                 }
 492                 /* If we cannot create pidfile from other reasons, only warn. */
 493                 pjdlog_errno(LOG_WARNING, "Cannot open or create pidfile");
 494         }
 495
 496         cfg = yy_config_parse(cfgpath);
 497         assert(cfg != NULL);
 498
 499         signal(SIGHUP, sighandler);
 500         signal(SIGCHLD, sighandler);
 501
 502         /* Listen on control address. */
 503         if (proto_server(cfg->hc_controladdr, &cfg->hc_controlconn) < 0) {
 504                 KEEP_ERRNO((void)pidfile_remove(pfh));
 505                 pjdlog_exit(EX_OSERR, "Unable to listen on control address %s",
 506                     cfg->hc_controladdr);
 507         }
 508         /* Listen for remote connections. */
 509         if (proto_server(cfg->hc_listenaddr, &cfg->hc_listenconn) < 0) {
 510                 KEEP_ERRNO((void)pidfile_remove(pfh));
 511                 pjdlog_exit(EX_OSERR, "Unable to listen on address %s",
 512                     cfg->hc_listenaddr);
 513         }
 514
 515         if (!foreground) {
 516                 if (daemon(0, 0) < 0) {
 517                         KEEP_ERRNO((void)pidfile_remove(pfh));
 518                         pjdlog_exit(EX_OSERR, "Unable to daemonize");
 519                 }
 520
 521                 /* Start logging to syslog. */
 522                 pjdlog_mode_set(PJDLOG_MODE_SYSLOG);
 523
 524                 /* Write PID to a file. */
 525                 if (pidfile_write(pfh) < 0) {
 526                         pjdlog_errno(LOG_WARNING,
 527                             "Unable to write PID to a file");
 528                 }
 529         }
 530
 531         main_loop();
 532
 533         exit(0);
 534 }