2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2015 Alexander Motin <mav@FreeBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer,
12 * without modification, immediately at the beginning of the file.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #include <sys/param.h>
30 #include <sys/condvar.h>
32 #include <sys/eventhandler.h>
33 #include <sys/kernel.h>
34 #include <sys/kthread.h>
35 #include <sys/limits.h>
37 #include <sys/malloc.h>
39 #include <sys/module.h>
40 #include <sys/mutex.h>
42 #include <sys/queue.h>
43 #include <sys/socket.h>
44 #include <sys/socketvar.h>
45 #include <sys/sysctl.h>
46 #include <sys/systm.h>
48 #include <netinet/in.h>
49 #include <netinet/tcp.h>
53 #include <cam/scsi/scsi_all.h>
54 #include <cam/scsi/scsi_da.h>
55 #include <cam/ctl/ctl_io.h>
56 #include <cam/ctl/ctl.h>
57 #include <cam/ctl/ctl_frontend.h>
58 #include <cam/ctl/ctl_util.h>
59 #include <cam/ctl/ctl_backend.h>
60 #include <cam/ctl/ctl_ioctl.h>
61 #include <cam/ctl/ctl_ha.h>
62 #include <cam/ctl/ctl_private.h>
63 #include <cam/ctl/ctl_debug.h>
64 #include <cam/ctl/ctl_error.h>
71 struct ha_dt_msg_wire {
72 ctl_ha_dt_cmd command;
79 struct ctl_softc *ha_ctl_softc;
80 ctl_evt_handler ha_handler[CTL_HA_CHAN_MAX];
82 struct sockaddr_in ha_peer_in;
83 struct socket *ha_lso;
85 struct mbufq ha_sendq;
86 struct mbuf *ha_sending;
95 eventhandler_tag ha_shutdown_eh;
96 TAILQ_HEAD(, ctl_ha_dt_req) ha_dts;
100 ctl_ha_conn_wake(struct ha_softc *softc)
103 mtx_lock(&softc->ha_lock);
104 softc->ha_wakeup = 1;
105 mtx_unlock(&softc->ha_lock);
106 wakeup(&softc->ha_wakeup);
110 ctl_ha_lupcall(struct socket *so, void *arg, int waitflag)
112 struct ha_softc *softc = arg;
114 ctl_ha_conn_wake(softc);
119 ctl_ha_rupcall(struct socket *so, void *arg, int waitflag)
121 struct ha_softc *softc = arg;
123 wakeup(&softc->ha_receiving);
128 ctl_ha_supcall(struct socket *so, void *arg, int waitflag)
130 struct ha_softc *softc = arg;
132 ctl_ha_conn_wake(softc);
137 ctl_ha_evt(struct ha_softc *softc, ctl_ha_channel ch, ctl_ha_event evt,
142 if (ch < CTL_HA_CHAN_MAX) {
143 if (softc->ha_handler[ch])
144 softc->ha_handler[ch](ch, evt, param);
147 for (i = 0; i < CTL_HA_CHAN_MAX; i++) {
148 if (softc->ha_handler[i])
149 softc->ha_handler[i](i, evt, param);
154 ctl_ha_close(struct ha_softc *softc)
156 struct socket *so = softc->ha_so;
159 if (softc->ha_connected || softc->ha_disconnect) {
160 softc->ha_connected = 0;
161 mbufq_drain(&softc->ha_sendq);
162 m_freem(softc->ha_sending);
163 softc->ha_sending = NULL;
167 SOCKBUF_LOCK(&so->so_rcv);
168 soupcall_clear(so, SO_RCV);
169 while (softc->ha_receiving) {
170 wakeup(&softc->ha_receiving);
171 msleep(&softc->ha_receiving, SOCKBUF_MTX(&so->so_rcv),
174 SOCKBUF_UNLOCK(&so->so_rcv);
175 SOCKBUF_LOCK(&so->so_snd);
176 soupcall_clear(so, SO_SND);
177 SOCKBUF_UNLOCK(&so->so_snd);
179 if (softc->ha_connect)
180 pause("reconnect", hz / 2);
184 ctl_ha_evt(softc, CTL_HA_CHAN_MAX, CTL_HA_EVT_LINK_CHANGE,
185 (softc->ha_connect || softc->ha_listen) ?
186 CTL_HA_LINK_UNKNOWN : CTL_HA_LINK_OFFLINE);
191 ctl_ha_lclose(struct ha_softc *softc)
195 if (SOLISTENING(softc->ha_lso)) {
196 SOLISTEN_LOCK(softc->ha_lso);
197 solisten_upcall_set(softc->ha_lso, NULL, NULL);
198 SOLISTEN_UNLOCK(softc->ha_lso);
200 soclose(softc->ha_lso);
201 softc->ha_lso = NULL;
206 ctl_ha_rx_thread(void *arg)
208 struct ha_softc *softc = arg;
209 struct socket *so = softc->ha_so;
210 struct ha_msg_wire wire_hdr;
213 int error, flags, next;
215 bzero(&wire_hdr, sizeof(wire_hdr));
217 if (wire_hdr.length > 0)
218 next = wire_hdr.length;
220 next = sizeof(wire_hdr);
221 SOCKBUF_LOCK(&so->so_rcv);
222 while (sbavail(&so->so_rcv) < next || softc->ha_disconnect) {
223 if (softc->ha_connected == 0 || softc->ha_disconnect ||
225 (so->so_rcv.sb_state & SBS_CANTRCVMORE)) {
228 so->so_rcv.sb_lowat = next;
229 msleep(&softc->ha_receiving, SOCKBUF_MTX(&so->so_rcv),
232 SOCKBUF_UNLOCK(&so->so_rcv);
234 if (wire_hdr.length == 0) {
235 iov.iov_base = &wire_hdr;
236 iov.iov_len = sizeof(wire_hdr);
239 uio.uio_rw = UIO_READ;
240 uio.uio_segflg = UIO_SYSSPACE;
241 uio.uio_td = curthread;
242 uio.uio_resid = sizeof(wire_hdr);
243 flags = MSG_DONTWAIT;
244 error = soreceive(softc->ha_so, NULL, &uio, NULL,
247 printf("%s: header receive error %d\n",
249 SOCKBUF_LOCK(&so->so_rcv);
253 ctl_ha_evt(softc, wire_hdr.channel,
254 CTL_HA_EVT_MSG_RECV, wire_hdr.length);
260 softc->ha_receiving = 0;
261 wakeup(&softc->ha_receiving);
262 SOCKBUF_UNLOCK(&so->so_rcv);
263 ctl_ha_conn_wake(softc);
268 ctl_ha_send(struct ha_softc *softc)
270 struct socket *so = softc->ha_so;
274 if (softc->ha_sending == NULL) {
275 mtx_lock(&softc->ha_lock);
276 softc->ha_sending = mbufq_dequeue(&softc->ha_sendq);
277 mtx_unlock(&softc->ha_lock);
278 if (softc->ha_sending == NULL) {
279 so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1;
283 SOCKBUF_LOCK(&so->so_snd);
284 if (sbspace(&so->so_snd) < softc->ha_sending->m_pkthdr.len) {
285 so->so_snd.sb_lowat = softc->ha_sending->m_pkthdr.len;
286 SOCKBUF_UNLOCK(&so->so_snd);
289 SOCKBUF_UNLOCK(&so->so_snd);
290 error = sosend(softc->ha_so, NULL, NULL, softc->ha_sending,
291 NULL, MSG_DONTWAIT, curthread);
292 softc->ha_sending = NULL;
294 printf("%s: sosend() error %d\n", __func__, error);
301 ctl_ha_sock_setup(struct ha_softc *softc)
304 struct socket *so = softc->ha_so;
308 error = soreserve(so, val, val);
310 printf("%s: soreserve failed %d\n", __func__, error);
312 SOCKBUF_LOCK(&so->so_rcv);
313 so->so_rcv.sb_lowat = sizeof(struct ha_msg_wire);
314 soupcall_set(so, SO_RCV, ctl_ha_rupcall, softc);
315 SOCKBUF_UNLOCK(&so->so_rcv);
316 SOCKBUF_LOCK(&so->so_snd);
317 so->so_snd.sb_lowat = sizeof(struct ha_msg_wire);
318 soupcall_set(so, SO_SND, ctl_ha_supcall, softc);
319 SOCKBUF_UNLOCK(&so->so_snd);
321 bzero(&opt, sizeof(struct sockopt));
322 opt.sopt_dir = SOPT_SET;
323 opt.sopt_level = SOL_SOCKET;
324 opt.sopt_name = SO_KEEPALIVE;
326 opt.sopt_valsize = sizeof(val);
328 error = sosetopt(so, &opt);
330 printf("%s: KEEPALIVE setting failed %d\n", __func__, error);
332 opt.sopt_level = IPPROTO_TCP;
333 opt.sopt_name = TCP_NODELAY;
335 error = sosetopt(so, &opt);
337 printf("%s: NODELAY setting failed %d\n", __func__, error);
339 opt.sopt_name = TCP_KEEPINIT;
341 error = sosetopt(so, &opt);
343 printf("%s: KEEPINIT setting failed %d\n", __func__, error);
345 opt.sopt_name = TCP_KEEPIDLE;
347 error = sosetopt(so, &opt);
349 printf("%s: KEEPIDLE setting failed %d\n", __func__, error);
351 opt.sopt_name = TCP_KEEPINTVL;
353 error = sosetopt(so, &opt);
355 printf("%s: KEEPINTVL setting failed %d\n", __func__, error);
357 opt.sopt_name = TCP_KEEPCNT;
359 error = sosetopt(so, &opt);
361 printf("%s: KEEPCNT setting failed %d\n", __func__, error);
365 ctl_ha_connect(struct ha_softc *softc)
367 struct thread *td = curthread;
368 struct sockaddr_in sa;
372 /* Create the socket */
373 error = socreate(PF_INET, &so, SOCK_STREAM,
374 IPPROTO_TCP, td->td_ucred, td);
376 printf("%s: socreate() error %d\n", __func__, error);
380 ctl_ha_sock_setup(softc);
382 memcpy(&sa, &softc->ha_peer_in, sizeof(sa));
383 error = soconnect(so, (struct sockaddr *)&sa, td);
386 printf("%s: soconnect() error %d\n", __func__, error);
397 ctl_ha_accept(struct ha_softc *softc)
399 struct socket *lso, *so;
400 struct sockaddr *sap;
405 error = solisten_dequeue(lso, &so, 0);
406 if (error == EWOULDBLOCK)
409 printf("%s: socket error %d\n", __func__, error);
414 error = soaccept(so, &sap);
416 printf("%s: soaccept() error %d\n", __func__, error);
424 ctl_ha_sock_setup(softc);
428 ctl_ha_lclose(softc);
433 ctl_ha_listen(struct ha_softc *softc)
435 struct thread *td = curthread;
436 struct sockaddr_in sa;
440 /* Create the socket */
441 if (softc->ha_lso == NULL) {
442 error = socreate(PF_INET, &softc->ha_lso, SOCK_STREAM,
443 IPPROTO_TCP, td->td_ucred, td);
445 printf("%s: socreate() error %d\n", __func__, error);
448 bzero(&opt, sizeof(struct sockopt));
449 opt.sopt_dir = SOPT_SET;
450 opt.sopt_level = SOL_SOCKET;
451 opt.sopt_name = SO_REUSEADDR;
453 opt.sopt_valsize = sizeof(val);
455 error = sosetopt(softc->ha_lso, &opt);
457 printf("%s: REUSEADDR setting failed %d\n",
460 bzero(&opt, sizeof(struct sockopt));
461 opt.sopt_dir = SOPT_SET;
462 opt.sopt_level = SOL_SOCKET;
463 opt.sopt_name = SO_REUSEPORT;
465 opt.sopt_valsize = sizeof(val);
467 error = sosetopt(softc->ha_lso, &opt);
469 printf("%s: REUSEPORT setting failed %d\n",
474 memcpy(&sa, &softc->ha_peer_in, sizeof(sa));
475 error = sobind(softc->ha_lso, (struct sockaddr *)&sa, td);
477 printf("%s: sobind() error %d\n", __func__, error);
480 error = solisten(softc->ha_lso, 1, td);
482 printf("%s: solisten() error %d\n", __func__, error);
485 SOLISTEN_LOCK(softc->ha_lso);
486 softc->ha_lso->so_state |= SS_NBIO;
487 solisten_upcall_set(softc->ha_lso, ctl_ha_lupcall, softc);
488 SOLISTEN_UNLOCK(softc->ha_lso);
492 ctl_ha_lclose(softc);
497 ctl_ha_conn_thread(void *arg)
499 struct ha_softc *softc = arg;
503 if (softc->ha_disconnect || softc->ha_shutdown) {
505 if (softc->ha_disconnect == 2 || softc->ha_shutdown)
506 ctl_ha_lclose(softc);
507 softc->ha_disconnect = 0;
508 if (softc->ha_shutdown)
510 } else if (softc->ha_so != NULL &&
511 (softc->ha_so->so_error ||
512 softc->ha_so->so_rcv.sb_state & SBS_CANTRCVMORE))
514 if (softc->ha_so == NULL) {
515 if (softc->ha_lso != NULL)
516 ctl_ha_accept(softc);
517 else if (softc->ha_listen)
518 ctl_ha_listen(softc);
519 else if (softc->ha_connect)
520 ctl_ha_connect(softc);
522 if (softc->ha_so != NULL) {
523 if (softc->ha_connected == 0 &&
524 softc->ha_so->so_error == 0 &&
525 (softc->ha_so->so_state & SS_ISCONNECTING) == 0) {
526 softc->ha_connected = 1;
527 ctl_ha_evt(softc, CTL_HA_CHAN_MAX,
528 CTL_HA_EVT_LINK_CHANGE,
530 softc->ha_receiving = 1;
531 error = kproc_kthread_add(ctl_ha_rx_thread,
532 softc, &softc->ha_ctl_softc->ctl_proc,
533 NULL, 0, 0, "ctl", "ha_rx");
535 printf("Error creating CTL HA rx thread!\n");
536 softc->ha_receiving = 0;
537 softc->ha_disconnect = 1;
542 mtx_lock(&softc->ha_lock);
543 if (softc->ha_so != NULL &&
544 (softc->ha_so->so_error ||
545 softc->ha_so->so_rcv.sb_state & SBS_CANTRCVMORE))
547 else if (!softc->ha_wakeup)
548 msleep(&softc->ha_wakeup, &softc->ha_lock, 0, "-", hz);
549 softc->ha_wakeup = 0;
550 mtx_unlock(&softc->ha_lock);
552 mtx_lock(&softc->ha_lock);
553 softc->ha_shutdown = 2;
554 wakeup(&softc->ha_wakeup);
555 mtx_unlock(&softc->ha_lock);
560 ctl_ha_peer_sysctl(SYSCTL_HANDLER_ARGS)
562 struct ha_softc *softc = (struct ha_softc *)arg1;
563 struct sockaddr_in *sa;
564 int error, b1, b2, b3, b4, p, num;
567 strlcpy(buf, softc->ha_peer, sizeof(buf));
568 error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
569 if ((error != 0) || (req->newptr == NULL) ||
570 strncmp(buf, softc->ha_peer, sizeof(buf)) == 0)
573 sa = &softc->ha_peer_in;
574 mtx_lock(&softc->ha_lock);
575 if ((num = sscanf(buf, "connect %d.%d.%d.%d:%d",
576 &b1, &b2, &b3, &b4, &p)) >= 4) {
577 softc->ha_connect = 1;
578 softc->ha_listen = 0;
579 } else if ((num = sscanf(buf, "listen %d.%d.%d.%d:%d",
580 &b1, &b2, &b3, &b4, &p)) >= 4) {
581 softc->ha_connect = 0;
582 softc->ha_listen = 1;
584 softc->ha_connect = 0;
585 softc->ha_listen = 0;
591 strlcpy(softc->ha_peer, buf, sizeof(softc->ha_peer));
592 if (softc->ha_connect || softc->ha_listen) {
593 memset(sa, 0, sizeof(*sa));
594 sa->sin_len = sizeof(struct sockaddr_in);
595 sa->sin_family = AF_INET;
596 sa->sin_port = htons((num >= 5) ? p : 999);
597 sa->sin_addr.s_addr =
598 htonl((b1 << 24) + (b2 << 16) + (b3 << 8) + b4);
600 softc->ha_disconnect = 2;
601 softc->ha_wakeup = 1;
602 mtx_unlock(&softc->ha_lock);
603 wakeup(&softc->ha_wakeup);
608 ctl_ha_msg_register(ctl_ha_channel channel, ctl_evt_handler handler)
610 struct ha_softc *softc = &ha_softc;
612 KASSERT(channel < CTL_HA_CHAN_MAX,
613 ("Wrong CTL HA channel %d", channel));
614 softc->ha_handler[channel] = handler;
615 return (CTL_HA_STATUS_SUCCESS);
619 ctl_ha_msg_deregister(ctl_ha_channel channel)
621 struct ha_softc *softc = &ha_softc;
623 KASSERT(channel < CTL_HA_CHAN_MAX,
624 ("Wrong CTL HA channel %d", channel));
625 softc->ha_handler[channel] = NULL;
626 return (CTL_HA_STATUS_SUCCESS);
630 * Receive a message of the specified size.
633 ctl_ha_msg_recv(ctl_ha_channel channel, void *addr, size_t len,
636 struct ha_softc *softc = &ha_softc;
641 if (!softc->ha_connected)
642 return (CTL_HA_STATUS_DISCONNECT);
648 uio.uio_rw = UIO_READ;
649 uio.uio_segflg = UIO_SYSSPACE;
650 uio.uio_td = curthread;
652 flags = wait ? 0 : MSG_DONTWAIT;
653 error = soreceive(softc->ha_so, NULL, &uio, NULL, NULL, &flags);
655 return (CTL_HA_STATUS_SUCCESS);
657 /* Consider all errors fatal for HA sanity. */
658 mtx_lock(&softc->ha_lock);
659 if (softc->ha_connected) {
660 softc->ha_disconnect = 1;
661 softc->ha_wakeup = 1;
662 wakeup(&softc->ha_wakeup);
664 mtx_unlock(&softc->ha_lock);
665 return (CTL_HA_STATUS_ERROR);
669 * Send a message of the specified size.
672 ctl_ha_msg_send2(ctl_ha_channel channel, const void *addr, size_t len,
673 const void *addr2, size_t len2, int wait)
675 struct ha_softc *softc = &ha_softc;
676 struct mbuf *mb, *newmb;
677 struct ha_msg_wire hdr;
680 if (!softc->ha_connected)
681 return (CTL_HA_STATUS_DISCONNECT);
683 newmb = m_getm2(NULL, sizeof(hdr) + len + len2, wait, MT_DATA,
686 /* Consider all errors fatal for HA sanity. */
687 mtx_lock(&softc->ha_lock);
688 if (softc->ha_connected) {
689 softc->ha_disconnect = 1;
690 softc->ha_wakeup = 1;
691 wakeup(&softc->ha_wakeup);
693 mtx_unlock(&softc->ha_lock);
694 printf("%s: Can't allocate mbuf chain\n", __func__);
695 return (CTL_HA_STATUS_ERROR);
697 hdr.channel = channel;
698 hdr.length = len + len2;
700 memcpy(mtodo(mb, 0), &hdr, sizeof(hdr));
701 mb->m_len += sizeof(hdr);
703 for (; mb != NULL && off < len; mb = mb->m_next) {
704 copylen = min(M_TRAILINGSPACE(mb), len - off);
705 memcpy(mtodo(mb, mb->m_len), (const char *)addr + off, copylen);
706 mb->m_len += copylen;
711 KASSERT(off == len, ("%s: off (%zu) != len (%zu)", __func__,
714 for (; mb != NULL && off < len2; mb = mb->m_next) {
715 copylen = min(M_TRAILINGSPACE(mb), len2 - off);
716 memcpy(mtodo(mb, mb->m_len), (const char *)addr2 + off, copylen);
717 mb->m_len += copylen;
720 KASSERT(off == len2, ("%s: off (%zu) != len2 (%zu)", __func__,
722 newmb->m_pkthdr.len = sizeof(hdr) + len + len2;
724 mtx_lock(&softc->ha_lock);
725 if (!softc->ha_connected) {
726 mtx_unlock(&softc->ha_lock);
728 return (CTL_HA_STATUS_DISCONNECT);
730 mbufq_enqueue(&softc->ha_sendq, newmb);
731 softc->ha_wakeup = 1;
732 mtx_unlock(&softc->ha_lock);
733 wakeup(&softc->ha_wakeup);
734 return (CTL_HA_STATUS_SUCCESS);
738 ctl_ha_msg_send(ctl_ha_channel channel, const void *addr, size_t len,
742 return (ctl_ha_msg_send2(channel, addr, len, NULL, 0, wait));
746 ctl_ha_msg_abort(ctl_ha_channel channel)
748 struct ha_softc *softc = &ha_softc;
750 mtx_lock(&softc->ha_lock);
751 softc->ha_disconnect = 1;
752 softc->ha_wakeup = 1;
753 mtx_unlock(&softc->ha_lock);
754 wakeup(&softc->ha_wakeup);
755 return (CTL_HA_STATUS_SUCCESS);
759 * Allocate a data transfer request structure.
761 struct ctl_ha_dt_req *
762 ctl_dt_req_alloc(void)
765 return (malloc(sizeof(struct ctl_ha_dt_req), M_CTL, M_WAITOK | M_ZERO));
769 * Free a data transfer request structure.
772 ctl_dt_req_free(struct ctl_ha_dt_req *req)
779 * Issue a DMA request for a single buffer.
782 ctl_dt_single(struct ctl_ha_dt_req *req)
784 struct ha_softc *softc = &ha_softc;
785 struct ha_dt_msg_wire wire_dt;
786 ctl_ha_status status;
788 wire_dt.command = req->command;
789 wire_dt.size = req->size;
790 wire_dt.local = req->local;
791 wire_dt.remote = req->remote;
792 if (req->command == CTL_HA_DT_CMD_READ && req->callback != NULL) {
793 mtx_lock(&softc->ha_lock);
794 TAILQ_INSERT_TAIL(&softc->ha_dts, req, links);
795 mtx_unlock(&softc->ha_lock);
796 ctl_ha_msg_send(CTL_HA_CHAN_DATA, &wire_dt, sizeof(wire_dt),
798 return (CTL_HA_STATUS_WAIT);
800 if (req->command == CTL_HA_DT_CMD_READ) {
801 status = ctl_ha_msg_send(CTL_HA_CHAN_DATA, &wire_dt,
802 sizeof(wire_dt), M_WAITOK);
804 status = ctl_ha_msg_send2(CTL_HA_CHAN_DATA, &wire_dt,
805 sizeof(wire_dt), req->local, req->size, M_WAITOK);
811 ctl_dt_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
813 struct ha_softc *softc = &ha_softc;
814 struct ctl_ha_dt_req *req;
815 ctl_ha_status isc_status;
817 if (event == CTL_HA_EVT_MSG_RECV) {
818 struct ha_dt_msg_wire wire_dt;
822 size = min(sizeof(wire_dt), param);
823 isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_DATA, &wire_dt,
825 if (isc_status != CTL_HA_STATUS_SUCCESS) {
826 printf("%s: Error receiving message: %d\n",
827 __func__, isc_status);
831 if (wire_dt.command == CTL_HA_DT_CMD_READ) {
832 wire_dt.command = CTL_HA_DT_CMD_WRITE;
834 wire_dt.local = wire_dt.remote;
835 wire_dt.remote = tmp;
836 ctl_ha_msg_send2(CTL_HA_CHAN_DATA, &wire_dt,
837 sizeof(wire_dt), wire_dt.local, wire_dt.size,
839 } else if (wire_dt.command == CTL_HA_DT_CMD_WRITE) {
840 isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_DATA,
841 wire_dt.remote, wire_dt.size, M_WAITOK);
842 mtx_lock(&softc->ha_lock);
843 TAILQ_FOREACH(req, &softc->ha_dts, links) {
844 if (req->local == wire_dt.remote) {
845 TAILQ_REMOVE(&softc->ha_dts, req, links);
849 mtx_unlock(&softc->ha_lock);
851 req->ret = isc_status;
855 } else if (event == CTL_HA_EVT_LINK_CHANGE) {
856 CTL_DEBUG_PRINT(("%s: Link state change to %d\n", __func__,
858 if (param != CTL_HA_LINK_ONLINE) {
859 mtx_lock(&softc->ha_lock);
860 while ((req = TAILQ_FIRST(&softc->ha_dts)) != NULL) {
861 TAILQ_REMOVE(&softc->ha_dts, req, links);
862 mtx_unlock(&softc->ha_lock);
863 req->ret = CTL_HA_STATUS_DISCONNECT;
865 mtx_lock(&softc->ha_lock);
867 mtx_unlock(&softc->ha_lock);
870 printf("%s: Unknown event %d\n", __func__, event);
875 ctl_ha_msg_init(struct ctl_softc *ctl_softc)
877 struct ha_softc *softc = &ha_softc;
880 softc->ha_ctl_softc = ctl_softc;
881 mtx_init(&softc->ha_lock, "CTL HA mutex", NULL, MTX_DEF);
882 mbufq_init(&softc->ha_sendq, INT_MAX);
883 TAILQ_INIT(&softc->ha_dts);
884 error = kproc_kthread_add(ctl_ha_conn_thread, softc,
885 &ctl_softc->ctl_proc, NULL, 0, 0, "ctl", "ha_tx");
887 printf("error creating CTL HA connection thread!\n");
888 mtx_destroy(&softc->ha_lock);
889 return (CTL_HA_STATUS_ERROR);
891 softc->ha_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_pre_sync,
892 ctl_ha_msg_shutdown, ctl_softc, SHUTDOWN_PRI_FIRST);
893 SYSCTL_ADD_PROC(&ctl_softc->sysctl_ctx,
894 SYSCTL_CHILDREN(ctl_softc->sysctl_tree),
896 CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
897 softc, 0, ctl_ha_peer_sysctl, "A", "HA peer connection method");
899 if (ctl_ha_msg_register(CTL_HA_CHAN_DATA, ctl_dt_event_handler)
900 != CTL_HA_STATUS_SUCCESS) {
901 printf("%s: ctl_ha_msg_register failed.\n", __func__);
904 return (CTL_HA_STATUS_SUCCESS);
908 ctl_ha_msg_shutdown(struct ctl_softc *ctl_softc)
910 struct ha_softc *softc = &ha_softc;
912 /* Disconnect and shutdown threads. */
913 mtx_lock(&softc->ha_lock);
914 if (softc->ha_shutdown < 2) {
915 softc->ha_shutdown = 1;
916 softc->ha_wakeup = 1;
917 wakeup(&softc->ha_wakeup);
918 while (softc->ha_shutdown < 2 && !SCHEDULER_STOPPED()) {
919 msleep(&softc->ha_wakeup, &softc->ha_lock, 0,
923 mtx_unlock(&softc->ha_lock);
927 ctl_ha_msg_destroy(struct ctl_softc *ctl_softc)
929 struct ha_softc *softc = &ha_softc;
931 if (softc->ha_shutdown_eh != NULL) {
932 EVENTHANDLER_DEREGISTER(shutdown_pre_sync,
933 softc->ha_shutdown_eh);
934 softc->ha_shutdown_eh = NULL;
937 ctl_ha_msg_shutdown(ctl_softc); /* Just in case. */
939 if (ctl_ha_msg_deregister(CTL_HA_CHAN_DATA) != CTL_HA_STATUS_SUCCESS)
940 printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
942 mtx_destroy(&softc->ha_lock);
943 return (CTL_HA_STATUS_SUCCESS);