2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2021 Ng Peng Nam Sean
5 * Copyright (c) 2022 Alexander V. Chernikov <melifaro@FreeBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 #include <sys/param.h>
34 #include <sys/malloc.h>
36 #include <sys/mutex.h>
37 #include <sys/socket.h>
38 #include <sys/socketvar.h>
39 #include <sys/syslog.h>
41 #include <netlink/netlink.h>
42 #include <netlink/netlink_ctl.h>
43 #include <netlink/netlink_linux.h>
44 #include <netlink/netlink_var.h>
46 #define DEBUG_MOD_NAME nl_io
47 #define DEBUG_MAX_LEVEL LOG_DEBUG3
48 #include <netlink/netlink_debug.h>
49 _DECLARE_DEBUG(LOG_DEBUG);
52 * The logic below provide a p2p interface for receiving and
53 * sending netlink data between the kernel and userland.
56 static const struct sockaddr_nl _nl_empty_src = {
57 .nl_len = sizeof(struct sockaddr_nl),
58 .nl_family = PF_NETLINK,
59 .nl_pid = 0 /* comes from the kernel */
61 static const struct sockaddr *nl_empty_src = (const struct sockaddr *)&_nl_empty_src;
63 static struct mbuf *nl_process_mbuf(struct mbuf *m, struct nlpcb *nlp);
67 queue_push(struct nl_io_queue *q, struct mbuf *mq)
74 q->length += m_length(m, NULL);
75 STAILQ_INSERT_TAIL(&q->head, m, m_stailqpkt);
80 queue_push_head(struct nl_io_queue *q, struct mbuf *m)
82 MPASS(m->m_nextpkt == NULL);
84 q->length += m_length(m, NULL);
85 STAILQ_INSERT_HEAD(&q->head, m, m_stailqpkt);
89 queue_pop(struct nl_io_queue *q)
91 if (!STAILQ_EMPTY(&q->head)) {
92 struct mbuf *m = STAILQ_FIRST(&q->head);
93 STAILQ_REMOVE_HEAD(&q->head, m_stailqpkt);
95 q->length -= m_length(m, NULL);
103 queue_head(const struct nl_io_queue *q)
105 return (STAILQ_FIRST(&q->head));
109 queue_empty(const struct nl_io_queue *q)
111 return (q->length == 0);
115 queue_free(struct nl_io_queue *q)
117 while (!STAILQ_EMPTY(&q->head)) {
118 struct mbuf *m = STAILQ_FIRST(&q->head);
119 STAILQ_REMOVE_HEAD(&q->head, m_stailqpkt);
128 nl_schedule_taskqueue(struct nlpcb *nlp)
130 if (!nlp->nl_task_pending) {
131 nlp->nl_task_pending = true;
132 taskqueue_enqueue(nlp->nl_taskqueue, &nlp->nl_task);
133 NL_LOG(LOG_DEBUG3, "taskqueue scheduled");
135 NL_LOG(LOG_DEBUG3, "taskqueue schedule skipped");
140 nl_receive_async(struct mbuf *m, struct socket *so)
142 struct nlpcb *nlp = sotonlpcb(so);
149 if ((__predict_true(nlp->nl_active))) {
150 sbappend(&so->so_snd, m, 0);
151 NL_LOG(LOG_DEBUG3, "enqueue %u bytes", m_length(m, NULL));
152 nl_schedule_taskqueue(nlp);
154 NL_LOG(LOG_DEBUG, "ignoring %u bytes on non-active socket",
166 tx_check_locked(struct nlpcb *nlp)
168 if (queue_empty(&nlp->tx_queue))
172 * Check if something can be moved from the internal TX queue
173 * to the socket queue.
176 bool appended = false;
177 struct sockbuf *sb = &nlp->nl_socket->so_rcv;
181 struct mbuf *m = queue_head(&nlp->tx_queue);
182 if (m && sbappendaddr_locked(sb, nl_empty_src, m, NULL) != 0) {
183 /* appended successfully */
184 queue_pop(&nlp->tx_queue);
193 sorwakeup(nlp->nl_socket);
195 return (queue_empty(&nlp->tx_queue));
199 nl_process_received_one(struct nlpcb *nlp)
201 bool reschedule = false;
204 nlp->nl_task_pending = false;
206 if (!tx_check_locked(nlp)) {
207 /* TX overflow queue still not empty, ignore RX */
212 if (queue_empty(&nlp->rx_queue)) {
214 * Grab all data we have from the socket TX queue
215 * and store it the internal queue, so it can be worked on
216 * w/o holding socket lock.
218 struct sockbuf *sb = &nlp->nl_socket->so_snd;
221 unsigned int avail = sbavail(sb);
223 NL_LOG(LOG_DEBUG3, "grabbed %u bytes", avail);
224 queue_push(&nlp->rx_queue, sbcut_locked(sb, avail));
228 /* Schedule another pass to read from the socket queue */
232 int prev_hiwat = nlp->tx_queue.hiwat;
235 while (!queue_empty(&nlp->rx_queue)) {
236 struct mbuf *m = queue_pop(&nlp->rx_queue);
238 m = nl_process_mbuf(m, nlp);
240 queue_push_head(&nlp->rx_queue, m);
245 if (nlp->tx_queue.hiwat > prev_hiwat) {
246 NLP_LOG(LOG_DEBUG, nlp, "TX override peaked to %d", nlp->tx_queue.hiwat);
254 nl_process_received(struct nlpcb *nlp)
256 NL_LOG(LOG_DEBUG3, "taskqueue called");
258 while (nl_process_received_one(nlp))
263 nl_init_io(struct nlpcb *nlp)
265 STAILQ_INIT(&nlp->rx_queue.head);
266 STAILQ_INIT(&nlp->tx_queue.head);
270 nl_free_io(struct nlpcb *nlp)
272 queue_free(&nlp->rx_queue);
273 queue_free(&nlp->tx_queue);
277 * Called after some data have been read from the socket.
280 nl_on_transmit(struct nlpcb *nlp)
284 struct socket *so = nlp->nl_socket;
285 if (__predict_false(nlp->nl_dropped_bytes > 0 && so != NULL)) {
286 unsigned long dropped_bytes = nlp->nl_dropped_bytes;
287 unsigned long dropped_messages = nlp->nl_dropped_messages;
288 nlp->nl_dropped_bytes = 0;
289 nlp->nl_dropped_messages = 0;
291 struct sockbuf *sb = &so->so_rcv;
292 NLP_LOG(LOG_DEBUG, nlp,
293 "socket RX overflowed, %lu messages (%lu bytes) dropped. "
294 "bytes: [%u/%u] mbufs: [%u/%u]", dropped_messages, dropped_bytes,
295 sb->sb_ccc, sb->sb_hiwat, sb->sb_mbcnt, sb->sb_mbmax);
296 /* TODO: send netlink message */
299 nl_schedule_taskqueue(nlp);
304 nl_taskqueue_handler(void *_arg, int pending)
306 struct nlpcb *nlp = (struct nlpcb *)_arg;
308 CURVNET_SET(nlp->nl_socket->so_vnet);
309 nl_process_received(nlp);
313 static __noinline void
314 queue_push_tx(struct nlpcb *nlp, struct mbuf *m)
316 queue_push(&nlp->tx_queue, m);
317 nlp->nl_tx_blocked = true;
319 if (nlp->tx_queue.length > nlp->tx_queue.hiwat)
320 nlp->tx_queue.hiwat = nlp->tx_queue.length;
324 * Tries to send @m to the socket @nlp.
326 * @m: mbuf(s) to send to. Consumed in any case.
327 * @nlp: socket to send to
328 * @cnt: number of messages in @m
329 * @io_flags: combination of NL_IOF_* flags
331 * Returns true on success.
332 * If no queue overrunes happened, wakes up socket owner.
335 nl_send_one(struct mbuf *m, struct nlpcb *nlp, int num_messages, int io_flags)
337 bool untranslated = io_flags & NL_IOF_UNTRANSLATED;
338 bool ignore_limits = io_flags & NL_IOF_IGNORE_LIMIT;
341 IF_DEBUG_LEVEL(LOG_DEBUG2) {
342 struct nlmsghdr *hdr = mtod(m, struct nlmsghdr *);
343 NLP_LOG(LOG_DEBUG2, nlp,
344 "TX mbuf len %u msgs %u msg type %d first hdrlen %u io_flags %X",
345 m_length(m, NULL), num_messages, hdr->nlmsg_type, hdr->nlmsg_len,
349 if (__predict_false(nlp->nl_linux && linux_netlink_p != NULL && untranslated)) {
350 m = linux_netlink_p->mbufs_to_linux(nlp->nl_proto, m, nlp);
357 if (__predict_false(nlp->nl_socket == NULL)) {
363 if (!queue_empty(&nlp->tx_queue)) {
365 queue_push_tx(nlp, m);
374 struct socket *so = nlp->nl_socket;
375 if (sbappendaddr(&so->so_rcv, nl_empty_src, m, NULL) != 0) {
377 NLP_LOG(LOG_DEBUG3, nlp, "appended data & woken up");
380 queue_push_tx(nlp, m);
383 * Store dropped data so it can be reported
386 nlp->nl_dropped_bytes += m_length(m, NULL);
387 nlp->nl_dropped_messages += num_messages;
388 NLP_LOG(LOG_DEBUG2, nlp, "RX oveflow: %lu m (+%d), %lu b (+%d)",
389 (unsigned long)nlp->nl_dropped_messages, num_messages,
390 (unsigned long)nlp->nl_dropped_bytes, m_length(m, NULL));
402 nl_receive_message(struct nlmsghdr *hdr, int remaining_length,
403 struct nlpcb *nlp, struct nl_pstate *npt)
405 nl_handler_f handler = nl_handlers[nlp->nl_proto].cb;
408 NLP_LOG(LOG_DEBUG2, nlp, "msg len: %u type: %d: flags: 0x%X seq: %u pid: %u",
409 hdr->nlmsg_len, hdr->nlmsg_type, hdr->nlmsg_flags, hdr->nlmsg_seq,
412 if (__predict_false(hdr->nlmsg_len > remaining_length)) {
413 NLP_LOG(LOG_DEBUG, nlp, "message is not entirely present: want %d got %d",
414 hdr->nlmsg_len, remaining_length);
416 } else if (__predict_false(hdr->nlmsg_len < sizeof(*hdr))) {
417 NL_LOG(LOG_DEBUG, "message too short: %d", hdr->nlmsg_len);
420 /* Stamp each message with sender pid */
421 hdr->nlmsg_pid = nlp->nl_port;
425 if (hdr->nlmsg_flags & NLM_F_REQUEST && hdr->nlmsg_type >= NLMSG_MIN_TYPE) {
426 NL_LOG(LOG_DEBUG2, "handling message with msg type: %d",
429 if (nlp->nl_linux && linux_netlink_p != NULL) {
430 struct nlmsghdr *hdr_orig = hdr;
431 hdr = linux_netlink_p->msg_from_linux(nlp->nl_proto, hdr, npt);
433 /* Failed to translate to kernel format. Report an error back */
436 if (hdr->nlmsg_flags & NLM_F_ACK)
437 nlmsg_ack(nlp, EOPNOTSUPP, hdr, npt);
441 error = handler(hdr, npt);
442 NL_LOG(LOG_DEBUG2, "retcode: %d", error);
444 if ((hdr->nlmsg_flags & NLM_F_ACK) || (error != 0 && error != EINTR)) {
445 if (!npt->nw->suppress_ack) {
446 NL_LOG(LOG_DEBUG3, "ack");
447 nlmsg_ack(nlp, error, hdr, npt);
455 npt_clear(struct nl_pstate *npt)
462 npt->nw->suppress_ack = false;
466 * Processes an incoming packet, which can contain multiple netlink messages
469 nl_process_mbuf(struct mbuf *m, struct nlpcb *nlp)
471 int offset, buffer_length;
472 struct nlmsghdr *hdr;
476 NL_LOG(LOG_DEBUG3, "RX netlink mbuf %p on %p", m, nlp->nl_socket);
478 struct nl_writer nw = {};
479 if (!nlmsg_get_unicast_writer(&nw, NLMSG_SMALL, nlp)) {
481 NL_LOG(LOG_DEBUG, "error allocating socket writer");
485 nlmsg_ignore_limit(&nw);
486 /* TODO: alloc this buf once for nlp */
487 int data_length = m_length(m, NULL);
488 buffer_length = roundup2(data_length, 8) + SCRATCH_BUFFER_SIZE;
490 buffer_length += roundup2(data_length, 8);
491 buffer = malloc(buffer_length, M_NETLINK, M_NOWAIT | M_ZERO);
492 if (buffer == NULL) {
495 NL_LOG(LOG_DEBUG, "Unable to allocate %d bytes of memory",
499 m_copydata(m, 0, data_length, buffer);
501 struct nl_pstate npt = {
503 .lb.base = &buffer[roundup2(data_length, 8)],
504 .lb.size = buffer_length - roundup2(data_length, 8),
506 .strict = nlp->nl_flags & NLF_STRICT,
509 for (offset = 0; offset + sizeof(struct nlmsghdr) <= data_length;) {
510 hdr = (struct nlmsghdr *)&buffer[offset];
511 /* Save length prior to calling handler */
512 int msglen = NLMSG_ALIGN(hdr->nlmsg_len);
513 NL_LOG(LOG_DEBUG3, "parsing offset %d/%d", offset, data_length);
515 error = nl_receive_message(hdr, data_length - offset, nlp, &npt);
517 if (__predict_false(error != 0 || nlp->nl_tx_blocked))
520 NL_LOG(LOG_DEBUG3, "packet parsing done");
521 free(buffer, M_NETLINK);
524 if (nlp->nl_tx_blocked) {
526 nlp->nl_tx_blocked = false;