2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Codel/FQ_Codel and PIE/FQ-PIE Code:
5 * Copyright (C) 2016 Centre for Advanced Internet Architectures,
6 * Swinburne University of Technology, Melbourne, Australia.
7 * Portions of this code were made possible in part by a gift from
8 * The Comcast Innovation Fund.
9 * Implemented by Rasool Al-Saadi <ralsaadi@swin.edu.au>
11 * Copyright (c) 1998-2002,2010 Luigi Rizzo, Universita` di Pisa
12 * Portions Copyright (c) 2000 Akamba Corp.
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
41 * Configuration and internal object management for dummynet.
44 #include "opt_inet6.h"
46 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/malloc.h>
51 #include <sys/kernel.h>
53 #include <sys/module.h>
54 #include <sys/mutex.h>
57 #include <sys/rwlock.h>
58 #include <sys/socket.h>
59 #include <sys/socketvar.h>
61 #include <sys/taskqueue.h>
62 #include <net/if.h> /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */
63 #include <netinet/in.h>
64 #include <netinet/ip_var.h> /* ip_output(), IP_FORWARDING */
65 #include <netinet/ip_fw.h>
66 #include <netinet/ip_dummynet.h>
69 #include <netpfil/ipfw/ip_fw_private.h>
70 #include <netpfil/ipfw/dn_heap.h>
71 #include <netpfil/ipfw/ip_dn_private.h>
73 #include <netpfil/ipfw/dn_aqm.h>
75 #include <netpfil/ipfw/dn_sched.h>
77 /* which objects to copy */
78 #define DN_C_LINK 0x01
80 #define DN_C_FLOW 0x04
82 #define DN_C_QUEUE 0x10
84 /* we use this argument in case of a schk_new */
90 /*---- callout hooks. ----*/
91 static struct callout dn_timeout;
92 static int dn_tasks_started = 0;
94 static struct task dn_task;
95 static struct taskqueue *dn_tq = NULL;
97 /* global scheduler list */
99 CK_LIST_HEAD(, dn_alg) schedlist;
101 CK_LIST_HEAD(, dn_aqm) aqmlist; /* list of AQMs */
108 (void)arg; /* UNUSED */
109 taskqueue_enqueue(dn_tq, &dn_task);
118 callout_reset_sbt(&dn_timeout, tick_sbt, 0, dummynet, NULL,
119 C_HARDCLOCK | C_DIRECT_EXEC);
121 /*----- end of callout hooks -----*/
124 /* Return AQM descriptor for given type or name. */
125 static struct dn_aqm *
126 find_aqm_type(int type, char *name)
132 CK_LIST_FOREACH(d, &aqmlist, next) {
133 if (d->type == type || (name && !strcasecmp(d->name, name)))
136 return NULL; /* not found */
140 /* Return a scheduler descriptor given the type or name. */
141 static struct dn_alg *
142 find_sched_type(int type, char *name)
148 CK_LIST_FOREACH(d, &schedlist, next) {
149 if (d->type == type || (name && !strcasecmp(d->name, name)))
152 return NULL; /* not found */
156 ipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg)
159 const char *op = NULL;
167 } else if (oldv > hi) {
172 if (op && msg && bootverbose)
173 printf("%s %s to %d (was %d)\n", op, msg, *v, oldv);
177 /*---- flow_id mask, hash and compare functions ---*/
179 * The flow_id includes the 5-tuple, the queue/pipe number
180 * which we store in the extra area in host order,
181 * and for ipv6 also the flow_id6.
182 * XXX see if we want the tos byte (can store in 'flags')
184 static struct ipfw_flow_id *
185 flow_id_mask(struct ipfw_flow_id *mask, struct ipfw_flow_id *id)
187 int is_v6 = IS_IP6_FLOW_ID(id);
189 id->dst_port &= mask->dst_port;
190 id->src_port &= mask->src_port;
191 id->proto &= mask->proto;
192 id->extra &= mask->extra;
194 APPLY_MASK(&id->dst_ip6, &mask->dst_ip6);
195 APPLY_MASK(&id->src_ip6, &mask->src_ip6);
196 id->flow_id6 &= mask->flow_id6;
198 id->dst_ip &= mask->dst_ip;
199 id->src_ip &= mask->src_ip;
204 /* computes an OR of two masks, result in dst and also returned */
205 static struct ipfw_flow_id *
206 flow_id_or(struct ipfw_flow_id *src, struct ipfw_flow_id *dst)
208 int is_v6 = IS_IP6_FLOW_ID(dst);
210 dst->dst_port |= src->dst_port;
211 dst->src_port |= src->src_port;
212 dst->proto |= src->proto;
213 dst->extra |= src->extra;
215 #define OR_MASK(_d, _s) \
216 (_d)->__u6_addr.__u6_addr32[0] |= (_s)->__u6_addr.__u6_addr32[0]; \
217 (_d)->__u6_addr.__u6_addr32[1] |= (_s)->__u6_addr.__u6_addr32[1]; \
218 (_d)->__u6_addr.__u6_addr32[2] |= (_s)->__u6_addr.__u6_addr32[2]; \
219 (_d)->__u6_addr.__u6_addr32[3] |= (_s)->__u6_addr.__u6_addr32[3];
220 OR_MASK(&dst->dst_ip6, &src->dst_ip6);
221 OR_MASK(&dst->src_ip6, &src->src_ip6);
223 dst->flow_id6 |= src->flow_id6;
225 dst->dst_ip |= src->dst_ip;
226 dst->src_ip |= src->src_ip;
232 nonzero_mask(struct ipfw_flow_id *m)
234 if (m->dst_port || m->src_port || m->proto || m->extra)
236 if (IS_IP6_FLOW_ID(m)) {
238 m->dst_ip6.__u6_addr.__u6_addr32[0] ||
239 m->dst_ip6.__u6_addr.__u6_addr32[1] ||
240 m->dst_ip6.__u6_addr.__u6_addr32[2] ||
241 m->dst_ip6.__u6_addr.__u6_addr32[3] ||
242 m->src_ip6.__u6_addr.__u6_addr32[0] ||
243 m->src_ip6.__u6_addr.__u6_addr32[1] ||
244 m->src_ip6.__u6_addr.__u6_addr32[2] ||
245 m->src_ip6.__u6_addr.__u6_addr32[3] ||
248 return m->dst_ip || m->src_ip;
252 /* XXX we may want a better hash function */
254 flow_id_hash(struct ipfw_flow_id *id)
258 if (IS_IP6_FLOW_ID(id)) {
259 uint32_t *d = (uint32_t *)&id->dst_ip6;
260 uint32_t *s = (uint32_t *)&id->src_ip6;
261 i = (d[0] ) ^ (d[1]) ^
263 (d[0] >> 15) ^ (d[1] >> 15) ^
264 (d[2] >> 15) ^ (d[3] >> 15) ^
265 (s[0] << 1) ^ (s[1] << 1) ^
266 (s[2] << 1) ^ (s[3] << 1) ^
267 (s[0] << 16) ^ (s[1] << 16) ^
268 (s[2] << 16) ^ (s[3] << 16) ^
269 (id->dst_port << 1) ^ (id->src_port) ^
271 (id->proto ) ^ (id->flow_id6);
273 i = (id->dst_ip) ^ (id->dst_ip >> 15) ^
274 (id->src_ip << 1) ^ (id->src_ip >> 16) ^
276 (id->dst_port << 1) ^ (id->src_port) ^ (id->proto);
281 /* Like bcmp, returns 0 if ids match, 1 otherwise. */
283 flow_id_cmp(struct ipfw_flow_id *id1, struct ipfw_flow_id *id2)
285 int is_v6 = IS_IP6_FLOW_ID(id1);
288 if (IS_IP6_FLOW_ID(id2))
289 return 1; /* different address families */
291 return (id1->dst_ip == id2->dst_ip &&
292 id1->src_ip == id2->src_ip &&
293 id1->dst_port == id2->dst_port &&
294 id1->src_port == id2->src_port &&
295 id1->proto == id2->proto &&
296 id1->extra == id2->extra) ? 0 : 1;
300 !bcmp(&id1->dst_ip6,&id2->dst_ip6, sizeof(id1->dst_ip6)) &&
301 !bcmp(&id1->src_ip6,&id2->src_ip6, sizeof(id1->src_ip6)) &&
302 id1->dst_port == id2->dst_port &&
303 id1->src_port == id2->src_port &&
304 id1->proto == id2->proto &&
305 id1->extra == id2->extra &&
306 id1->flow_id6 == id2->flow_id6) ? 0 : 1;
308 /*--------- end of flow-id mask, hash and compare ---------*/
310 /*--- support functions for the qht hashtable ----
311 * Entries are hashed by flow-id
314 q_hash(uintptr_t key, int flags, void *arg)
316 /* compute the hash slot from the flow id */
317 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ?
318 &((struct dn_queue *)key)->ni.fid :
319 (struct ipfw_flow_id *)key;
321 return flow_id_hash(id);
325 q_match(void *obj, uintptr_t key, int flags, void *arg)
327 struct dn_queue *o = (struct dn_queue *)obj;
328 struct ipfw_flow_id *id2;
330 if (flags & DNHT_KEY_IS_OBJ) {
331 /* compare pointers */
332 id2 = &((struct dn_queue *)key)->ni.fid;
334 id2 = (struct ipfw_flow_id *)key;
336 return (0 == flow_id_cmp(&o->ni.fid, id2));
340 * create a new queue instance for the given 'key'.
343 q_new(uintptr_t key, int flags, void *arg)
345 struct dn_queue *q, *template = arg;
346 struct dn_fsk *fs = template->fs;
347 int size = sizeof(*q) + fs->sched->fp->q_datalen;
349 q = malloc(size, M_DUMMYNET, M_NOWAIT | M_ZERO);
351 D("no memory for new queue");
355 set_oid(&q->ni.oid, DN_QUEUE, size);
356 if (fs->fs.flags & DN_QHT_HASH)
357 q->ni.fid = *(struct ipfw_flow_id *)key;
359 q->_si = template->_si;
362 if (fs->sched->fp->new_queue)
363 fs->sched->fp->new_queue(q);
366 /* call AQM init function after creating a queue*/
367 if (fs->aqmfp && fs->aqmfp->init)
368 if(fs->aqmfp->init(q))
369 D("unable to init AQM for fs %d", fs->fs.fs_nr);
371 V_dn_cfg.queue_count++;
377 * Notify schedulers that a queue is going away.
378 * If (flags & DN_DESTROY), also free the packets.
379 * The version for callbacks is called q_delete_cb().
382 dn_delete_queue(struct dn_queue *q, int flags)
384 struct dn_fsk *fs = q->fs;
387 /* clean up AQM status for queue 'q'
388 * cleanup here is called just with MULTIQUEUE
390 if (fs && fs->aqmfp && fs->aqmfp->cleanup)
391 fs->aqmfp->cleanup(q);
393 // D("fs %p si %p\n", fs, q->_si);
394 /* notify the parent scheduler that the queue is going away */
395 if (fs && fs->sched->fp->free_queue)
396 fs->sched->fp->free_queue(q);
399 if (flags & DN_DESTROY) {
401 dn_free_pkts(q->mq.head);
402 bzero(q, sizeof(*q)); // safety
404 V_dn_cfg.queue_count--;
409 q_delete_cb(void *q, void *arg)
411 int flags = (int)(uintptr_t)arg;
412 dn_delete_queue(q, flags);
413 return (flags & DN_DESTROY) ? DNHT_SCAN_DEL : 0;
417 * calls dn_delete_queue/q_delete_cb on all queues,
418 * which notifies the parent scheduler and possibly drains packets.
419 * flags & DN_DESTROY: drains queues and destroy qht;
422 qht_delete(struct dn_fsk *fs, int flags)
424 ND("fs %d start flags %d qht %p",
425 fs->fs.fs_nr, flags, fs->qht);
428 if (fs->fs.flags & DN_QHT_HASH) {
429 dn_ht_scan(fs->qht, q_delete_cb, (void *)(uintptr_t)flags);
430 if (flags & DN_DESTROY) {
431 dn_ht_free(fs->qht, 0);
435 dn_delete_queue((struct dn_queue *)(fs->qht), flags);
436 if (flags & DN_DESTROY)
442 * Find and possibly create the queue for a MULTIQUEUE scheduler.
443 * We never call it for !MULTIQUEUE (the queue is in the sch_inst).
446 ipdn_q_find(struct dn_fsk *fs, struct dn_sch_inst *si,
447 struct ipfw_flow_id *id)
449 struct dn_queue template;
454 if (fs->fs.flags & DN_QHT_HASH) {
455 struct ipfw_flow_id masked_id;
456 if (fs->qht == NULL) {
457 fs->qht = dn_ht_init(NULL, fs->fs.buckets,
458 offsetof(struct dn_queue, q_next),
459 q_hash, q_match, q_new);
464 flow_id_mask(&fs->fsk_mask, &masked_id);
465 return dn_ht_find(fs->qht, (uintptr_t)&masked_id,
466 DNHT_INSERT, &template);
469 fs->qht = q_new(0, 0, &template);
470 return (struct dn_queue *)fs->qht;
473 /*--- end of queue hash table ---*/
475 /*--- support functions for the sch_inst hashtable ----
477 * These are hashed by flow-id
480 si_hash(uintptr_t key, int flags, void *arg)
482 /* compute the hash slot from the flow id */
483 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ?
484 &((struct dn_sch_inst *)key)->ni.fid :
485 (struct ipfw_flow_id *)key;
487 return flow_id_hash(id);
491 si_match(void *obj, uintptr_t key, int flags, void *arg)
493 struct dn_sch_inst *o = obj;
494 struct ipfw_flow_id *id2;
496 id2 = (flags & DNHT_KEY_IS_OBJ) ?
497 &((struct dn_sch_inst *)key)->ni.fid :
498 (struct ipfw_flow_id *)key;
499 return flow_id_cmp(&o->ni.fid, id2) == 0;
503 * create a new instance for the given 'key'
504 * Allocate memory for instance, delay line and scheduler private data.
507 si_new(uintptr_t key, int flags, void *arg)
509 struct dn_schk *s = arg;
510 struct dn_sch_inst *si;
511 int l = sizeof(*si) + s->fp->si_datalen;
513 si = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO);
517 /* Set length only for the part passed up to userland. */
518 set_oid(&si->ni.oid, DN_SCH_I, sizeof(struct dn_flow));
519 set_oid(&(si->dline.oid), DN_DELAY_LINE,
520 sizeof(struct delay_line));
521 /* mark si and dline as outside the event queue */
522 si->ni.oid.id = si->dline.oid.id = -1;
527 if (s->fp->new_sched && s->fp->new_sched(si)) {
528 D("new_sched error");
531 if (s->sch.flags & DN_HAVE_MASK)
532 si->ni.fid = *(struct ipfw_flow_id *)key;
535 /* init AQM status for !DN_MULTIQUEUE sched*/
536 if (!(s->fp->flags & DN_MULTIQUEUE))
537 if (s->fs->aqmfp && s->fs->aqmfp->init)
538 if(s->fs->aqmfp->init((struct dn_queue *)(si + 1))) {
539 D("unable to init AQM for fs %d", s->fs->fs.fs_nr);
549 bzero(si, sizeof(*si)); // safety
550 free(si, M_DUMMYNET);
556 * Callback from siht to delete all scheduler instances. Remove
557 * si and delay line from the system heap, destroy all queues.
558 * We assume that all flowset have been notified and do not
559 * point to us anymore.
562 si_destroy(void *_si, void *arg)
564 struct dn_sch_inst *si = _si;
565 struct dn_schk *s = si->sched;
566 struct delay_line *dl = &si->dline;
568 if (dl->oid.subtype) /* remove delay line from event heap */
569 heap_extract(&V_dn_cfg.evheap, dl);
570 dn_free_pkts(dl->mq.head); /* drain delay line */
571 if (si->kflags & DN_ACTIVE) /* remove si from event heap */
572 heap_extract(&V_dn_cfg.evheap, si);
575 /* clean up AQM status for !DN_MULTIQUEUE sched
576 * Note that all queues belong to fs were cleaned up in fsk_detach.
577 * When drain_scheduler is called s->fs and q->fs are pointing
578 * to a correct fs, so we can use fs in this case.
580 if (!(s->fp->flags & DN_MULTIQUEUE)) {
581 struct dn_queue *q = (struct dn_queue *)(si + 1);
582 if (q->aqm_status && q->fs->aqmfp)
583 if (q->fs->aqmfp->cleanup)
584 q->fs->aqmfp->cleanup(q);
587 if (s->fp->free_sched)
588 s->fp->free_sched(si);
589 bzero(si, sizeof(*si)); /* safety */
590 free(si, M_DUMMYNET);
592 return DNHT_SCAN_DEL;
596 * Find the scheduler instance for this packet. If we need to apply
597 * a mask, do on a local copy of the flow_id to preserve the original.
598 * Assume siht is always initialized if we have a mask.
601 ipdn_si_find(struct dn_schk *s, struct ipfw_flow_id *id)
604 if (s->sch.flags & DN_HAVE_MASK) {
605 struct ipfw_flow_id id_t = *id;
606 flow_id_mask(&s->sch.sched_mask, &id_t);
607 return dn_ht_find(s->siht, (uintptr_t)&id_t,
611 s->siht = si_new(0, 0, s);
612 return (struct dn_sch_inst *)s->siht;
615 /* callback to flush credit for the scheduler instance */
617 si_reset_credit(void *_si, void *arg)
619 struct dn_sch_inst *si = _si;
620 struct dn_link *p = &si->sched->link;
622 si->credit = p->burst + (V_dn_cfg.io_fast ? p->bandwidth : 0);
627 schk_reset_credit(struct dn_schk *s)
629 if (s->sch.flags & DN_HAVE_MASK)
630 dn_ht_scan(s->siht, si_reset_credit, NULL);
632 si_reset_credit(s->siht, NULL);
634 /*---- end of sch_inst hashtable ---------------------*/
636 /*-------------------------------------------------------
637 * flowset hash (fshash) support. Entries are hashed by fs_nr.
638 * New allocations are put in the fsunlinked list, from which
639 * they are removed when they point to a specific scheduler.
642 fsk_hash(uintptr_t key, int flags, void *arg)
644 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key :
645 ((struct dn_fsk *)key)->fs.fs_nr;
647 return ( (i>>8)^(i>>4)^i );
651 fsk_match(void *obj, uintptr_t key, int flags, void *arg)
653 struct dn_fsk *fs = obj;
654 int i = !(flags & DNHT_KEY_IS_OBJ) ? key :
655 ((struct dn_fsk *)key)->fs.fs_nr;
657 return (fs->fs.fs_nr == i);
661 fsk_new(uintptr_t key, int flags, void *arg)
665 fs = malloc(sizeof(*fs), M_DUMMYNET, M_NOWAIT | M_ZERO);
667 set_oid(&fs->fs.oid, DN_FS, sizeof(fs->fs));
668 V_dn_cfg.fsk_count++;
669 fs->drain_bucket = 0;
670 SLIST_INSERT_HEAD(&V_dn_cfg.fsu, fs, sch_chain);
676 /* callback function for cleaning up AQM queue status belongs to a flowset
677 * connected to scheduler instance '_si' (for !DN_MULTIQUEUE only).
680 si_cleanup_q(void *_si, void *arg)
682 struct dn_sch_inst *si = _si;
684 if (!(si->sched->fp->flags & DN_MULTIQUEUE)) {
685 if (si->sched->fs->aqmfp && si->sched->fs->aqmfp->cleanup)
686 si->sched->fs->aqmfp->cleanup((struct dn_queue *) (si+1));
691 /* callback to clean up queue AQM status.*/
693 q_cleanup_q(void *_q, void *arg)
695 struct dn_queue *q = _q;
696 q->fs->aqmfp->cleanup(q);
700 /* Clean up all AQM queues status belongs to flowset 'fs' and then
701 * deconfig AQM for flowset 'fs'
704 aqm_cleanup_deconfig_fs(struct dn_fsk *fs)
706 struct dn_sch_inst *si;
708 /* clean up AQM status for all queues for !DN_MULTIQUEUE sched*/
709 if (fs->fs.fs_nr > DN_MAX_ID) {
710 if (fs->sched && !(fs->sched->fp->flags & DN_MULTIQUEUE)) {
711 if (fs->sched->sch.flags & DN_HAVE_MASK)
712 dn_ht_scan(fs->sched->siht, si_cleanup_q, NULL);
714 /* single si i.e. no sched mask */
715 si = (struct dn_sch_inst *) fs->sched->siht;
716 if (si && fs->aqmfp && fs->aqmfp->cleanup)
717 fs->aqmfp->cleanup((struct dn_queue *) (si+1));
722 /* clean up AQM status for all queues for DN_MULTIQUEUE sched*/
723 if (fs->sched && fs->sched->fp->flags & DN_MULTIQUEUE && fs->qht) {
724 if (fs->fs.flags & DN_QHT_HASH)
725 dn_ht_scan(fs->qht, q_cleanup_q, NULL);
727 fs->aqmfp->cleanup((struct dn_queue *)(fs->qht));
731 if(fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig)
732 fs->aqmfp->deconfig(fs);
737 * detach flowset from its current scheduler. Flags as follows:
738 * DN_DETACH removes from the fsk_list
739 * DN_DESTROY deletes individual queues
740 * DN_DELETE_FS destroys the flowset (otherwise goes in unlinked).
743 fsk_detach(struct dn_fsk *fs, int flags)
745 if (flags & DN_DELETE_FS)
747 ND("fs %d from sched %d flags %s %s %s",
748 fs->fs.fs_nr, fs->fs.sched_nr,
749 (flags & DN_DELETE_FS) ? "DEL_FS":"",
750 (flags & DN_DESTROY) ? "DEL":"",
751 (flags & DN_DETACH) ? "DET":"");
752 if (flags & DN_DETACH) { /* detach from the list */
753 struct dn_fsk_head *h;
754 h = fs->sched ? &fs->sched->fsk_list : &V_dn_cfg.fsu;
755 SLIST_REMOVE(h, fs, dn_fsk, sch_chain);
757 /* Free the RED parameters, they will be recomputed on
758 * subsequent attach if needed.
760 free(fs->w_q_lookup, M_DUMMYNET);
761 fs->w_q_lookup = NULL;
762 qht_delete(fs, flags);
764 aqm_cleanup_deconfig_fs(fs);
767 if (fs->sched && fs->sched->fp->free_fsk)
768 fs->sched->fp->free_fsk(fs);
770 if (flags & DN_DELETE_FS) {
771 bzero(fs, sizeof(*fs)); /* safety */
772 free(fs, M_DUMMYNET);
773 V_dn_cfg.fsk_count--;
775 SLIST_INSERT_HEAD(&V_dn_cfg.fsu, fs, sch_chain);
780 * Detach or destroy all flowsets in a list.
781 * flags specifies what to do:
782 * DN_DESTROY: flush all queues
783 * DN_DELETE_FS: DN_DESTROY + destroy flowset
784 * DN_DELETE_FS implies DN_DESTROY
787 fsk_detach_list(struct dn_fsk_head *h, int flags)
790 int n = 0; /* only for stats */
792 ND("head %p flags %x", h, flags);
793 while ((fs = SLIST_FIRST(h))) {
794 SLIST_REMOVE_HEAD(h, sch_chain);
796 fsk_detach(fs, flags);
798 ND("done %d flowsets", n);
802 * called on 'queue X delete' -- removes the flowset from fshash,
803 * deletes all queues for the flowset, and removes the flowset.
806 delete_fs(int i, int locked)
813 fs = dn_ht_find(V_dn_cfg.fshash, i, DNHT_REMOVE, NULL);
814 ND("fs %d found %p", i, fs);
816 fsk_detach(fs, DN_DETACH | DN_DELETE_FS);
825 /*----- end of flowset hashtable support -------------*/
827 /*------------------------------------------------------------
828 * Scheduler hash. When searching by index we pass sched_nr,
829 * otherwise we pass struct dn_sch * which is the first field in
830 * struct dn_schk so we can cast between the two. We use this trick
831 * because in the create phase (but it should be fixed).
834 schk_hash(uintptr_t key, int flags, void *_arg)
836 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key :
837 ((struct dn_schk *)key)->sch.sched_nr;
838 return ( (i>>8)^(i>>4)^i );
842 schk_match(void *obj, uintptr_t key, int flags, void *_arg)
844 struct dn_schk *s = (struct dn_schk *)obj;
845 int i = !(flags & DNHT_KEY_IS_OBJ) ? key :
846 ((struct dn_schk *)key)->sch.sched_nr;
847 return (s->sch.sched_nr == i);
851 * Create the entry and intialize with the sched hash if needed.
852 * Leave s->fp unset so we can tell whether a dn_ht_find() returns
853 * a new object or a previously existing one.
856 schk_new(uintptr_t key, int flags, void *arg)
858 struct schk_new_arg *a = arg;
860 int l = sizeof(*s) +a->fp->schk_datalen;
862 s = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO);
865 set_oid(&s->link.oid, DN_LINK, sizeof(s->link));
866 s->sch = *a->sch; // copy initial values
867 s->link.link_nr = s->sch.sched_nr;
868 SLIST_INIT(&s->fsk_list);
869 /* initialize the hash table or create the single instance */
870 s->fp = a->fp; /* si_new needs this */
872 if (s->sch.flags & DN_HAVE_MASK) {
873 s->siht = dn_ht_init(NULL, s->sch.buckets,
874 offsetof(struct dn_sch_inst, si_next),
875 si_hash, si_match, si_new);
876 if (s->siht == NULL) {
881 s->fp = NULL; /* mark as a new scheduler */
882 V_dn_cfg.schk_count++;
887 * Callback for sched delete. Notify all attached flowsets to
888 * detach from the scheduler, destroy the internal flowset, and
889 * all instances. The scheduler goes away too.
890 * arg is 0 (only detach flowsets and destroy instances)
891 * DN_DESTROY (detach & delete queues, delete schk)
892 * or DN_DELETE_FS (delete queues and flowsets, delete schk)
895 schk_delete_cb(void *obj, void *arg)
897 struct dn_schk *s = obj;
900 ND("sched %d arg %s%s",
902 a&DN_DESTROY ? "DEL ":"",
903 a&DN_DELETE_FS ? "DEL_FS":"");
905 fsk_detach_list(&s->fsk_list, arg ? DN_DESTROY : 0);
906 /* no more flowset pointing to us now */
907 if (s->sch.flags & DN_HAVE_MASK) {
908 dn_ht_scan(s->siht, si_destroy, NULL);
909 dn_ht_free(s->siht, 0);
911 si_destroy(s->siht, NULL);
913 free(s->profile, M_DUMMYNET);
918 bzero(s, sizeof(*s)); // safety
919 free(obj, M_DUMMYNET);
920 V_dn_cfg.schk_count--;
921 return DNHT_SCAN_DEL;
925 * called on a 'sched X delete' command. Deletes a single scheduler.
926 * This is done by removing from the schedhash, unlinking all
927 * flowsets and deleting their traffic.
934 s = dn_ht_find(V_dn_cfg.schedhash, i, DNHT_REMOVE, NULL);
938 delete_fs(i + DN_MAX_ID, 1); /* first delete internal fs */
939 /* then detach flowsets, delete traffic */
940 schk_delete_cb(s, (void*)(uintptr_t)DN_DESTROY);
943 /*--- end of schk hashtable support ---*/
946 copy_obj(char **start, char *end, void *_o, const char *msg, int i)
953 int have = end - *start;
955 memcpy(&o, _o, sizeof(o));
956 if (have < o.len || o.len == 0 || o.type == 0) {
957 D("(WARN) type %d %s %d have %d need %d",
958 o.type, msg, i, have, o.len);
961 ND("type %d %s %d len %d", o.type, msg, i, o.len);
962 if (o.type == DN_LINK) {
963 memcpy(&dn.l, _o, sizeof(dn.l));
964 /* Adjust burst parameter for link */
965 dn.l.burst = div64(dn.l.burst, 8 * hz);
966 dn.l.delay = dn.l.delay * 1000 / hz;
967 memcpy(*start, &dn.l, sizeof(dn.l));
968 } else if (o.type == DN_SCH) {
969 /* Set dn.s.sch.oid.id to the number of instances */
970 memcpy(&dn.s, _o, sizeof(dn.s));
971 dn.s.sch.oid.id = (dn.s.sch.flags & DN_HAVE_MASK) ?
972 dn_ht_entries(dn.s.siht) : (dn.s.siht ? 1 : 0);
973 memcpy(*start, &dn.s, sizeof(dn.s));
975 memcpy(*start, _o, o.len);
980 /* Specific function to copy a queue.
981 * Copies only the user-visible part of a queue (which is in
982 * a struct dn_flow), and sets len accordingly.
985 copy_obj_q(char **start, char *end, void *_o, const char *msg, int i)
987 struct dn_id *o = _o;
988 int have = end - *start;
989 int len = sizeof(struct dn_flow); /* see above comment */
991 if (have < len || o->len == 0 || o->type != DN_QUEUE) {
992 D("ERROR type %d %s %d have %d need %d",
993 o->type, msg, i, have, len);
996 ND("type %d %s %d len %d", o->type, msg, i, len);
997 memcpy(*start, _o, len);
998 ((struct dn_id*)(*start))->len = len;
1004 copy_q_cb(void *obj, void *arg)
1006 struct dn_queue *q = obj;
1007 struct copy_args *a = arg;
1008 struct dn_flow *ni = (struct dn_flow *)(*a->start);
1009 if (copy_obj_q(a->start, a->end, &q->ni, "queue", -1))
1010 return DNHT_SCAN_END;
1011 ni->oid.type = DN_FLOW; /* override the DN_QUEUE */
1012 ni->oid.id = si_hash((uintptr_t)&ni->fid, 0, NULL);
1017 copy_q(struct copy_args *a, struct dn_fsk *fs, int flags)
1021 if (fs->fs.flags & DN_QHT_HASH)
1022 dn_ht_scan(fs->qht, copy_q_cb, a);
1024 copy_q_cb(fs->qht, a);
1029 * This routine only copies the initial part of a profile ? XXX
1032 copy_profile(struct copy_args *a, struct dn_profile *p)
1034 int have = a->end - *a->start;
1035 /* XXX here we check for max length */
1036 int profile_len = sizeof(struct dn_profile) -
1037 ED_MAX_SAMPLES_NO*sizeof(int);
1041 if (have < profile_len) {
1042 D("error have %d need %d", have, profile_len);
1045 memcpy(*a->start, p, profile_len);
1046 ((struct dn_id *)(*a->start))->len = profile_len;
1047 *a->start += profile_len;
1052 copy_flowset(struct copy_args *a, struct dn_fsk *fs, int flags)
1054 struct dn_fs *ufs = (struct dn_fs *)(*a->start);
1057 ND("flowset %d", fs->fs.fs_nr);
1058 if (copy_obj(a->start, a->end, &fs->fs, "flowset", fs->fs.fs_nr))
1059 return DNHT_SCAN_END;
1060 ufs->oid.id = (fs->fs.flags & DN_QHT_HASH) ?
1061 dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0);
1062 if (flags) { /* copy queues */
1069 copy_si_cb(void *obj, void *arg)
1071 struct dn_sch_inst *si = obj;
1072 struct copy_args *a = arg;
1073 struct dn_flow *ni = (struct dn_flow *)(*a->start);
1074 if (copy_obj(a->start, a->end, &si->ni, "inst",
1075 si->sched->sch.sched_nr))
1076 return DNHT_SCAN_END;
1077 ni->oid.type = DN_FLOW; /* override the DN_SCH_I */
1078 ni->oid.id = si_hash((uintptr_t)si, DNHT_KEY_IS_OBJ, NULL);
1083 copy_si(struct copy_args *a, struct dn_schk *s, int flags)
1085 if (s->sch.flags & DN_HAVE_MASK)
1086 dn_ht_scan(s->siht, copy_si_cb, a);
1088 copy_si_cb(s->siht, a);
1093 * compute a list of children of a scheduler and copy up
1096 copy_fsk_list(struct copy_args *a, struct dn_schk *s, int flags)
1102 int n = 0, space = sizeof(*o);
1103 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) {
1104 if (fs->fs.fs_nr < DN_MAX_ID)
1107 space += n * sizeof(uint32_t);
1108 DX(3, "sched %d has %d flowsets", s->sch.sched_nr, n);
1109 if (a->end - *(a->start) < space)
1110 return DNHT_SCAN_END;
1111 o = (struct dn_id *)(*(a->start));
1113 *a->start += o->len;
1115 p = (uint32_t *)(o+1);
1116 SLIST_FOREACH(fs, &s->fsk_list, sch_chain)
1117 if (fs->fs.fs_nr < DN_MAX_ID)
1118 *p++ = fs->fs.fs_nr;
1123 copy_data_helper(void *_o, void *_arg)
1125 struct copy_args *a = _arg;
1126 uint32_t *r = a->extra->r; /* start of first range */
1127 uint32_t *lim; /* first invalid pointer */
1130 lim = (uint32_t *)((char *)(a->extra) + a->extra->o.len);
1132 if (a->type == DN_LINK || a->type == DN_SCH) {
1133 /* pipe|sched show, we receive a dn_schk */
1134 struct dn_schk *s = _o;
1136 n = s->sch.sched_nr;
1137 if (a->type == DN_SCH && n >= DN_MAX_ID)
1138 return 0; /* not a scheduler */
1139 if (a->type == DN_LINK && n <= DN_MAX_ID)
1140 return 0; /* not a pipe */
1142 /* see if the object is within one of our ranges */
1143 for (;r < lim; r += 2) {
1144 if (n < r[0] || n > r[1])
1146 /* Found a valid entry, copy and we are done */
1147 if (a->flags & DN_C_LINK) {
1148 if (copy_obj(a->start, a->end,
1149 &s->link, "link", n))
1150 return DNHT_SCAN_END;
1151 if (copy_profile(a, s->profile))
1152 return DNHT_SCAN_END;
1153 if (copy_flowset(a, s->fs, 0))
1154 return DNHT_SCAN_END;
1156 if (a->flags & DN_C_SCH) {
1157 if (copy_obj(a->start, a->end,
1158 &s->sch, "sched", n))
1159 return DNHT_SCAN_END;
1160 /* list all attached flowsets */
1161 if (copy_fsk_list(a, s, 0))
1162 return DNHT_SCAN_END;
1164 if (a->flags & DN_C_FLOW)
1168 } else if (a->type == DN_FS) {
1169 /* queue show, skip internal flowsets */
1170 struct dn_fsk *fs = _o;
1175 /* see if the object is within one of our ranges */
1176 for (;r < lim; r += 2) {
1177 if (n < r[0] || n > r[1])
1179 if (copy_flowset(a, fs, 0))
1180 return DNHT_SCAN_END;
1182 break; /* we are done */
1188 static inline struct dn_schk *
1189 locate_scheduler(int i)
1191 return dn_ht_find(V_dn_cfg.schedhash, i, 0, NULL);
1195 * red parameters are in fixed point arithmetic.
1198 config_red(struct dn_fsk *fs)
1200 int64_t s, idle, weight, w0;
1203 fs->w_q = fs->fs.w_q;
1204 fs->max_p = fs->fs.max_p;
1206 /* Doing stuff that was in userland */
1207 i = fs->sched->link.bandwidth;
1209 hz * V_dn_cfg.red_avg_pkt_size * 8 * SCALE(1) / i;
1211 idle = div64((s * 3) , fs->w_q); /* s, fs->w_q scaled; idle not scaled */
1212 fs->lookup_step = div64(idle , V_dn_cfg.red_lookup_depth);
1213 /* fs->lookup_step not scaled, */
1214 if (!fs->lookup_step)
1215 fs->lookup_step = 1;
1216 w0 = weight = SCALE(1) - fs->w_q; //fs->w_q scaled
1218 for (t = fs->lookup_step; t > 1; --t)
1219 weight = SCALE_MUL(weight, w0);
1220 fs->lookup_weight = (int)(weight); // scaled
1222 /* Now doing stuff that was in kerneland */
1223 fs->min_th = SCALE(fs->fs.min_th);
1224 fs->max_th = SCALE(fs->fs.max_th);
1226 if (fs->fs.max_th == fs->fs.min_th)
1227 fs->c_1 = fs->max_p;
1229 fs->c_1 = SCALE((int64_t)(fs->max_p)) / (fs->fs.max_th - fs->fs.min_th);
1230 fs->c_2 = SCALE_MUL(fs->c_1, SCALE(fs->fs.min_th));
1232 if (fs->fs.flags & DN_IS_GENTLE_RED) {
1233 fs->c_3 = (SCALE(1) - fs->max_p) / fs->fs.max_th;
1234 fs->c_4 = SCALE(1) - 2 * fs->max_p;
1237 /* If the lookup table already exist, free and create it again. */
1238 free(fs->w_q_lookup, M_DUMMYNET);
1239 fs->w_q_lookup = NULL;
1240 if (V_dn_cfg.red_lookup_depth == 0) {
1241 printf("\ndummynet: net.inet.ip.dummynet.red_lookup_depth"
1243 fs->fs.flags &= ~DN_IS_RED;
1244 fs->fs.flags &= ~DN_IS_GENTLE_RED;
1247 fs->lookup_depth = V_dn_cfg.red_lookup_depth;
1248 fs->w_q_lookup = (u_int *)malloc(fs->lookup_depth * sizeof(int),
1249 M_DUMMYNET, M_NOWAIT);
1250 if (fs->w_q_lookup == NULL) {
1251 printf("dummynet: sorry, cannot allocate red lookup table\n");
1252 fs->fs.flags &= ~DN_IS_RED;
1253 fs->fs.flags &= ~DN_IS_GENTLE_RED;
1257 /* Fill the lookup table with (1 - w_q)^x */
1258 fs->w_q_lookup[0] = SCALE(1) - fs->w_q;
1260 for (i = 1; i < fs->lookup_depth; i++)
1262 SCALE_MUL(fs->w_q_lookup[i - 1], fs->lookup_weight);
1264 if (V_dn_cfg.red_avg_pkt_size < 1)
1265 V_dn_cfg.red_avg_pkt_size = 512;
1266 fs->avg_pkt_size = V_dn_cfg.red_avg_pkt_size;
1267 if (V_dn_cfg.red_max_pkt_size < 1)
1268 V_dn_cfg.red_max_pkt_size = 1500;
1269 fs->max_pkt_size = V_dn_cfg.red_max_pkt_size;
1274 /* Scan all flowset attached to this scheduler and update red */
1276 update_red(struct dn_schk *s)
1279 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) {
1280 if (fs && (fs->fs.flags & DN_IS_RED))
1285 /* attach flowset to scheduler s, possibly requeue */
1287 fsk_attach(struct dn_fsk *fs, struct dn_schk *s)
1289 ND("remove fs %d from fsunlinked, link to sched %d",
1290 fs->fs.fs_nr, s->sch.sched_nr);
1291 SLIST_REMOVE(&V_dn_cfg.fsu, fs, dn_fsk, sch_chain);
1293 SLIST_INSERT_HEAD(&s->fsk_list, fs, sch_chain);
1296 /* XXX compute fsk_mask */
1297 fs->fsk_mask = fs->fs.flow_mask;
1298 if (fs->sched->sch.flags & DN_HAVE_MASK)
1299 flow_id_or(&fs->sched->sch.sched_mask, &fs->fsk_mask);
1302 * we must drain qht according to the old
1303 * type, and reinsert according to the new one.
1304 * The requeue is complex -- in general we need to
1305 * reclassify every single packet.
1306 * For the time being, let's hope qht is never set
1307 * when we reach this point.
1309 D("XXX TODO requeue from fs %d to sch %d",
1310 fs->fs.fs_nr, s->sch.sched_nr);
1313 /* set the new type for qht */
1314 if (nonzero_mask(&fs->fsk_mask))
1315 fs->fs.flags |= DN_QHT_HASH;
1317 fs->fs.flags &= ~DN_QHT_HASH;
1319 /* XXX config_red() can fail... */
1320 if (fs->fs.flags & DN_IS_RED)
1324 /* update all flowsets which may refer to this scheduler */
1326 update_fs(struct dn_schk *s)
1328 struct dn_fsk *fs, *tmp;
1330 SLIST_FOREACH_SAFE(fs, &V_dn_cfg.fsu, sch_chain, tmp) {
1331 if (s->sch.sched_nr != fs->fs.sched_nr) {
1332 D("fs %d for sch %d not %d still unlinked",
1333 fs->fs.fs_nr, fs->fs.sched_nr,
1342 /* Retrieve AQM configurations to ipfw userland
1345 get_aqm_parms(struct sockopt *sopt)
1347 struct dn_extra_parms *ep;
1349 size_t sopt_valsize;
1352 sopt_valsize = sopt->sopt_valsize;
1354 if (sopt->sopt_valsize < l) {
1355 D("bad len sopt->sopt_valsize %d len %d",
1356 (int) sopt->sopt_valsize , l);
1360 ep = malloc(l, M_DUMMYNET, M_NOWAIT);
1366 err = sooptcopyin(sopt, ep, l, l);
1369 sopt->sopt_valsize = sopt_valsize;
1370 if (ep->oid.len < l) {
1375 fs = dn_ht_find(V_dn_cfg.fshash, ep->nr, 0, NULL);
1377 D("fs %d not found", ep->nr);
1382 if (fs->aqmfp && fs->aqmfp->getconfig) {
1383 if(fs->aqmfp->getconfig(fs, ep)) {
1384 D("Error while trying to get AQM params");
1389 err = sooptcopyout(sopt, ep, l);
1393 free(ep, M_DUMMYNET);
1397 /* Retrieve AQM configurations to ipfw userland
1400 get_sched_parms(struct sockopt *sopt)
1402 struct dn_extra_parms *ep;
1403 struct dn_schk *schk;
1404 size_t sopt_valsize;
1407 sopt_valsize = sopt->sopt_valsize;
1409 if (sopt->sopt_valsize < l) {
1410 D("bad len sopt->sopt_valsize %d len %d",
1411 (int) sopt->sopt_valsize , l);
1415 ep = malloc(l, M_DUMMYNET, M_NOWAIT);
1421 err = sooptcopyin(sopt, ep, l, l);
1424 sopt->sopt_valsize = sopt_valsize;
1425 if (ep->oid.len < l) {
1430 schk = locate_scheduler(ep->nr);
1432 D("sched %d not found", ep->nr);
1437 if (schk->fp && schk->fp->getconfig) {
1438 if(schk->fp->getconfig(schk, ep)) {
1439 D("Error while trying to get sched params");
1444 err = sooptcopyout(sopt, ep, l);
1447 free(ep, M_DUMMYNET);
1452 /* Configure AQM for flowset 'fs'.
1453 * extra parameters are passed from userland.
1456 config_aqm(struct dn_fsk *fs, struct dn_extra_parms *ep, int busy)
1463 /* no configurations */
1469 /* no AQM for this flowset*/
1470 if (!strcmp(ep->name,"")) {
1474 if (ep->oid.len < sizeof(*ep)) {
1475 D("short aqm len %d", ep->oid.len);
1481 D("Unable to configure flowset, flowset busy!");
1486 /* deconfigure old aqm if exist */
1487 if (fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig) {
1488 aqm_cleanup_deconfig_fs(fs);
1491 if (!(fs->aqmfp = find_aqm_type(0, ep->name))) {
1492 D("AQM functions not found for type %s!", ep->name);
1493 fs->fs.flags &= ~DN_IS_AQM;
1497 fs->fs.flags |= DN_IS_AQM;
1499 if (ep->oid.subtype != DN_AQM_PARAMS) {
1505 if (fs->aqmfp->config) {
1506 err = fs->aqmfp->config(fs, ep, ep->oid.len);
1508 D("Unable to configure AQM for FS %d", fs->fs.fs_nr );
1509 fs->fs.flags &= ~DN_IS_AQM;
1521 * Configuration -- to preserve backward compatibility we use
1522 * the following scheme (N is 65536)
1523 * NUMBER SCHED LINK FLOWSET
1524 * 1 .. N-1 (1)WFQ (2)WFQ (3)queue
1525 * N+1 .. 2N-1 (4)FIFO (5)FIFO (6)FIFO for sched 1..N-1
1526 * 2N+1 .. 3N-1 -- -- (7)FIFO for sched N+1..2N-1
1528 * "pipe i config" configures #1, #2 and #3
1529 * "sched i config" configures #1 and possibly #6
1530 * "queue i config" configures #3
1531 * #1 is configured with 'pipe i config' or 'sched i config'
1532 * #2 is configured with 'pipe i config', and created if not
1533 * existing with 'sched i config'
1534 * #3 is configured with 'queue i config'
1535 * #4 is automatically configured after #1, can only be FIFO
1536 * #5 is automatically configured after #2
1537 * #6 is automatically created when #1 is !MULTIQUEUE,
1538 * and can be updated.
1539 * #7 is automatically configured after #2
1543 * configure a link (and its FIFO instance)
1546 config_link(struct dn_link *p, struct dn_id *arg)
1550 if (p->oid.len != sizeof(*p)) {
1551 D("invalid pipe len %d", p->oid.len);
1555 if (i <= 0 || i >= DN_MAX_ID)
1558 * The config program passes parameters as follows:
1559 * bw = bits/second (0 means no limits),
1560 * delay = ms, must be translated into ticks.
1561 * qsize = slots/bytes
1564 p->delay = (p->delay * hz) / 1000;
1565 /* Scale burst size: bytes -> bits * hz */
1569 /* do it twice, base link and FIFO link */
1570 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) {
1571 struct dn_schk *s = locate_scheduler(i);
1574 D("sched %d not found", i);
1577 /* remove profile if exists */
1578 free(s->profile, M_DUMMYNET);
1581 /* copy all parameters */
1582 s->link.oid = p->oid;
1583 s->link.link_nr = i;
1584 s->link.delay = p->delay;
1585 if (s->link.bandwidth != p->bandwidth) {
1586 /* XXX bandwidth changes, need to update red params */
1587 s->link.bandwidth = p->bandwidth;
1590 s->link.burst = p->burst;
1591 schk_reset_credit(s);
1599 * configure a flowset. Can be called from inside with locked=1,
1601 static struct dn_fsk *
1602 config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked)
1607 struct dn_extra_parms *ep;
1610 if (nfs->oid.len != sizeof(*nfs)) {
1611 D("invalid flowset len %d", nfs->oid.len);
1615 if (i <= 0 || i >= 3*DN_MAX_ID)
1620 ep = malloc(sizeof(*ep), M_TEMP, M_NOWAIT);
1623 memcpy(ep, arg, sizeof(*ep));
1626 ND("flowset %d", i);
1627 /* XXX other sanity checks */
1628 if (nfs->flags & DN_QSIZE_BYTES) {
1629 ipdn_bound_var(&nfs->qsize, 16384,
1630 1500, V_dn_cfg.byte_limit, NULL); // "queue byte size");
1632 ipdn_bound_var(&nfs->qsize, 50,
1633 1, V_dn_cfg.slot_limit, NULL); // "queue slot size");
1635 if (nfs->flags & DN_HAVE_MASK) {
1636 /* make sure we have some buckets */
1637 ipdn_bound_var((int *)&nfs->buckets, V_dn_cfg.hash_size,
1638 1, V_dn_cfg.max_hash_size, "flowset buckets");
1640 nfs->buckets = 1; /* we only need 1 */
1644 do { /* exit with break when done */
1646 int flags = nfs->sched_nr ? DNHT_INSERT : 0;
1648 int oldc = V_dn_cfg.fsk_count;
1649 fs = dn_ht_find(V_dn_cfg.fshash, i, flags, NULL);
1651 D("missing sched for flowset %d", i);
1654 /* grab some defaults from the existing one */
1655 if (nfs->sched_nr == 0) /* reuse */
1656 nfs->sched_nr = fs->fs.sched_nr;
1657 for (j = 0; j < sizeof(nfs->par)/sizeof(nfs->par[0]); j++) {
1658 if (nfs->par[j] == -1) /* reuse */
1659 nfs->par[j] = fs->fs.par[j];
1661 if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) {
1662 ND("flowset %d unchanged", i);
1666 * Reconfigure AQM as the parameters can be changed.
1667 * We consider the flowset as busy if it has scheduler
1670 s = locate_scheduler(nfs->sched_nr);
1671 config_aqm(fs, ep, s != NULL && s->siht != NULL);
1674 break; /* no change, nothing to do */
1676 if (oldc != V_dn_cfg.fsk_count) /* new item */
1678 s = locate_scheduler(nfs->sched_nr);
1679 /* detach from old scheduler if needed, preserving
1680 * queues if we need to reattach. Then update the
1681 * configuration, and possibly attach to the new sched.
1683 DX(2, "fs %d changed sched %d@%p to %d@%p",
1685 fs->fs.sched_nr, fs->sched, nfs->sched_nr, s);
1687 int flags = s ? DN_DETACH : (DN_DETACH | DN_DESTROY);
1688 flags |= DN_DESTROY; /* XXX temporary */
1689 fsk_detach(fs, flags);
1691 fs->fs = *nfs; /* copy configuration */
1695 config_aqm(fs, ep, s != NULL &&
1710 * config/reconfig a scheduler and its FIFO variant.
1711 * For !MULTIQUEUE schedulers, also set up the flowset.
1713 * On reconfigurations (detected because s->fp is set),
1714 * detach existing flowsets preserving traffic, preserve link,
1715 * and delete the old scheduler creating a new one.
1718 config_sched(struct dn_sch *_nsch, struct dn_id *arg)
1721 struct schk_new_arg a; /* argument for schk_new */
1723 struct dn_link p; /* copy of oldlink */
1724 struct dn_profile *pf = NULL; /* copy of old link profile */
1725 /* Used to preserv mask parameter */
1726 struct ipfw_flow_id new_mask;
1727 int new_buckets = 0;
1735 if (a.sch->oid.len != sizeof(*a.sch)) {
1736 D("bad sched len %d", a.sch->oid.len);
1739 i = a.sch->sched_nr;
1740 if (i <= 0 || i >= DN_MAX_ID)
1742 /* make sure we have some buckets */
1743 if (a.sch->flags & DN_HAVE_MASK)
1744 ipdn_bound_var((int *)&a.sch->buckets, V_dn_cfg.hash_size,
1745 1, V_dn_cfg.max_hash_size, "sched buckets");
1746 /* XXX other sanity checks */
1747 bzero(&p, sizeof(p));
1749 pipe_cmd = a.sch->flags & DN_PIPE_CMD;
1750 a.sch->flags &= ~DN_PIPE_CMD; //XXX do it even if is not set?
1752 /* Copy mask parameter */
1753 new_mask = a.sch->sched_mask;
1754 new_buckets = a.sch->buckets;
1755 new_flags = a.sch->flags;
1758 again: /* run twice, for wfq and fifo */
1760 * lookup the type. If not supplied, use the previous one
1761 * or default to WF2Q+. Otherwise, return an error.
1764 a.fp = find_sched_type(a.sch->oid.subtype, a.sch->name);
1766 /* found. Lookup or create entry */
1767 s = dn_ht_find(V_dn_cfg.schedhash, i, DNHT_INSERT, &a);
1768 } else if (a.sch->oid.subtype == 0 && !a.sch->name[0]) {
1769 /* No type. search existing s* or retry with WF2Q+ */
1770 s = dn_ht_find(V_dn_cfg.schedhash, i, 0, &a);
1773 /* Scheduler exists, skip to FIFO scheduler
1774 * if command was pipe config...
1779 /* New scheduler, create a wf2q+ with no mask
1780 * if command was pipe config...
1783 /* clear mask parameter */
1784 bzero(&a.sch->sched_mask, sizeof(new_mask));
1786 a.sch->flags &= ~DN_HAVE_MASK;
1788 a.sch->oid.subtype = DN_SCHED_WF2QP;
1792 D("invalid scheduler type %d %s",
1793 a.sch->oid.subtype, a.sch->name);
1797 /* normalize name and subtype */
1798 a.sch->oid.subtype = a.fp->type;
1799 bzero(a.sch->name, sizeof(a.sch->name));
1800 strlcpy(a.sch->name, a.fp->name, sizeof(a.sch->name));
1802 D("cannot allocate scheduler %d", i);
1805 /* restore existing link if any */
1808 if (!pf || pf->link_nr != p.link_nr) { /* no saved value */
1809 s->profile = NULL; /* XXX maybe not needed */
1811 s->profile = malloc(sizeof(struct dn_profile),
1812 M_DUMMYNET, M_NOWAIT | M_ZERO);
1813 if (s->profile == NULL) {
1814 D("cannot allocate profile");
1817 memcpy(s->profile, pf, sizeof(*pf));
1821 if (s->fp == NULL) {
1822 DX(2, "sched %d new type %s", i, a.fp->name);
1823 } else if (s->fp != a.fp ||
1824 bcmp(a.sch, &s->sch, sizeof(*a.sch)) ) {
1825 /* already existing. */
1826 DX(2, "sched %d type changed from %s to %s",
1827 i, s->fp->name, a.fp->name);
1828 DX(4, " type/sub %d/%d -> %d/%d",
1829 s->sch.oid.type, s->sch.oid.subtype,
1830 a.sch->oid.type, a.sch->oid.subtype);
1831 if (s->link.link_nr == 0)
1832 D("XXX WARNING link 0 for sched %d", i);
1833 p = s->link; /* preserve link */
1834 if (s->profile) {/* preserve profile */
1836 pf = malloc(sizeof(*pf),
1837 M_DUMMYNET, M_NOWAIT | M_ZERO);
1838 if (pf) /* XXX should issue a warning otherwise */
1839 memcpy(pf, s->profile, sizeof(*pf));
1841 /* remove from the hash */
1842 dn_ht_find(V_dn_cfg.schedhash, i, DNHT_REMOVE, NULL);
1843 /* Detach flowsets, preserve queues. */
1844 // schk_delete_cb(s, NULL);
1845 // XXX temporarily, kill queues
1846 schk_delete_cb(s, (void *)DN_DESTROY);
1849 DX(4, "sched %d unchanged type %s", i, a.fp->name);
1851 /* complete initialization */
1855 // XXX schk_reset_credit(s);
1856 /* create the internal flowset if needed,
1857 * trying to reuse existing ones if available
1859 if (!(s->fp->flags & DN_MULTIQUEUE) && !s->fs) {
1860 s->fs = dn_ht_find(V_dn_cfg.fshash, i, 0, NULL);
1863 bzero(&fs, sizeof(fs));
1864 set_oid(&fs.oid, DN_FS, sizeof(fs));
1865 fs.fs_nr = i + DN_MAX_ID;
1867 s->fs = config_fs(&fs, NULL, 1 /* locked */);
1870 schk_delete_cb(s, (void *)DN_DESTROY);
1871 D("error creating internal fs for %d", i);
1875 /* call init function after the flowset is created */
1880 if (i < DN_MAX_ID) { /* now configure the FIFO instance */
1883 /* Restore mask parameter for FIFO */
1884 a.sch->sched_mask = new_mask;
1885 a.sch->buckets = new_buckets;
1886 a.sch->flags = new_flags;
1888 /* sched config shouldn't modify the FIFO scheduler */
1889 if (dn_ht_find(V_dn_cfg.schedhash, i, 0, &a) != NULL) {
1890 /* FIFO already exist, don't touch it */
1891 err = 0; /* and this is not an error */
1895 a.sch->sched_nr = i;
1896 a.sch->oid.subtype = DN_SCHED_FIFO;
1897 bzero(a.sch->name, sizeof(a.sch->name));
1903 free(pf, M_DUMMYNET);
1908 * attach a profile to a link
1911 config_profile(struct dn_profile *pf, struct dn_id *arg)
1914 int i, olen, err = 0;
1916 if (pf->oid.len < sizeof(*pf)) {
1917 D("short profile len %d", pf->oid.len);
1921 if (i <= 0 || i >= DN_MAX_ID)
1923 /* XXX other sanity checks */
1925 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) {
1926 s = locate_scheduler(i);
1934 * If we had a profile and the new one does not fit,
1935 * or it is deleted, then we need to free memory.
1937 if (s->profile && (pf->samples_no == 0 ||
1938 s->profile->oid.len < pf->oid.len)) {
1939 free(s->profile, M_DUMMYNET);
1942 if (pf->samples_no == 0)
1945 * new profile, possibly allocate memory
1948 if (s->profile == NULL)
1949 s->profile = malloc(pf->oid.len,
1950 M_DUMMYNET, M_NOWAIT | M_ZERO);
1951 if (s->profile == NULL) {
1952 D("no memory for profile %d", i);
1956 /* preserve larger length XXX double check */
1957 olen = s->profile->oid.len;
1958 if (olen < pf->oid.len)
1960 memcpy(s->profile, pf, pf->oid.len);
1961 s->profile->oid.len = olen;
1968 * Delete all objects:
1971 dummynet_flush(void)
1974 /* delete all schedulers and related links/queues/flowsets */
1975 dn_ht_scan(V_dn_cfg.schedhash, schk_delete_cb,
1976 (void *)(uintptr_t)DN_DELETE_FS);
1977 /* delete all remaining (unlinked) flowsets */
1978 DX(4, "still %d unlinked fs", V_dn_cfg.fsk_count);
1979 dn_ht_free(V_dn_cfg.fshash, DNHT_REMOVE);
1980 fsk_detach_list(&V_dn_cfg.fsu, DN_DELETE_FS);
1981 /* Reinitialize system heap... */
1982 heap_init(&V_dn_cfg.evheap, 16, offsetof(struct dn_id, id));
1986 * Main handler for configuration. We are guaranteed to be called
1987 * with an oid which is at least a dn_id.
1988 * - the first object is the command (config, delete, flush, ...)
1989 * - config_link must be issued after the corresponding config_sched
1990 * - parameters (DN_TXT) for an object must precede the object
1991 * processed on a config_sched.
1994 do_config(void *p, size_t l)
1998 struct dn_profile profile;
2000 struct dn_link link;
2001 struct dn_sch sched;
2007 memcpy(&o, p, sizeof(o));
2008 if (o.id != DN_API_VERSION) {
2009 D("invalid api version got %d need %d", o.id, DN_API_VERSION);
2015 while (l >= sizeof(o)) {
2016 memcpy(&o, (char *)p + off, sizeof(o));
2017 if (o.len < sizeof(o) || l < o.len) {
2018 D("bad len o.len %d len %zu", o.len, l);
2026 D("cmd %d not implemented", o.type);
2029 #ifdef EMULATE_SYSCTL
2030 /* sysctl emulation.
2031 * if we recognize the command, jump to the correct
2032 * handler and return
2035 err = kesysctl_emu_set(p, l);
2039 case DN_CMD_CONFIG: /* simply a header */
2043 /* the argument is in the first uintptr_t after o */
2044 if (o.len < sizeof(o) + sizeof(a)) {
2048 memcpy(&a, (char *)p + off + sizeof(o), sizeof(a));
2049 switch (o.subtype) {
2051 /* delete base and derived schedulers */
2053 err = delete_schk(a);
2054 err2 = delete_schk(a + DN_MAX_ID);
2061 D("invalid delete type %d", o.subtype);
2066 err = (a < 1 || a >= DN_MAX_ID) ?
2067 EINVAL : delete_fs(a, 0) ;
2077 case DN_TEXT: /* store argument of next block */
2079 arg = malloc(o.len, M_TEMP, M_NOWAIT);
2084 memcpy(arg, (char *)p + off, o.len);
2088 dn = malloc(sizeof(*dn), M_TEMP, M_NOWAIT);
2093 memcpy(&dn->link, (char *)p + off, sizeof(dn->link));
2094 err = config_link(&dn->link, arg);
2098 dn = malloc(sizeof(*dn), M_TEMP, M_NOWAIT);
2103 memcpy(&dn->profile, (char *)p + off,
2104 sizeof(dn->profile));
2105 err = config_profile(&dn->profile, arg);
2109 dn = malloc(sizeof(*dn), M_TEMP, M_NOWAIT);
2114 memcpy(&dn->sched, (char *)p + off,
2116 err = config_sched(&dn->sched, arg);
2120 dn = malloc(sizeof(*dn), M_TEMP, M_NOWAIT);
2125 memcpy(&dn->fs, (char *)p + off, sizeof(dn->fs));
2126 err = (NULL == config_fs(&dn->fs, arg, 0));
2139 compute_space(struct dn_id *cmd, struct copy_args *a)
2141 int x = 0, need = 0;
2142 int profile_size = sizeof(struct dn_profile) -
2143 ED_MAX_SAMPLES_NO*sizeof(int);
2145 /* NOTE about compute space:
2146 * NP = V_dn_cfg.schk_count
2147 * NSI = V_dn_cfg.si_count
2148 * NF = V_dn_cfg.fsk_count
2149 * NQ = V_dn_cfg.queue_count
2151 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler
2152 * link, scheduler template, flowset
2153 * integrated in scheduler and header
2155 * (NSI)*(dn_flow) all scheduler instance (includes
2156 * the queue instance)
2158 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler
2159 * link, scheduler template, flowset
2160 * integrated in scheduler and header
2162 * (NSI * dn_flow) all scheduler instances
2163 * (NF * sizeof(uint_32)) space for flowset list linked to scheduler
2164 * (NQ * dn_queue) all queue [XXXfor now not listed]
2166 * (NF * dn_fs) all flowset
2167 * (NQ * dn_queue) all queues
2169 switch (cmd->subtype) {
2172 /* XXX where do LINK and SCH differ ? */
2173 /* 'ipfw sched show' could list all queues associated to
2174 * a scheduler. This feature for now is disabled
2176 case DN_LINK: /* pipe show */
2177 x = DN_C_LINK | DN_C_SCH | DN_C_FLOW;
2178 need += V_dn_cfg.schk_count *
2179 (sizeof(struct dn_fs) + profile_size) / 2;
2180 need += V_dn_cfg.fsk_count * sizeof(uint32_t);
2182 case DN_SCH: /* sched show */
2183 need += V_dn_cfg.schk_count *
2184 (sizeof(struct dn_fs) + profile_size) / 2;
2185 need += V_dn_cfg.fsk_count * sizeof(uint32_t);
2186 x = DN_C_SCH | DN_C_LINK | DN_C_FLOW;
2188 case DN_FS: /* queue show */
2189 x = DN_C_FS | DN_C_QUEUE;
2191 case DN_GET_COMPAT: /* compatibility mode */
2192 need = dn_compat_calc_size();
2197 need += V_dn_cfg.schk_count * sizeof(struct dn_sch) / 2;
2198 /* NOT also, each fs might be attached to a sched */
2199 need += V_dn_cfg.schk_count * sizeof(struct dn_id) / 2;
2202 need += V_dn_cfg.fsk_count * sizeof(struct dn_fs);
2203 if (x & DN_C_LINK) {
2204 need += V_dn_cfg.schk_count * sizeof(struct dn_link) / 2;
2207 * When exporting a queue to userland, only pass up the
2208 * struct dn_flow, which is the only visible part.
2212 need += V_dn_cfg.queue_count * sizeof(struct dn_flow);
2214 need += V_dn_cfg.si_count * (sizeof(struct dn_flow));
2219 * If compat != NULL dummynet_get is called in compatibility mode.
2220 * *compat will be the pointer to the buffer to pass to ipfw
2223 dummynet_get(struct sockopt *sopt, void **compat)
2225 int have, i, need, error;
2226 char *start = NULL, *buf;
2227 size_t sopt_valsize;
2230 struct copy_range r;
2231 int l = sizeof(struct dn_id);
2233 bzero(&a, sizeof(a));
2234 bzero(&r, sizeof(r));
2236 /* save and restore original sopt_valsize around copyin */
2237 sopt_valsize = sopt->sopt_valsize;
2242 /* copy at least an oid, and possibly a full object */
2243 error = sooptcopyin(sopt, cmd, sizeof(r), sizeof(*cmd));
2244 sopt->sopt_valsize = sopt_valsize;
2248 #ifdef EMULATE_SYSCTL
2249 /* sysctl emulation. */
2250 if (cmd->type == DN_SYSCTL_GET)
2251 return kesysctl_emu_get(sopt);
2253 if (l > sizeof(r)) {
2254 /* request larger than default, allocate buffer */
2255 cmd = malloc(l, M_DUMMYNET, M_NOWAIT);
2260 error = sooptcopyin(sopt, cmd, l, l);
2261 sopt->sopt_valsize = sopt_valsize;
2265 } else { /* compatibility */
2267 cmd->type = DN_CMD_GET;
2268 cmd->len = sizeof(struct dn_id);
2269 cmd->subtype = DN_GET_COMPAT;
2270 // cmd->id = sopt_valsize;
2271 D("compatibility mode");
2275 /* get AQM params */
2276 if(cmd->subtype == DN_AQM_PARAMS) {
2277 error = get_aqm_parms(sopt);
2279 /* get Scheduler params */
2280 } else if (cmd->subtype == DN_SCH_PARAMS) {
2281 error = get_sched_parms(sopt);
2286 a.extra = (struct copy_range *)cmd;
2287 if (cmd->len == sizeof(*cmd)) { /* no range, create a default */
2288 uint32_t *rp = (uint32_t *)(cmd + 1);
2289 cmd->len += 2* sizeof(uint32_t);
2291 rp[1] = DN_MAX_ID - 1;
2292 if (cmd->subtype == DN_LINK) {
2297 /* Count space (under lock) and allocate (outside lock).
2298 * Exit with lock held if we manage to get enough buffer.
2299 * Try a few times then give up.
2301 for (have = 0, i = 0; i < 10; i++) {
2303 need = compute_space(cmd, &a);
2305 /* if there is a range, ignore value from compute_space() */
2306 if (l > sizeof(*cmd))
2307 need = sopt_valsize - sizeof(*cmd);
2314 need += sizeof(*cmd);
2320 free(start, M_DUMMYNET);
2322 if (need > sopt_valsize)
2326 start = malloc(have, M_DUMMYNET, M_NOWAIT | M_ZERO);
2329 if (start == NULL) {
2334 error = sooptcopyout(sopt, cmd, sizeof(*cmd));
2338 ND("have %d:%d sched %d, %d:%d links %d, %d:%d flowsets %d, "
2339 "%d:%d si %d, %d:%d queues %d",
2340 V_dn_cfg.schk_count, sizeof(struct dn_sch), DN_SCH,
2341 V_dn_cfg.schk_count, sizeof(struct dn_link), DN_LINK,
2342 V_dn_cfg.fsk_count, sizeof(struct dn_fs), DN_FS,
2343 V_dn_cfg.si_count, sizeof(struct dn_flow), DN_SCH_I,
2344 V_dn_cfg.queue_count, sizeof(struct dn_queue), DN_QUEUE);
2345 sopt->sopt_valsize = sopt_valsize;
2346 a.type = cmd->subtype;
2348 if (compat == NULL) {
2349 memcpy(start, cmd, sizeof(*cmd));
2350 ((struct dn_id*)(start))->len = sizeof(struct dn_id);
2351 buf = start + sizeof(*cmd);
2355 a.end = start + have;
2356 /* start copying other objects */
2358 a.type = DN_COMPAT_PIPE;
2359 dn_ht_scan(V_dn_cfg.schedhash, copy_data_helper_compat, &a);
2360 a.type = DN_COMPAT_QUEUE;
2361 dn_ht_scan(V_dn_cfg.fshash, copy_data_helper_compat, &a);
2362 } else if (a.type == DN_FS) {
2363 dn_ht_scan(V_dn_cfg.fshash, copy_data_helper, &a);
2365 dn_ht_scan(V_dn_cfg.schedhash, copy_data_helper, &a);
2371 sopt->sopt_valsize = buf - start;
2372 /* free() is done by ip_dummynet_compat() */
2373 start = NULL; //XXX hack
2375 error = sooptcopyout(sopt, start, buf - start);
2379 free(cmd, M_DUMMYNET);
2380 free(start, M_DUMMYNET);
2384 /* Callback called on scheduler instance to delete it if idle */
2386 drain_scheduler_cb(void *_si, void *arg)
2388 struct dn_sch_inst *si = _si;
2390 if ((si->kflags & DN_ACTIVE) || si->dline.mq.head != NULL)
2393 if (si->sched->fp->flags & DN_MULTIQUEUE) {
2394 if (si->q_count == 0)
2395 return si_destroy(si, NULL);
2398 } else { /* !DN_MULTIQUEUE */
2399 if ((si+1)->ni.length == 0)
2400 return si_destroy(si, NULL);
2404 return 0; /* unreachable */
2407 /* Callback called on scheduler to check if it has instances */
2409 drain_scheduler_sch_cb(void *_s, void *arg)
2411 struct dn_schk *s = _s;
2413 if (s->sch.flags & DN_HAVE_MASK) {
2414 dn_ht_scan_bucket(s->siht, &s->drain_bucket,
2415 drain_scheduler_cb, NULL);
2419 if (drain_scheduler_cb(s->siht, NULL) == DNHT_SCAN_DEL)
2426 /* Called every tick, try to delete a 'bucket' of scheduler */
2428 dn_drain_scheduler(void)
2430 dn_ht_scan_bucket(V_dn_cfg.schedhash, &V_dn_cfg.drain_sch,
2431 drain_scheduler_sch_cb, NULL);
2432 V_dn_cfg.drain_sch++;
2435 /* Callback called on queue to delete if it is idle */
2437 drain_queue_cb(void *_q, void *arg)
2439 struct dn_queue *q = _q;
2441 if (q->ni.length == 0) {
2442 dn_delete_queue(q, DN_DESTROY);
2443 return DNHT_SCAN_DEL; /* queue is deleted */
2446 return 0; /* queue isn't deleted */
2449 /* Callback called on flowset used to check if it has queues */
2451 drain_queue_fs_cb(void *_fs, void *arg)
2453 struct dn_fsk *fs = _fs;
2455 if (fs->fs.flags & DN_QHT_HASH) {
2456 /* Flowset has a hash table for queues */
2457 dn_ht_scan_bucket(fs->qht, &fs->drain_bucket,
2458 drain_queue_cb, NULL);
2461 /* No hash table for this flowset, null the pointer
2462 * if the queue is deleted
2465 if (drain_queue_cb(fs->qht, NULL) == DNHT_SCAN_DEL)
2472 /* Called every tick, try to delete a 'bucket' of queue */
2474 dn_drain_queue(void)
2476 /* scan a bucket of flowset */
2477 dn_ht_scan_bucket(V_dn_cfg.fshash, &V_dn_cfg.drain_fs,
2478 drain_queue_fs_cb, NULL);
2479 V_dn_cfg.drain_fs++;
2483 * Handler for the various dummynet socket options
2486 ip_dn_ctl(struct sockopt *sopt)
2488 struct epoch_tracker et;
2493 error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET);
2497 /* Disallow sets in really-really secure mode. */
2498 if (sopt->sopt_dir == SOPT_SET) {
2499 error = securelevel_ge(sopt->sopt_td->td_ucred, 3);
2504 NET_EPOCH_ENTER(et);
2506 switch (sopt->sopt_name) {
2508 D("dummynet: unknown option %d", sopt->sopt_name);
2512 case IP_DUMMYNET_FLUSH:
2513 case IP_DUMMYNET_CONFIGURE:
2514 case IP_DUMMYNET_DEL: /* remove a pipe or queue */
2515 case IP_DUMMYNET_GET:
2516 D("dummynet: compat option %d", sopt->sopt_name);
2517 error = ip_dummynet_compat(sopt);
2521 if (sopt->sopt_dir == SOPT_GET) {
2522 error = dummynet_get(sopt, NULL);
2525 l = sopt->sopt_valsize;
2526 if (l < sizeof(struct dn_id) || l > 12000) {
2527 D("argument len %zu invalid", l);
2530 p = malloc(l, M_TEMP, M_NOWAIT);
2535 error = sooptcopyin(sopt, p, l, l);
2537 error = do_config(p, l);
2549 ip_dn_vnet_init(void)
2551 if (V_dn_cfg.init_done)
2553 V_dn_cfg.init_done = 1;
2554 /* Set defaults here. MSVC does not accept initializers,
2555 * and this is also useful for vimages
2558 V_dn_cfg.slot_limit = 100; /* Foot shooting limit for queues. */
2559 V_dn_cfg.byte_limit = 1024 * 1024;
2560 V_dn_cfg.expire = 1;
2562 /* RED parameters */
2563 V_dn_cfg.red_lookup_depth = 256; /* default lookup table depth */
2564 V_dn_cfg.red_avg_pkt_size = 512; /* default medium packet size */
2565 V_dn_cfg.red_max_pkt_size = 1500; /* default max packet size */
2568 V_dn_cfg.max_hash_size = 65536; /* max in the hash tables */
2569 V_dn_cfg.hash_size = 64; /* default hash size */
2571 /* create hash tables for schedulers and flowsets.
2572 * In both we search by key and by pointer.
2574 V_dn_cfg.schedhash = dn_ht_init(NULL, V_dn_cfg.hash_size,
2575 offsetof(struct dn_schk, schk_next),
2576 schk_hash, schk_match, schk_new);
2577 V_dn_cfg.fshash = dn_ht_init(NULL, V_dn_cfg.hash_size,
2578 offsetof(struct dn_fsk, fsk_next),
2579 fsk_hash, fsk_match, fsk_new);
2581 /* bucket index to drain object */
2582 V_dn_cfg.drain_fs = 0;
2583 V_dn_cfg.drain_sch = 0;
2585 heap_init(&V_dn_cfg.evheap, 16, offsetof(struct dn_id, id));
2586 SLIST_INIT(&V_dn_cfg.fsu);
2590 /* Initialize curr_time adjustment mechanics. */
2591 getmicrouptime(&V_dn_cfg.prev_t);
2595 ip_dn_vnet_destroy(void)
2601 dn_ht_free(V_dn_cfg.schedhash, 0);
2602 dn_ht_free(V_dn_cfg.fshash, 0);
2603 heap_free(&V_dn_cfg.evheap);
2611 if (dn_tasks_started)
2614 mtx_init(&sched_mtx, "dn_sched", NULL, MTX_DEF);
2616 dn_tasks_started = 1;
2617 TASK_INIT(&dn_task, 0, dummynet_task, NULL);
2618 dn_tq = taskqueue_create_fast("dummynet", M_WAITOK,
2619 taskqueue_thread_enqueue, &dn_tq);
2620 taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet");
2622 CK_LIST_INIT(&schedlist);
2623 callout_init(&dn_timeout, 1);
2628 ip_dn_destroy(int last)
2630 /* ensure no more callouts are started */
2633 /* check for last */
2635 ND("removing last instance\n");
2636 ip_dn_ctl_ptr = NULL;
2637 ip_dn_io_ptr = NULL;
2640 callout_drain(&dn_timeout);
2641 taskqueue_drain(dn_tq, &dn_task);
2642 taskqueue_free(dn_tq);
2646 dummynet_modevent(module_t mod, int type, void *data)
2649 if (type == MOD_LOAD) {
2651 printf("DUMMYNET already loaded\n");
2655 ip_dn_ctl_ptr = ip_dn_ctl;
2656 ip_dn_io_ptr = dummynet_io;
2658 } else if (type == MOD_UNLOAD) {
2659 ip_dn_destroy(1 /* last */);
2665 /* modevent helpers for the modules */
2667 load_dn_sched(struct dn_alg *d)
2672 return 1; /* error */
2673 ip_dn_init(); /* just in case, we need the lock */
2675 /* Check that mandatory funcs exists */
2676 if (d->enqueue == NULL || d->dequeue == NULL) {
2677 D("missing enqueue or dequeue for %s", d->name);
2681 /* Search if scheduler already exists */
2682 mtx_lock(&sched_mtx);
2683 CK_LIST_FOREACH(s, &schedlist, next) {
2684 if (strcmp(s->name, d->name) == 0) {
2685 D("%s already loaded", d->name);
2686 break; /* scheduler already exists */
2690 CK_LIST_INSERT_HEAD(&schedlist, d, next);
2691 mtx_unlock(&sched_mtx);
2692 D("dn_sched %s %sloaded", d->name, s ? "not ":"");
2697 unload_dn_sched(struct dn_alg *s)
2699 struct dn_alg *tmp, *r;
2702 ND("called for %s", s->name);
2704 mtx_lock(&sched_mtx);
2705 CK_LIST_FOREACH_SAFE(r, &schedlist, next, tmp) {
2706 if (strcmp(s->name, r->name) != 0)
2708 ND("ref_count = %d", r->ref_count);
2709 err = (r->ref_count != 0) ? EBUSY : 0;
2711 CK_LIST_REMOVE(r, next);
2714 mtx_unlock(&sched_mtx);
2716 D("dn_sched %s %sunloaded", s->name, err ? "not ":"");
2721 dn_sched_modevent(module_t mod, int cmd, void *arg)
2723 struct dn_alg *sch = arg;
2725 if (cmd == MOD_LOAD)
2726 return load_dn_sched(sch);
2727 else if (cmd == MOD_UNLOAD)
2728 return unload_dn_sched(sch);
2733 static moduledata_t dummynet_mod = {
2734 "dummynet", dummynet_modevent, NULL
2737 #define DN_SI_SUB SI_SUB_PROTO_FIREWALL
2738 #define DN_MODEV_ORD (SI_ORDER_ANY - 128) /* after ipfw */
2739 DECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD);
2740 MODULE_VERSION(dummynet, 3);
2743 * Starting up. Done in order after dummynet_modevent() has been called.
2744 * VNET_SYSINIT is also called for each existing vnet and each new vnet.
2746 VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_vnet_init, NULL);
2749 * Shutdown handlers up shop. These are done in REVERSE ORDER, but still
2750 * after dummynet_modevent() has been called. Not called on reboot.
2751 * VNET_SYSUNINIT is also called for each exiting vnet as it exits.
2752 * or when the module is unloaded.
2754 VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_vnet_destroy, NULL);
2758 /* modevent helpers for the AQM modules */
2760 load_dn_aqm(struct dn_aqm *d)
2762 struct dn_aqm *aqm=NULL;
2765 return 1; /* error */
2766 ip_dn_init(); /* just in case, we need the lock */
2768 /* Check that mandatory funcs exists */
2769 if (d->enqueue == NULL || d->dequeue == NULL) {
2770 D("missing enqueue or dequeue for %s", d->name);
2774 mtx_lock(&sched_mtx);
2776 /* Search if AQM already exists */
2777 CK_LIST_FOREACH(aqm, &aqmlist, next) {
2778 if (strcmp(aqm->name, d->name) == 0) {
2779 D("%s already loaded", d->name);
2780 break; /* AQM already exists */
2784 CK_LIST_INSERT_HEAD(&aqmlist, d, next);
2786 mtx_unlock(&sched_mtx);
2788 D("dn_aqm %s %sloaded", d->name, aqm ? "not ":"");
2792 /* Callback to clean up AQM status for queues connected to a flowset
2793 * and then deconfigure the flowset.
2794 * This function is called before an AQM module is unloaded
2797 fs_cleanup(void *_fs, void *arg)
2799 struct dn_fsk *fs = _fs;
2800 uint32_t type = *(uint32_t *)arg;
2802 if (fs->aqmfp && fs->aqmfp->type == type)
2803 aqm_cleanup_deconfig_fs(fs);
2809 unload_dn_aqm(struct dn_aqm *aqm)
2811 struct dn_aqm *tmp, *r;
2814 ND("called for %s", aqm->name);
2816 /* clean up AQM status and deconfig flowset */
2817 dn_ht_scan(V_dn_cfg.fshash, fs_cleanup, &aqm->type);
2819 mtx_lock(&sched_mtx);
2821 CK_LIST_FOREACH_SAFE(r, &aqmlist, next, tmp) {
2822 if (strcmp(aqm->name, r->name) != 0)
2824 ND("ref_count = %d", r->ref_count);
2825 err = (r->ref_count != 0 || r->cfg_ref_count != 0) ? EBUSY : 0;
2827 CK_LIST_REMOVE(r, next);
2831 mtx_unlock(&sched_mtx);
2834 D("%s %sunloaded", aqm->name, err ? "not ":"");
2836 D("ref_count=%d, cfg_ref_count=%d", r->ref_count, r->cfg_ref_count);
2841 dn_aqm_modevent(module_t mod, int cmd, void *arg)
2843 struct dn_aqm *aqm = arg;
2845 if (cmd == MOD_LOAD)
2846 return load_dn_aqm(aqm);
2847 else if (cmd == MOD_UNLOAD)
2848 return unload_dn_aqm(aqm);