The following is an example of the putnexts.d script. When investigating the operation of kernel streams, it can be extreamly useful to trace who (which stack trace) is calling putnext to who (the kernel modele). This script does that, # putnexts.d dtrace: script './putnexts.d' matched 1 probe ^C bufmod bufmod`sbrput+0xb9 unix`putnext+0x1b7 pts`ptswput+0x1e1 unix`putnext+0x1b7 ptem`ptemwput+0x22f 1 ip ip`ip_fanout_proto+0x4d2 ip`ip_proto_input+0x616 ip`ip_fanout_proto_again+0x160 ip`ip_proto_input+0x530 ip`ip_rput+0x50d 1 ip ip`ip_wput_ipsec_out+0x60a ip`ipsec_out_process+0x322 ip`ip_wput_ire+0x18d0 ip`ip_output+0x70a ip`ip_wput+0x14 1 ip ip`ip_wput_ire+0x17d1 ip`ip_output+0x70a ip`tcp_send_data+0x68c ip`tcp_ack_timer+0xb0 ip`tcp_timer_handler+0x1d 1 ip ip`ip_udp_input+0x4e4 ip`ip_rput+0x540 unix`putnext+0x1b7 gld`gld_recv_tagged+0xed gld`gld_recv+0x10 1 ldterm genunix`putnextctl1+0x52 ldterm`ldterm_dosig+0x7b ldterm`ldtermrput+0x6fe unix`putnext+0x1b7 ptem`ptemrput+0xff 1 ldterm ldterm`ldterm_ioctl_reply+0x93 ldterm`ldtermrput+0x186 genunix`qdrain_syncq+0x68 genunix`drain_syncq+0x1a4 genunix`putnext_tail+0x88 1 ldterm ldterm`ldtermrput+0x12a unix`putnext+0x1b7 ptem`ptemrput+0xff genunix`qdrain_syncq+0x68 genunix`drain_syncq+0x1a4 1 ldterm ldterm`ldterm_dosig+0x16f ldterm`ldtermrput+0x6fe unix`putnext+0x1b7 ptem`ptemrput+0xff unix`putnext+0x1b7 1 ldterm genunix`putnextctl1+0x52 ldterm`ldterm_dosig+0x124 ldterm`ldtermrput+0x6fe unix`putnext+0x1b7 ptem`ptemrput+0xff 1 ldterm ldterm`ldterm_do_ioctl+0xd8b ldterm`ldtermwmsg+0x41 ldterm`ldtermwput+0x8e unix`putnext+0x1b7 ttcompat`ttcompat_do_ioctl+0x425 1 ldterm genunix`putnextctl1+0x52 ldterm`ldterm_dosig+0xe3 ldterm`ldtermrput+0x6fe unix`putnext+0x1b7 ptem`ptemrput+0xff 1 ptem genunix`qreply+0x23 ptem`ptemwmsg+0x2b9 ptem`ptemwput+0xe1 unix`putnext+0x1b7 ldterm`ldterm_do_ioctl+0xd8b 1 ptem ptem`ptemwput+0x22f genunix`qdrain_syncq+0x68 genunix`drain_syncq+0x1a4 genunix`putnext_tail+0x88 unix`putnext+0x38e 1 ptem ptem`ptemrput+0xff genunix`qdrain_syncq+0x68 genunix`drain_syncq+0x1a4 genunix`putnext_tail+0x88 unix`putnext+0x38e 1 ptem ptem`ptemwmsg+0x44d ptem`ptemwput+0xe1 genunix`qdrain_syncq+0x68 genunix`drain_syncq+0x1a4 genunix`putnext_tail+0x88 1 ptm pts`ptswsrv+0x7c genunix`runservice+0x2a genunix`queue_service+0x30 genunix`stream_service+0x69 genunix`taskq_d_thread+0x8a 1 ptm pts`ptswput+0x1e1 unix`putnext+0x1b7 ptem`ptemwput+0x22f genunix`qdrain_syncq+0x68 genunix`drain_syncq+0x1a4 1 pts genunix`qreply+0x23 pts`ptswput+0x201 unix`putnext+0x1b7 ptem`ptemwput+0x22f genunix`qdrain_syncq+0x68 1 strwhead genunix`qreply+0x23 genunix`strrput_nondata+0x22d genunix`strrput+0x256 unix`putnext+0x1b7 ttcompat`ttcompatrput+0x1d 1 strwhead genunix`strdoioctl+0x30d genunix`strioctl+0x6ae specfs`spec_ioctl+0x48 genunix`fop_ioctl+0x1e genunix`ioctl+0x199 1 tcp ip`tcp_rput_data+0x2221 ip`tcp_input+0x39 ip`squeue_enter+0x1bf ip`ip_tcp_input+0x9f8 ip`ip_rput+0x583 1 ttcompat ttcompat`ttcompat_ioctl_ack+0x398 ttcompat`ttcompatrput+0x39 genunix`qdrain_syncq+0x68 genunix`drain_syncq+0x1a4 genunix`putnext_tail+0x88 1 ttcompat ttcompat`ttcompatrput+0x1d unix`putnext+0x1b7 ldterm`ldtermrput+0x12a unix`putnext+0x1b7 ptem`ptemrput+0xff 1 ttcompat ttcompat`ttcompatrput+0x1d unix`putnext+0x1b7 genunix`putnextctl1+0x52 ldterm`ldterm_dosig+0x124 ldterm`ldtermrput+0x6fe 1 ttcompat ttcompat`ttcompatwput+0x32 genunix`qdrain_syncq+0x68 genunix`drain_syncq+0x1a4 genunix`putnext_tail+0x88 unix`putnext+0x38e 1 ttcompat ttcompat`ttcompat_do_ioctl+0x425 ttcompat`ttcompatwput+0x152 unix`putnext+0x1b7 genunix`strdoioctl+0x30d genunix`strioctl+0x6ae 1 ttcompat ttcompat`ttcompatrput+0x1d unix`putnext+0x1b7 genunix`putnextctl1+0x52 ldterm`ldterm_dosig+0xe3 ldterm`ldtermrput+0x6fe 1 tun tun`tun_rdata_v4+0x58c tun`tun_rproc+0x256 tun`tun_rput+0x23 unix`putnext+0x1b7 ip`ip_fanout_proto+0x4d2 1 tun tun`tun_wputnext_v4+0x1f8 tun`tun_wproc_mdata+0x71 tun`tun_wproc+0xdf tun`tun_wput+0x23 unix`putnext+0x1b7 1 udp udp`udp_rput+0x975 unix`putnext+0x1b7 ip`ip_udp_input+0x4e4 ip`ip_rput+0x540 unix`putnext+0x1b7 1 conskbd genunix`qreply+0x23 conskbd`conskbd_mux_upstream_msg+0x24f conskbd`conskbd_lqs_ack_complete+0x65 conskbd`conskbdlrput+0x9d unix`putnext+0x1b7 2 conskbd conskbd`conskbdlwserv+0x2d genunix`runservice+0x2a genunix`queue_service+0x30 genunix`stream_service+0x69 genunix`taskq_d_thread+0x8a 2 kb8042 genunix`qreply+0x23 genunix`miocack+0x2a kb8042`kb8042_ioctlmsg+0x281 kb8042`kb8042_wsrv+0xcb genunix`runservice+0x2a 2 strwhead genunix`strdoioctl+0x30d genunix`strioctl+0x587 specfs`spec_ioctl+0x48 genunix`fop_ioctl+0x1e genunix`ioctl+0x199 2 consms consms`consmslrput+0x15a unix`putnext+0x1b7 vuid3ps2`vuid3ps2_putnext+0x94 vuid3ps2`sendButtonEvent+0x58 vuid3ps2`vuid3ps2+0x8ee 4 vuid3ps2 vuid3ps2`vuid3ps2_putnext+0x94 vuid3ps2`sendButtonEvent+0x58 vuid3ps2`vuid3ps2+0x8ee vuid3ps2`vuidmice_rsrv+0x69 genunix`runservice+0x2a 4 ip ip`tcp_send_data+0x55d ip`tcp_ack_timer+0xb0 ip`tcp_timer_handler+0x1d ip`squeue_drain+0xbb ip`squeue_worker+0xeb 10 ip ip`tcp_send_data+0x55d ip`tcp_rput_data+0x259e ip`squeue_enter+0x1bf ip`ip_tcp_input+0xcfb ip`ip_rput+0x583 11 ip ip`ip_fanout_udp_conn+0x14b ip`ip_fanout_udp+0x373 ip`ip_wput_local+0x16f ip`ip_wput_ire+0x1436 ip`ip_output+0x70a 14 strwhead genunix`strput+0x168 genunix`strputmsg+0x1d5 genunix`msgio+0x142 genunix`putmsg+0x6e unix`sys_sysenter+0xdc 14 timod timod`timodwput+0xea unix`putnext+0x1b7 genunix`strput+0x168 genunix`strputmsg+0x1d5 genunix`msgio+0x142 14 timod timod`timodrput+0xa9 unix`putnext+0x1b7 udp`udp_rput+0x975 unix`putnext+0x1b7 ip`ip_fanout_udp_conn+0x14b 14 udp udp`udp_rput+0x975 unix`putnext+0x1b7 ip`ip_fanout_udp_conn+0x14b ip`ip_fanout_udp+0x373 ip`ip_wput_local+0x16f 14 udp udp`udp_wput+0x378 unix`putnext+0x1b7 timod`timodwput+0xea unix`putnext+0x1b7 genunix`strput+0x168 14 bufmod bufmod`sbsendit+0x5a bufmod`sbclosechunk+0x2e bufmod`sbrput+0xee genunix`qdrain_syncq+0x68 genunix`drain_syncq+0x1a4 21 conskbd kbtrans`kbtrans_queueevent+0x5c kbtrans`kbtrans_keyreleased+0x3d kbtrans`kbtrans_untrans_keyreleased_raw+0x10 kbtrans`kbtrans_processkey+0x20 kbtrans`kbtrans_streams_key+0x95 22 kb8042 kbtrans`kbtrans_queueevent+0x5c kbtrans`kbtrans_keyreleased+0x3d kbtrans`kbtrans_untrans_keyreleased_raw+0x10 kbtrans`kbtrans_processkey+0x20 kbtrans`kbtrans_streams_key+0x95 22 ldterm ldterm`ldterm_msg_upstream+0x2c ldterm`vmin_satisfied+0x5e ldterm`ldterm_dononcanon+0x230 ldterm`ldtermrmsg+0x252 ldterm`ldtermrput+0x7e9 22 ttcompat ttcompat`ttcompatrput+0x1d unix`putnext+0x1b7 ldterm`ldterm_msg_upstream+0x2c ldterm`vmin_satisfied+0x5e ldterm`ldterm_dononcanon+0x230 22 bufmod bufmod`sbwput+0x33 unix`putnext+0x1b7 genunix`strput+0x168 genunix`strwrite+0x151 specfs`spec_write+0x4e 23 ptem ptem`ptemrput+0xff unix`putnext+0x1b7 ptm`ptmwsrv+0x90 genunix`runservice+0x2a genunix`queue_service+0x30 23 pts ptm`ptmwsrv+0x90 genunix`runservice+0x2a genunix`queue_service+0x30 genunix`stream_runservice+0x96 genunix`strput+0x171 23 conskbd kbtrans`kbtrans_queueevent+0x5c kbtrans`kbtrans_queuepress+0x5f kbtrans`kbtrans_untrans_keypressed_raw+0x36 kbtrans`kbtrans_processkey+0x30 kbtrans`kbtrans_streams_key+0x95 24 kb8042 kbtrans`kbtrans_queueevent+0x5c kbtrans`kbtrans_queuepress+0x5f kbtrans`kbtrans_untrans_keypressed_raw+0x36 kbtrans`kbtrans_processkey+0x30 kbtrans`kbtrans_streams_key+0x95 24 ip ip`tcp_send_data+0x55d ip`tcp_output+0x562 ip`squeue_enter+0x1bf ip`tcp_wput+0x234 unix`putnext+0x1b7 32 ldterm ldterm`ldtermwmsg+0x100 ldterm`ldtermwput+0x8e unix`putnext+0x1b7 ttcompat`ttcompatwput+0x32 unix`putnext+0x1b7 36 ptem ptem`ptemwmsg+0x44d ptem`ptemwput+0xe1 unix`putnext+0x1b7 ldterm`ldtermwmsg+0x100 ldterm`ldtermwput+0x8e 36 ptm pts`ptswsrv+0x7c genunix`runservice+0x2a genunix`queue_service+0x30 genunix`stream_runservice+0x96 genunix`strput+0x171 36 ttcompat ttcompat`ttcompatwput+0x32 unix`putnext+0x1b7 genunix`strput+0x168 genunix`strwrite+0x151 specfs`spec_write+0x4e 36 tcp ip`tcp_rput_data+0x2221 ip`squeue_enter+0x1bf ip`ip_tcp_input+0xcfb ip`ip_rput+0x583 unix`putnext+0x1b7 40 rtls gld`gld_recv_tagged+0xed gld`gld_recv+0x10 rtls`rtls_receive+0x18f rtls`rtls_gld_intr+0x133 gld`gld_intr+0x1e 46 consms consms`consmslrput+0x15a unix`putnext+0x1b7 vuid3ps2`vuid3ps2_putnext+0x94 vuid3ps2`vuid3ps2+0x82b vuid3ps2`vuidmice_rsrv+0x69 59 strwhead genunix`strput+0x168 genunix`strwrite+0x151 specfs`spec_write+0x4e genunix`fop_write+0x1b genunix`write+0x29a 59 vuid3ps2 vuid3ps2`vuid3ps2_putnext+0x94 vuid3ps2`vuid3ps2+0x82b vuid3ps2`vuidmice_rsrv+0x69 genunix`runservice+0x2a genunix`queue_service+0x30 59 consms consms`consmslrput+0x15a unix`putnext+0x1b7 vuid3ps2`vuid3ps2_putnext+0x94 vuid3ps2`vuid3ps2+0x809 vuid3ps2`vuidmice_rsrv+0x69 77 vuid3ps2 vuid3ps2`vuid3ps2_putnext+0x94 vuid3ps2`vuid3ps2+0x809 vuid3ps2`vuidmice_rsrv+0x69 genunix`runservice+0x2a genunix`queue_service+0x30 77 strwhead genunix`strput+0x168 genunix`strwrite+0x151 sockfs`socktpi_write+0xcb genunix`fop_write+0x1b genunix`writev+0x308 108 rtls gld`gld_passon+0x9e gld`gld_sendup+0xfc gld`gld_recv_tagged+0x15f gld`gld_recv+0x10 rtls`rtls_receive+0x18f 124 strwhead genunix`strput+0x168 genunix`strwrite+0x151 sockfs`socktpi_write+0xcb genunix`fop_write+0x1b genunix`write+0x29a 138 tl tl`tl_wput_data_ser+0x5e genunix`serializer_exec+0x1d genunix`serializer_enter+0x81 tl`tl_serializer_enter+0x40 tl`tl_wput+0x1c7 214 mouse8042 mouse8042`mouse8042_intr+0x68 i8042`i8042_intr+0xa6 unix`intr_thread+0x107 261 Highlights of the above output include, - gld calling putnext to rtls (my laptop network device driver) - ip receive path calling putnext to tcp - tcp_send_data() calling putnext to ip