1 # Chelsio T5 Factory Default configuration file.
3 # Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
5 # DO NOT MODIFY THIS FILE UNDER ANY CIRCUMSTANCES. MODIFICATION OF
6 # THIS FILE WILL RESULT IN A NON-FUNCTIONAL T4 ADAPTER AND MAY RESULT
7 # IN PHYSICAL DAMAGE TO T4 ADAPTERS.
9 # This file provides the default, power-on configuration for 4-port T4-based
10 # adapters shipped from the factory. These defaults are designed to address
11 # the needs of the vast majority of T4 customers. The basic idea is to have
12 # a default configuration which allows a customer to plug a T4 adapter in and
13 # have it work regardless of OS, driver or application except in the most
14 # unusual and/or demanding customer applications.
16 # Many of the T4 resources which are described by this configuration are
17 # finite. This requires balancing the configuration/operation needs of
18 # device drivers across OSes and a large number of customer application.
20 # Some of the more important resources to allocate and their constaints are:
21 # 1. Virtual Interfaces: 128.
22 # 2. Ingress Queues with Free Lists: 1024. PCI-E SR-IOV Virtual Functions
23 # must use a power of 2 Ingress Queues.
24 # 3. Egress Queues: 128K. PCI-E SR-IOV Virtual Functions must use a
25 # power of 2 Egress Queues.
26 # 4. MSI-X Vectors: 1088. A complication here is that the PCI-E SR-IOV
27 # Virtual Functions based off of a Physical Function all get the
28 # same umber of MSI-X Vectors as the base Physical Function.
29 # Additionally, regardless of whether Virtual Functions are enabled or
30 # not, their MSI-X "needs" are counted by the PCI-E implementation.
31 # And finally, all Physical Funcations capable of supporting Virtual
32 # Functions (PF0-3) must have the same number of configured TotalVFs in
33 # their SR-IOV Capabilities.
34 # 5. Multi-Port Support (MPS) TCAM: 336 entries to support MAC destination
35 # address matching on Ingress Packets.
37 # Some of the important OS/Driver resource needs are:
38 # 6. Some OS Drivers will manage all resources through a single Physical
39 # Function (currently PF0 but it could be any Physical Function). Thus,
40 # this "Unified PF" will need to have enough resources allocated to it
41 # to allow for this. And because of the MSI-X resource allocation
42 # constraints mentioned above, this probably means we'll either have to
43 # severely limit the TotalVFs if we continue to use PF0 as the Unified PF
44 # or we'll need to move the Unified PF into the PF4-7 range since those
45 # Physical Functions don't have any Virtual Functions associated with
47 # 7. Some OS Drivers will manage different ports and functions (NIC,
48 # storage, etc.) on different Physical Functions. For example, NIC
49 # functions for ports 0-3 on PF0-3, FCoE on PF4, iSCSI on PF5, etc.
51 # Some of the customer application needs which need to be accommodated:
52 # 8. Some customers will want to support large CPU count systems with
53 # good scaling. Thus, we'll need to accommodate a number of
54 # Ingress Queues and MSI-X Vectors to allow up to some number of CPUs
55 # to be involved per port and per application function. For example,
56 # in the case where all ports and application functions will be
57 # managed via a single Unified PF and we want to accommodate scaling up
58 # to 8 CPUs, we would want:
61 # 3 application functions (NIC, FCoE, iSCSI) per port *
62 # 8 Ingress Queue/MSI-X Vectors per application function
64 # for a total of 96 Ingress Queues and MSI-X Vectors on the Unified PF.
65 # (Plus a few for Firmware Event Queues, etc.)
67 # 9. Some customers will want to use T4's PCI-E SR-IOV Capability to allow
68 # Virtual Machines to directly access T4 functionality via SR-IOV
69 # Virtual Functions and "PCI Device Passthrough" -- this is especially
70 # true for the NIC application functionality. (Note that there is
71 # currently no ability to use the TOE, FCoE, iSCSI, etc. via Virtual
72 # Functions so this is in fact solely limited to NIC.)
76 # Global configuration settings.
79 rss_glb_config_mode = basicvirtual
80 rss_glb_config_options = tnlmapen,hashtoeplitz,tnlalllkp
83 pl_timeout_value = 200 # the timeout value in units of us
85 # The following Scatter Gather Engine (SGE) settings assume a 4KB Host
86 # Page Size and a 64B L1 Cache Line Size. It programs the
87 # EgrStatusPageSize and IngPadBoundary to 64B and the PktShift to 2.
88 # If a Master PF Driver finds itself on a machine with different
89 # parameters, then the Master PF Driver is responsible for initializing
90 # these parameters to appropriate values.
93 # 1. The Free List Buffer Sizes below are raw and the firmware will
94 # round them up to the Ingress Padding Boundary.
95 # 2. The SGE Timer Values below are expressed below in microseconds.
96 # The firmware will convert these values to Core Clock Ticks when
97 # it processes the configuration parameters.
99 reg[0x1008] = 0x40810/0x21c70 # SGE_CONTROL
100 reg[0x100c] = 0x22222222 # SGE_HOST_PAGE_SIZE
101 reg[0x10a0] = 0x01040810 # SGE_INGRESS_RX_THRESHOLD
102 reg[0x1044] = 4096 # SGE_FL_BUFFER_SIZE0
103 reg[0x1048] = 65536 # SGE_FL_BUFFER_SIZE1
104 reg[0x104c] = 1536 # SGE_FL_BUFFER_SIZE2
105 reg[0x1050] = 9024 # SGE_FL_BUFFER_SIZE3
106 reg[0x1054] = 9216 # SGE_FL_BUFFER_SIZE4
107 reg[0x1058] = 2048 # SGE_FL_BUFFER_SIZE5
108 reg[0x105c] = 128 # SGE_FL_BUFFER_SIZE6
109 reg[0x1060] = 8192 # SGE_FL_BUFFER_SIZE7
110 reg[0x1064] = 16384 # SGE_FL_BUFFER_SIZE8
111 reg[0x10a4] = 0xa000a000/0xf000f000 # SGE_DBFIFO_STATUS
112 reg[0x10a8] = 0x402000/0x402000 # SGE_DOORBELL_CONTROL
114 # SGE_THROTTLE_CONTROL
115 bar2throttlecount = 500 # bar2throttlecount in us
117 sge_timer_value = 5, 10, 20, 50, 100, 200 # SGE_TIMER_VALUE* in usecs
120 reg[0x1124] = 0x00000400/0x00000400 # SGE_CONTROL2, enable VFIFO; if
121 # SGE_VFIFO_SIZE is not set, then
122 # firmware will set it up in function
123 # of number of egress queues used
125 reg[0x1130] = 0x00d5ffeb # SGE_DBP_FETCH_THRESHOLD, fetch
126 # threshold set to queue depth
127 # minus 128-entries for FL and HP
128 # queues, and 0xfff for LP which
129 # prompts the firmware to set it up
130 # in function of egress queues
133 reg[0x113c] = 0x0002ffc0 # SGE_VFIFO_SIZE, set to 0x2ffc0 which
134 # prompts the firmware to set it up in
135 # function of number of egress queues
138 reg[0x7dc0] = 0x062f8849 # TP_SHIFT_CNT
140 # Selection of tuples for LE filter lookup, fields (and widths which
141 # must sum to <= 36): { IP Fragment (1), MPS Match Type (3),
142 # IP Protocol (8), [Inner] VLAN (17), Port (3), FCoE (1) }
144 filterMode = srvrsram, fragmentation, mpshittype, protocol, vlan, port, fcoe
145 filterMask = protocol, fcoe
147 # Percentage of dynamic memory (in either the EDRAM or external MEM)
148 # to use for TP RX payload
151 # TP RX payload page size
152 tp_pmrx_pagesize = 64K
154 # TP number of RX channels
155 tp_nrxch = 0 # 0 (auto) = 1
157 # Percentage of dynamic memory (in either the EDRAM or external MEM)
158 # to use for TP TX payload
161 # TP TX payload page size
162 tp_pmtx_pagesize = 64K
164 # TP number of TX channels
165 tp_ntxch = 0 # 0 (auto) = equal number of ports
168 reg[0x7d08] = 0x00000800/0x00000800 # set IssFromCplEnable
171 reg[0x19c04] = 0x00400000/0x00400000 # LE Server SRAM Enable
173 # Some "definitions" to make the rest of this a bit more readable. We support
174 # 4 ports, 3 functions (NIC, FCoE and iSCSI), scaling up to 8 "CPU Queue Sets"
175 # per function per port ...
177 # NMSIX = 1088 # available MSI-X Vectors
178 # NVI = 128 # available Virtual Interfaces
179 # NMPSTCAM = 336 # MPS TCAM entries
182 # NCPUS = 8 # CPUs we want to support scalably
183 # NFUNCS = 3 # functions per port (NIC, FCoE, iSCSI)
185 # Breakdown of Virtual Interface/Queue/Interrupt resources for the "Unified
186 # PF" which many OS Drivers will use to manage most or all functions.
188 # Each Ingress Queue can use one MSI-X interrupt but some Ingress Queues can
189 # use Forwarded Interrupt Ingress Queues. For these latter, an Ingress Queue
190 # would be created and the Queue ID of a Forwarded Interrupt Ingress Queue
191 # will be specified as the "Ingress Queue Asynchronous Destination Index."
192 # Thus, the number of MSI-X Vectors assigned to the Unified PF will be less
193 # than or equal to the number of Ingress Queues ...
195 # NVI_NIC = 4 # NIC access to NPORTS
196 # NFLIQ_NIC = 32 # NIC Ingress Queues with Free Lists
197 # NETHCTRL_NIC = 32 # NIC Ethernet Control/TX Queues
198 # NEQ_NIC = 64 # NIC Egress Queues (FL, ETHCTRL/TX)
199 # NMPSTCAM_NIC = 16 # NIC MPS TCAM Entries (NPORTS*4)
200 # NMSIX_NIC = 32 # NIC MSI-X Interrupt Vectors (FLIQ)
202 # NVI_OFLD = 0 # Offload uses NIC function to access ports
203 # NFLIQ_OFLD = 16 # Offload Ingress Queues with Free Lists
204 # NETHCTRL_OFLD = 0 # Offload Ethernet Control/TX Queues
205 # NEQ_OFLD = 16 # Offload Egress Queues (FL)
206 # NMPSTCAM_OFLD = 0 # Offload MPS TCAM Entries (uses NIC's)
207 # NMSIX_OFLD = 16 # Offload MSI-X Interrupt Vectors (FLIQ)
209 # NVI_RDMA = 0 # RDMA uses NIC function to access ports
210 # NFLIQ_RDMA = 4 # RDMA Ingress Queues with Free Lists
211 # NETHCTRL_RDMA = 0 # RDMA Ethernet Control/TX Queues
212 # NEQ_RDMA = 4 # RDMA Egress Queues (FL)
213 # NMPSTCAM_RDMA = 0 # RDMA MPS TCAM Entries (uses NIC's)
214 # NMSIX_RDMA = 4 # RDMA MSI-X Interrupt Vectors (FLIQ)
216 # NEQ_WD = 128 # Wire Direct TX Queues and FLs
217 # NETHCTRL_WD = 64 # Wire Direct TX Queues
218 # NFLIQ_WD = 64 ` # Wire Direct Ingress Queues with Free Lists
220 # NVI_ISCSI = 4 # ISCSI access to NPORTS
221 # NFLIQ_ISCSI = 4 # ISCSI Ingress Queues with Free Lists
222 # NETHCTRL_ISCSI = 0 # ISCSI Ethernet Control/TX Queues
223 # NEQ_ISCSI = 4 # ISCSI Egress Queues (FL)
224 # NMPSTCAM_ISCSI = 4 # ISCSI MPS TCAM Entries (NPORTS)
225 # NMSIX_ISCSI = 4 # ISCSI MSI-X Interrupt Vectors (FLIQ)
227 # NVI_FCOE = 4 # FCOE access to NPORTS
228 # NFLIQ_FCOE = 34 # FCOE Ingress Queues with Free Lists
229 # NETHCTRL_FCOE = 32 # FCOE Ethernet Control/TX Queues
230 # NEQ_FCOE = 66 # FCOE Egress Queues (FL)
231 # NMPSTCAM_FCOE = 32 # FCOE MPS TCAM Entries (NPORTS)
232 # NMSIX_FCOE = 34 # FCOE MSI-X Interrupt Vectors (FLIQ)
234 # Two extra Ingress Queues per function for Firmware Events and Forwarded
235 # Interrupts, and two extra interrupts per function for Firmware Events (or a
236 # Forwarded Interrupt Queue) and General Interrupts per function.
238 # NFLIQ_EXTRA = 6 # "extra" Ingress Queues 2*NFUNCS (Firmware and
239 # # Forwarded Interrupts
240 # NMSIX_EXTRA = 6 # extra interrupts 2*NFUNCS (Firmware and
241 # # General Interrupts
243 # Microsoft HyperV resources. The HyperV Virtual Ingress Queues will have
244 # their interrupts forwarded to another set of Forwarded Interrupt Queues.
246 # NVI_HYPERV = 16 # VMs we want to support
247 # NVIIQ_HYPERV = 2 # Virtual Ingress Queues with Free Lists per VM
248 # NFLIQ_HYPERV = 40 # VIQs + NCPUS Forwarded Interrupt Queues
249 # NEQ_HYPERV = 32 # VIQs Free Lists
250 # NMPSTCAM_HYPERV = 16 # MPS TCAM Entries (NVI_HYPERV)
251 # NMSIX_HYPERV = 8 # NCPUS Forwarded Interrupt Queues
253 # Adding all of the above Unified PF resource needs together: (NIC + OFLD +
254 # RDMA + ISCSI + FCOE + EXTRA + HYPERV)
257 # NFLIQ_UNIFIED = 106
258 # NETHCTRL_UNIFIED = 32
260 # NMPSTCAM_UNIFIED = 40
262 # The sum of all the MSI-X resources above is 74 MSI-X Vectors but we'll round
263 # that up to 128 to make sure the Unified PF doesn't run out of resources.
265 # NMSIX_UNIFIED = 128
267 # The Storage PFs could need up to NPORTS*NCPUS + NMSIX_EXTRA MSI-X Vectors
268 # which is 34 but they're probably safe with 32.
272 # Note: The UnifiedPF is PF4 which doesn't have any Virtual Functions
273 # associated with it. Thus, the MSI-X Vector allocations we give to the
274 # UnifiedPF aren't inherited by any Virtual Functions. As a result we can
275 # provision many more Virtual Functions than we can if the UnifiedPF were
279 # All of the below PCI-E parameters are actually stored in various *_init.txt
280 # files. We include them below essentially as comments.
282 # For PF0-3 we assign 8 vectors each for NIC Ingress Queues of the associated
285 # For PF4, the Unified PF, we give it an MSI-X Table Size as outlined above.
287 # For PF5-6 we assign enough MSI-X Vectors to support FCoE and iSCSI
288 # storage applications across all four possible ports.
290 # Additionally, since the UnifiedPF isn't one of the per-port Physical
291 # Functions, we give the UnifiedPF and the PF0-3 Physical Functions
292 # different PCI Device IDs which will allow Unified and Per-Port Drivers
293 # to directly select the type of Physical Function to which they wish to be
296 # Note that the actual values used for the PCI-E Intelectual Property will be
297 # 1 less than those below since that's the way it "counts" things. For
298 # readability, we use the number we actually mean ...
300 # PF0_INT = 8 # NCPUS
301 # PF1_INT = 8 # NCPUS
302 # PF2_INT = 8 # NCPUS
303 # PF3_INT = 8 # NCPUS
304 # PF0_3_INT = 32 # PF0_INT + PF1_INT + PF2_INT + PF3_INT
306 # PF4_INT = 128 # NMSIX_UNIFIED
307 # PF5_INT = 32 # NMSIX_STORAGE
308 # PF6_INT = 32 # NMSIX_STORAGE
309 # PF7_INT = 0 # Nothing Assigned
310 # PF4_7_INT = 192 # PF4_INT + PF5_INT + PF6_INT + PF7_INT
312 # PF0_7_INT = 224 # PF0_3_INT + PF4_7_INT
314 # With the above we can get 17 VFs/PF0-3 (limited by 336 MPS TCAM entries)
315 # but we'll lower that to 16 to make our total 64 and a nice power of 2 ...
319 # For those OSes which manage different ports on different PFs, we need
320 # only enough resources to support a single port's NIC application functions
321 # on PF0-3. The below assumes that we're only doing NIC with NCPUS "Queue
322 # Sets" for ports 0-3. The FCoE and iSCSI functions for such OSes will be
323 # managed on the "storage PFs" (see below).
326 nvf = 16 # NVF on this function
327 wx_caps = all # write/execute permissions for all commands
328 r_caps = all # read permissions for all commands
330 niqflint = 8 # NCPUS "Queue Sets"
331 nethctrl = 8 # NCPUS "Queue Sets"
332 neq = 16 # niqflint + nethctrl Egress Queues
333 nexactf = 8 # number of exact MPSTCAM MAC filters
334 cmask = all # access to all channels
335 pmask = 0x1 # access to only one port
338 nvf = 16 # NVF on this function
339 wx_caps = all # write/execute permissions for all commands
340 r_caps = all # read permissions for all commands
342 niqflint = 8 # NCPUS "Queue Sets"
343 nethctrl = 8 # NCPUS "Queue Sets"
344 neq = 16 # niqflint + nethctrl Egress Queues
345 nexactf = 8 # number of exact MPSTCAM MAC filters
346 cmask = all # access to all channels
347 pmask = 0x2 # access to only one port
350 nvf = 16 # NVF on this function
351 wx_caps = all # write/execute permissions for all commands
352 r_caps = all # read permissions for all commands
354 niqflint = 8 # NCPUS "Queue Sets"
355 nethctrl = 8 # NCPUS "Queue Sets"
356 neq = 16 # niqflint + nethctrl Egress Queues
357 nexactf = 8 # number of exact MPSTCAM MAC filters
358 cmask = all # access to all channels
359 pmask = 0x4 # access to only one port
362 nvf = 16 # NVF on this function
363 wx_caps = all # write/execute permissions for all commands
364 r_caps = all # read permissions for all commands
366 niqflint = 8 # NCPUS "Queue Sets"
367 nethctrl = 8 # NCPUS "Queue Sets"
368 neq = 16 # niqflint + nethctrl Egress Queues
369 nexactf = 8 # number of exact MPSTCAM MAC filters
370 cmask = all # access to all channels
371 pmask = 0x8 # access to only one port
373 # Some OS Drivers manage all application functions for all ports via PF4.
374 # Thus we need to provide a large number of resources here. For Egress
375 # Queues we need to account for both TX Queues as well as Free List Queues
376 # (because the host is responsible for producing Free List Buffers for the
377 # hardware to consume).
380 wx_caps = all # write/execute permissions for all commands
381 r_caps = all # read permissions for all commands
382 nvi = 28 # NVI_UNIFIED
383 niqflint = 170 # NFLIQ_UNIFIED + NLFIQ_WD
384 nethctrl = 100 # NETHCTRL_UNIFIED + NETHCTRL_WD
385 neq = 256 # NEQ_UNIFIED + NEQ_WD
386 nexactf = 40 # NMPSTCAM_UNIFIED
387 cmask = all # access to all channels
388 pmask = all # access to all four ports ...
389 nethofld = 1024 # number of user mode ethernet flow contexts
390 nroute = 32 # number of routing region entries
391 nclip = 32 # number of clip region entries
392 nfilter = 496 # number of filter region entries
393 nserver = 496 # number of server region entries
394 nhash = 12288 # number of hash region entries
395 protocol = nic_vm, ofld, rddp, rdmac, iscsi_initiator_pdu, iscsi_target_pdu
403 # We have FCoE and iSCSI storage functions on PF5 and PF6 each of which may
404 # need to have Virtual Interfaces on each of the four ports with up to NCPUS
408 wx_caps = all # write/execute permissions for all commands
409 r_caps = all # read permissions for all commands
411 niqflint = 34 # NPORTS*NCPUS + NMSIX_EXTRA
412 nethctrl = 32 # NPORTS*NCPUS
413 neq = 64 # NPORTS*NCPUS * 2 (FL, ETHCTRL/TX)
415 cmask = all # access to all channels
416 pmask = all # access to all four ports ...
420 protocol = iscsi_initiator_fofld
424 iscsi_nconn_per_session = 1
425 iscsi_ninitiator_instance = 64
428 wx_caps = all # write/execute permissions for all commands
429 r_caps = all # read permissions for all commands
431 niqflint = 34 # NPORTS*NCPUS + NMSIX_EXTRA
432 nethctrl = 32 # NPORTS*NCPUS
433 neq = 66 # NPORTS*NCPUS * 2 (FL, ETHCTRL/TX) + 2 (EXTRA)
434 nexactf = 32 # NPORTS + adding 28 exact entries for FCoE
435 # which is OK since < MIN(SUM PF0..3, PF4)
436 # and we never load PF0..3 and PF4 concurrently
437 cmask = all # access to all channels
438 pmask = all # access to all four ports ...
440 protocol = fcoe_initiator
446 # The following function, 1023, is not an actual PCIE function but is used to
447 # configure and reserve firmware internal resources that come from the global
451 wx_caps = all # write/execute permissions for all commands
452 r_caps = all # read permissions for all commands
453 nvi = 4 # NVI_UNIFIED
454 cmask = all # access to all channels
455 pmask = all # access to all four ports ...
456 nexactf = 8 # NPORTS + DCBX +
457 nfilter = 16 # number of filter region entries
459 # For Virtual functions, we only allow NIC functionality and we only allow
460 # access to one port (1 << PF). Note that because of limitations in the
461 # Scatter Gather Engine (SGE) hardware which checks writes to VF KDOORBELL
462 # and GTS registers, the number of Ingress and Egress Queues must be a power
465 [function "0/*"] # NVF
466 wx_caps = 0x82 # DMAQ | VF
467 r_caps = 0x86 # DMAQ | VF | PORT
469 niqflint = 4 # 2 "Queue Sets" + NXIQ
470 nethctrl = 2 # 2 "Queue Sets"
471 neq = 4 # 2 "Queue Sets" * 2
473 cmask = all # access to all channels
474 pmask = 0x1 # access to only one port ...
476 [function "1/*"] # NVF
477 wx_caps = 0x82 # DMAQ | VF
478 r_caps = 0x86 # DMAQ | VF | PORT
480 niqflint = 4 # 2 "Queue Sets" + NXIQ
481 nethctrl = 2 # 2 "Queue Sets"
482 neq = 4 # 2 "Queue Sets" * 2
484 cmask = all # access to all channels
485 pmask = 0x2 # access to only one port ...
487 [function "2/*"] # NVF
488 wx_caps = 0x82 # DMAQ | VF
489 r_caps = 0x86 # DMAQ | VF | PORT
491 niqflint = 4 # 2 "Queue Sets" + NXIQ
492 nethctrl = 2 # 2 "Queue Sets"
493 neq = 4 # 2 "Queue Sets" * 2
495 cmask = all # access to all channels
496 pmask = 0x4 # access to only one port ...
498 [function "3/*"] # NVF
499 wx_caps = 0x82 # DMAQ | VF
500 r_caps = 0x86 # DMAQ | VF | PORT
502 niqflint = 4 # 2 "Queue Sets" + NXIQ
503 nethctrl = 2 # 2 "Queue Sets"
504 neq = 4 # 2 "Queue Sets" * 2
506 cmask = all # access to all channels
507 pmask = 0x8 # access to only one port ...
509 # MPS features a 196608 bytes ingress buffer that is used for ingress buffering
510 # for packets from the wire as well as the loopback path of the L2 switch. The
511 # folling params control how the buffer memory is distributed and the L2 flow
514 # bg_mem: %-age of mem to use for port/buffer group
515 # lpbk_mem: %-age of port/bg mem to use for loopback
516 # hwm: high watermark; bytes available when starting to send pause
517 # frames (in units of 0.1 MTU)
518 # lwm: low watermark; bytes remaining when sending 'unpause' frame
519 # (in inuits of 0.1 MTU)
520 # dwm: minimum delta between high and low watermark (in units of 100
524 dcb = ppp, dcbx # configure for DCB PPP and enable DCBX offload
557 checksum = 0x23a2d850
559 # Total resources used by above allocations:
560 # Virtual Interfaces: 104
561 # Ingress Queues/w Free Lists and Interrupts: 526
563 # MPS TCAM Entries: 336
565 # Virtual Functions: 64