2 * Copyright (c) 2004-2006 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 #endif /* HAVE_CONFIG_H */
41 #include <vendor/osm_vendor_mlx_sender.h>
42 #include <vendor/osm_vendor_mlx_transport.h>
43 #include <vendor/osm_vendor_mlx_svc.h>
44 #include <vendor/osm_pkt_randomizer.h>
46 static ib_api_status_t
47 __osmv_rmpp_send_segment(IN osm_bind_handle_t h_bind,
48 IN osmv_txn_ctx_t * p_txn, IN uint32_t seg_num);
50 /****d* OSM Vendor/osmv_simple_send_madw
52 * osmv_simple_send_madw
55 * Send a single MAD (256 bytes).
57 * If this MAD requires a response, set the timeout event.
58 * The function call returns when the MAD's send completion is received.
63 osmv_simple_send_madw(IN osm_bind_handle_t h_bind,
64 IN osm_madw_t * const p_madw,
65 IN osmv_txn_ctx_t * p_txn, IN boolean_t is_retry)
68 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
69 osm_mad_addr_t *p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
70 uint8_t mad_buf[MAD_BLOCK_SIZE];
71 ib_mad_t *p_mad = (ib_mad_t *) mad_buf;
74 OSM_LOG_ENTER(p_bo->p_vendor->p_log);
76 CL_ASSERT(p_madw->mad_size <= MAD_BLOCK_SIZE);
78 memset(p_mad, 0, MAD_BLOCK_SIZE);
79 memcpy(p_mad, osm_madw_get_mad_ptr(p_madw), p_madw->mad_size);
82 /* Push a fake txn id to the MAD */
83 key = osmv_txn_get_key(p_txn);
84 p_mad->trans_id = cl_hton64(key);
88 Add call for packet drop randomizer.
89 This is a testing feature. If run_randomizer flag is set to TRUE,
90 the randomizer will be called, and randomally will drop
91 a packet. This is used for simulating unstable fabric.
93 if (p_bo->p_vendor->run_randomizer == TRUE) {
94 /* Try the randomizer */
95 if (osm_pkt_randomizer_mad_drop(p_bo->p_vendor->p_log,
99 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
100 "The MAD will not be sent. \n");
104 osmv_transport_mad_send(h_bind, p_mad, p_mad_addr);
107 ret = osmv_transport_mad_send(h_bind, p_mad, p_mad_addr);
110 if ((IB_SUCCESS == ret) && (NULL != p_txn) && (!is_retry)) {
111 /* Set the timeout for receiving the response MAD */
112 ret = osmv_txn_set_timeout_ev(h_bind, key,
113 p_bo->p_vendor->resp_timeout);
116 OSM_LOG_EXIT(p_bo->p_vendor->p_log);
120 /***** OSM Vendor/osmv_rmpp_send_madw
122 * osmv_rmpp_send_madw
125 * Send a single message (MAD wrapper of arbitrary length).
126 * Follow the RMPP semantics
127 * (segmentation, send window, timeouts etc).
129 * The function call returns either when the whole message
130 * has been acknowledged, or upon error.
133 * The RMPP sender context is set up
137 osmv_rmpp_send_madw(IN osm_bind_handle_t h_bind,
138 IN osm_madw_t * const p_madw,
139 IN osmv_txn_ctx_t * p_txn, IN boolean_t is_rmpp_ds)
141 ib_api_status_t ret = IB_SUCCESS;
142 uint32_t i, total_segs;
144 osmv_rmpp_send_ctx_t *p_send_ctx = osmv_txn_get_rmpp_send_ctx(p_txn);
145 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
147 OSM_LOG_ENTER(p_bo->p_vendor->p_log);
149 total_segs = osmv_rmpp_send_ctx_get_num_segs(p_send_ctx);
150 CL_ASSERT(total_segs >= 1);
152 /* In the double-sided transfer, wait for ACK 0 */
156 if (p_send_ctx->window_first > total_segs) {
158 /* Every segment is acknowledged */
162 /* Send the next burst. */
163 for (i = p_send_ctx->window_first; i <= p_send_ctx->window_last;
166 /* Send a segment and setup a timeout timer */
167 ret = __osmv_rmpp_send_segment(h_bind, p_txn, i);
168 if (IB_SUCCESS != ret) {
173 /* Set the Response Timeout for the ACK on the last DATA segment */
174 ret = osmv_txn_set_timeout_ev(h_bind, osmv_txn_get_key(p_txn),
175 p_bo->p_vendor->resp_timeout);
176 if (IB_SUCCESS != ret) {
180 /* Going to sleep. Let the others access the transaction DB */
181 osmv_txn_unlock(p_bo);
183 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
184 "RMPP Sender thread (madw=%p) going to sleep ...\n",
187 /* Await the next event to happen */
188 cl_event_wait_on(&p_send_ctx->event,
189 EVENT_NO_TIMEOUT, TRUE /* interruptible */ );
191 /* Got a signal from the MAD dispatcher/timeout handler */
192 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
193 "RMPP Sender thread (madw=%p) waking up on a signal ...\n",
196 /* Let's see what changed... Make this atomic - re-acquire the lock. */
199 if (TRUE == p_bo->is_closing) {
200 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
201 "osmv_rmpp_send_madw: ERR 6601: "
202 "The bind handle %p is being closed. "
203 "Stopping the RMPP Send of MADW %p\n",
207 return IB_INTERRUPTED;
210 /* STOP? ABORT? TIMEOUT? */
211 if (IB_SUCCESS != p_send_ctx->status) {
212 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
213 "osmv_rmpp_send_madw: ERR 6602: "
214 "An error (%s) happened during the RMPP send of %p. Bailing out.\n",
215 ib_get_err_str(p_send_ctx->status), p_madw);
216 ret = p_send_ctx->status;
221 if (TRUE == is_rmpp_ds) {
222 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
223 "Double-sided RMPP - switching to be the receiver.\n");
225 ret = osmv_txn_init_rmpp_receiver(h_bind, p_txn, FALSE
226 /*Send was initiated by me */
229 if (IB_SUCCESS == ret) {
230 /* Send ACK on the 0 segment */
231 ret = __osmv_rmpp_send_segment(h_bind, p_txn, 0);
236 OSM_LOG_EXIT(p_bo->p_vendor->p_log);
241 * NAME osmv_rmpp_send_ack
248 osmv_rmpp_send_ack(IN osm_bind_handle_t h_bind,
249 IN const ib_mad_t * p_req_mad,
251 IN uint32_t nwl, IN const osm_mad_addr_t * p_mad_addr)
253 uint8_t resp_mad[MAD_BLOCK_SIZE];
254 ib_rmpp_mad_t *p_resp_mad = (ib_rmpp_mad_t *) resp_mad;
256 #ifdef OSMV_RANDOM_DROP
257 if (TRUE == osmv_random_drop()) {
258 osm_log(((osmv_bind_obj_t *) h_bind)->p_vendor->p_log,
260 "Error injection - dropping the RMPP ACK\n");
265 memcpy(p_resp_mad, p_req_mad, MAD_BLOCK_SIZE);
267 p_resp_mad->common_hdr.method = osmv_invert_method(p_req_mad->method);
268 p_resp_mad->rmpp_type = IB_RMPP_TYPE_ACK;
269 p_resp_mad->seg_num = cl_hton32(seg_num);
270 p_resp_mad->paylen_newwin = cl_hton32(nwl);
271 p_resp_mad->rmpp_flags = IB_RMPP_FLAG_ACTIVE;
273 return osmv_transport_mad_send(h_bind, p_resp_mad, p_mad_addr);
277 * NAME osmv_rmpp_send_nak
279 * DESCRIPTION Send the RMPP ABORT or STOP packet
283 osmv_rmpp_send_nak(IN osm_bind_handle_t h_bind,
284 IN const ib_mad_t * p_req_mad,
285 IN const osm_mad_addr_t * p_mad_addr,
286 IN uint8_t nak_type, IN uint8_t status)
288 uint8_t resp_mad[MAD_BLOCK_SIZE];
289 ib_rmpp_mad_t *p_resp_mad = (ib_rmpp_mad_t *) resp_mad;
291 memcpy(p_resp_mad, p_req_mad, MAD_BLOCK_SIZE);
293 p_resp_mad->common_hdr.method = osmv_invert_method(p_req_mad->method);
294 p_resp_mad->rmpp_type = nak_type;
295 p_resp_mad->rmpp_status = status;
297 return osmv_transport_mad_send(h_bind, p_resp_mad, p_mad_addr);
301 * NAME __osmv_rmpp_send_segment
303 * DESCRIPTION Build a MAD for a specific segment and send it
306 static ib_api_status_t
307 __osmv_rmpp_send_segment(IN osm_bind_handle_t h_bind,
308 IN osmv_txn_ctx_t * p_txn, IN uint32_t seg_num)
311 osmv_rmpp_send_ctx_t *p_send_ctx;
312 uint8_t mad_buf[MAD_BLOCK_SIZE];
313 ib_mad_t *p_mad = (ib_mad_t *) mad_buf;
314 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
315 osm_mad_addr_t *p_mad_addr =
316 osm_madw_get_mad_addr_ptr(osmv_txn_get_madw(p_txn));
317 uint32_t timeout = p_bo->p_vendor->resp_timeout;
320 OSM_LOG_ENTER(p_bo->p_vendor->p_log);
322 #ifdef OSMV_RANDOM_DROP
323 if (TRUE == osmv_random_drop()) {
325 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
326 "Error injection - simulating the RMPP segment drop\n");
331 p_send_ctx = osmv_txn_get_rmpp_send_ctx(p_txn);
332 key = osmv_txn_get_key(p_txn);
336 osmv_rmpp_send_ctx_get_seg(p_send_ctx, seg_num, timeout,
338 CL_ASSERT(IB_SUCCESS == ret);
340 /* Put the segment to the wire ! */
341 p_mad->trans_id = cl_hton64(key);
343 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
344 "Sending RMPP segment #%d, on-wire TID=0x%llX\n",
345 seg_num, p_mad->trans_id);
348 Add call for packet drop randomizer.
349 This is a testing feature. If run_randomizer flag is set to TRUE,
350 the randomizer will be called, and randomally will drop
351 a packet. This is used for simulating unstable fabric.
353 if (p_bo->p_vendor->run_randomizer == TRUE) {
354 /* Try the randomizer */
355 if (osm_pkt_randomizer_mad_drop(p_bo->p_vendor->p_log,
359 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
360 "The MAD will not be sent. \n");
364 osmv_transport_mad_send((osm_bind_handle_t)
370 osmv_transport_mad_send((osm_bind_handle_t) p_bo,
374 /* This is an ACK for double-sided handshake. Give it a special treatment. */
376 /* It doesn't really matter which data to put. Only the header matters. */
377 ret = osmv_rmpp_send_ctx_get_seg(p_send_ctx, 1, timeout, p_mad);
378 CL_ASSERT(IB_SUCCESS == ret);
380 p_mad->trans_id = cl_hton64(key);
382 osmv_rmpp_send_ack((osm_bind_handle_t) p_bo, p_mad,
384 OSMV_RMPP_RECV_WIN /* NWL */ ,
388 OSM_LOG_EXIT(p_bo->p_vendor->p_log);