2 * Copyright (c) 2009-2012 Microsoft Corp.
3 * Copyright (c) 2012 NetApp Inc.
4 * Copyright (c) 2012 Citrix Inc.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
32 #include <sys/param.h>
34 #include <sys/mutex.h>
36 #include "hv_vmbus_priv.h"
38 /* Amount of space to write to */
39 #define HV_BYTES_AVAIL_TO_WRITE(r, w, z) ((w) >= (r))? \
40 ((z) - ((w) - (r))):((r) - (w))
43 * @brief Get number of bytes available to read and to write to
44 * for the specified ring buffer
47 get_ring_buffer_avail_bytes(
48 hv_vmbus_ring_buffer_info* rbi,
52 uint32_t read_loc, write_loc;
55 * Capture the read/write indices before they changed
57 read_loc = rbi->ring_buffer->read_index;
58 write_loc = rbi->ring_buffer->write_index;
60 *write = HV_BYTES_AVAIL_TO_WRITE(
61 read_loc, write_loc, rbi->ring_data_size);
62 *read = rbi->ring_data_size - *write;
66 * @brief Get the next write location for the specified ring buffer
68 static inline uint32_t
69 get_next_write_location(hv_vmbus_ring_buffer_info* ring_info)
71 uint32_t next = ring_info->ring_buffer->write_index;
76 * @brief Set the next write location for the specified ring buffer
79 set_next_write_location(
80 hv_vmbus_ring_buffer_info* ring_info,
81 uint32_t next_write_location)
83 ring_info->ring_buffer->write_index = next_write_location;
87 * @brief Get the next read location for the specified ring buffer
89 static inline uint32_t
90 get_next_read_location(hv_vmbus_ring_buffer_info* ring_info)
92 uint32_t next = ring_info->ring_buffer->read_index;
97 * @brief Get the next read location + offset for the specified ring buffer.
98 * This allows the caller to skip.
100 static inline uint32_t
101 get_next_read_location_with_offset(
102 hv_vmbus_ring_buffer_info* ring_info,
105 uint32_t next = ring_info->ring_buffer->read_index;
107 next %= ring_info->ring_data_size;
112 * @brief Set the next read location for the specified ring buffer
115 set_next_read_location(
116 hv_vmbus_ring_buffer_info* ring_info,
117 uint32_t next_read_location)
119 ring_info->ring_buffer->read_index = next_read_location;
123 * @brief Get the start of the ring buffer
126 get_ring_buffer(hv_vmbus_ring_buffer_info* ring_info)
128 return (void *) ring_info->ring_buffer->buffer;
132 * @brief Get the size of the ring buffer.
134 static inline uint32_t
135 get_ring_buffer_size(hv_vmbus_ring_buffer_info* ring_info)
137 return ring_info->ring_data_size;
141 * Get the read and write indices as uint64_t of the specified ring buffer.
143 static inline uint64_t
144 get_ring_buffer_indices(hv_vmbus_ring_buffer_info* ring_info)
146 return (uint64_t) ring_info->ring_buffer->write_index << 32;
150 hv_ring_buffer_read_begin(
151 hv_vmbus_ring_buffer_info* ring_info)
153 ring_info->ring_buffer->interrupt_mask = 1;
158 hv_ring_buffer_read_end(
159 hv_vmbus_ring_buffer_info* ring_info)
161 uint32_t read, write;
163 ring_info->ring_buffer->interrupt_mask = 0;
167 * Now check to see if the ring buffer is still empty.
168 * If it is not, we raced and we need to process new
171 get_ring_buffer_avail_bytes(ring_info, &read, &write);
177 * When we write to the ring buffer, check if the host needs to
178 * be signaled. Here is the details of this protocol:
180 * 1. The host guarantees that while it is draining the
181 * ring buffer, it will set the interrupt_mask to
182 * indicate it does not need to be interrupted when
183 * new data is placed.
185 * 2. The host guarantees that it will completely drain
186 * the ring buffer before exiting the read loop. Further,
187 * once the ring buffer is empty, it will clear the
188 * interrupt_mask and re-check to see if new data has
192 hv_ring_buffer_needsig_on_write(
193 uint32_t old_write_location,
194 hv_vmbus_ring_buffer_info* rbi)
197 if (rbi->ring_buffer->interrupt_mask)
200 /* Read memory barrier */
203 * This is the only case we need to signal when the
204 * ring transitions from being empty to non-empty.
206 if (old_write_location == rbi->ring_buffer->read_index)
212 static uint32_t copy_to_ring_buffer(
213 hv_vmbus_ring_buffer_info* ring_info,
214 uint32_t start_write_offset,
218 static uint32_t copy_from_ring_buffer(
219 hv_vmbus_ring_buffer_info* ring_info,
222 uint32_t start_read_offset);
226 * @brief Get the interrupt mask for the specified ring buffer.
229 hv_vmbus_get_ring_buffer_interrupt_mask(hv_vmbus_ring_buffer_info *rbi)
231 return rbi->ring_buffer->interrupt_mask;
235 * @brief Initialize the ring buffer.
238 hv_vmbus_ring_buffer_init(
239 hv_vmbus_ring_buffer_info* ring_info,
243 memset(ring_info, 0, sizeof(hv_vmbus_ring_buffer_info));
245 ring_info->ring_buffer = (hv_vmbus_ring_buffer*) buffer;
246 ring_info->ring_buffer->read_index =
247 ring_info->ring_buffer->write_index = 0;
249 ring_info->ring_size = buffer_len;
250 ring_info->ring_data_size = buffer_len - sizeof(hv_vmbus_ring_buffer);
252 mtx_init(&ring_info->ring_lock, "vmbus ring buffer", NULL, MTX_SPIN);
258 * @brief Cleanup the ring buffer.
260 void hv_ring_buffer_cleanup(hv_vmbus_ring_buffer_info* ring_info)
262 mtx_destroy(&ring_info->ring_lock);
266 * @brief Write to the ring buffer.
269 hv_ring_buffer_write(
270 hv_vmbus_ring_buffer_info* out_ring_info,
271 hv_vmbus_sg_buffer_list sg_buffers[],
272 uint32_t sg_buffer_count,
276 uint32_t byte_avail_to_write;
277 uint32_t byte_avail_to_read;
278 uint32_t old_write_location;
279 uint32_t total_bytes_to_write = 0;
281 volatile uint32_t next_write_location;
282 uint64_t prev_indices = 0;
284 for (i = 0; i < sg_buffer_count; i++) {
285 total_bytes_to_write += sg_buffers[i].length;
288 total_bytes_to_write += sizeof(uint64_t);
290 mtx_lock_spin(&out_ring_info->ring_lock);
292 get_ring_buffer_avail_bytes(out_ring_info, &byte_avail_to_read,
293 &byte_avail_to_write);
296 * If there is only room for the packet, assume it is full.
297 * Otherwise, the next time around, we think the ring buffer
298 * is empty since the read index == write index
301 if (byte_avail_to_write <= total_bytes_to_write) {
303 mtx_unlock_spin(&out_ring_info->ring_lock);
308 * Write to the ring buffer
310 next_write_location = get_next_write_location(out_ring_info);
312 old_write_location = next_write_location;
314 for (i = 0; i < sg_buffer_count; i++) {
315 next_write_location = copy_to_ring_buffer(out_ring_info,
316 next_write_location, (char *) sg_buffers[i].data,
317 sg_buffers[i].length);
321 * Set previous packet start
323 prev_indices = get_ring_buffer_indices(out_ring_info);
325 next_write_location = copy_to_ring_buffer(
326 out_ring_info, next_write_location,
327 (char *) &prev_indices, sizeof(uint64_t));
330 * Full memory barrier before upding the write index.
335 * Now, update the write location
337 set_next_write_location(out_ring_info, next_write_location);
339 mtx_unlock_spin(&out_ring_info->ring_lock);
341 *need_sig = hv_ring_buffer_needsig_on_write(old_write_location,
348 * @brief Read without advancing the read index.
352 hv_vmbus_ring_buffer_info* in_ring_info,
356 uint32_t bytesAvailToWrite;
357 uint32_t bytesAvailToRead;
358 uint32_t nextReadLocation = 0;
360 mtx_lock_spin(&in_ring_info->ring_lock);
362 get_ring_buffer_avail_bytes(in_ring_info, &bytesAvailToRead,
366 * Make sure there is something to read
368 if (bytesAvailToRead < buffer_len) {
369 mtx_unlock_spin(&in_ring_info->ring_lock);
374 * Convert to byte offset
376 nextReadLocation = get_next_read_location(in_ring_info);
378 nextReadLocation = copy_from_ring_buffer(
379 in_ring_info, (char *)buffer, buffer_len, nextReadLocation);
381 mtx_unlock_spin(&in_ring_info->ring_lock);
387 * @brief Read and advance the read index.
391 hv_vmbus_ring_buffer_info* in_ring_info,
396 uint32_t bytes_avail_to_write;
397 uint32_t bytes_avail_to_read;
398 uint32_t next_read_location = 0;
399 uint64_t prev_indices = 0;
404 mtx_lock_spin(&in_ring_info->ring_lock);
406 get_ring_buffer_avail_bytes(
407 in_ring_info, &bytes_avail_to_read,
408 &bytes_avail_to_write);
411 * Make sure there is something to read
413 if (bytes_avail_to_read < buffer_len) {
414 mtx_unlock_spin(&in_ring_info->ring_lock);
418 next_read_location = get_next_read_location_with_offset(
422 next_read_location = copy_from_ring_buffer(
428 next_read_location = copy_from_ring_buffer(
430 (char *) &prev_indices,
435 * Make sure all reads are done before we update the read index since
436 * the writer may start writing to the read area once the read index
442 * Update the read index
444 set_next_read_location(in_ring_info, next_read_location);
446 mtx_unlock_spin(&in_ring_info->ring_lock);
452 * @brief Helper routine to copy from source to ring buffer.
454 * Assume there is enough room. Handles wrap-around in dest case only!
458 hv_vmbus_ring_buffer_info* ring_info,
459 uint32_t start_write_offset,
463 char *ring_buffer = get_ring_buffer(ring_info);
464 uint32_t ring_buffer_size = get_ring_buffer_size(ring_info);
467 if (src_len > ring_buffer_size - start_write_offset) {
468 /* wrap-around detected! */
469 fragLen = ring_buffer_size - start_write_offset;
470 memcpy(ring_buffer + start_write_offset, src, fragLen);
471 memcpy(ring_buffer, src + fragLen, src_len - fragLen);
473 memcpy(ring_buffer + start_write_offset, src, src_len);
476 start_write_offset += src_len;
477 start_write_offset %= ring_buffer_size;
479 return (start_write_offset);
483 * @brief Helper routine to copy to source from ring buffer.
485 * Assume there is enough room. Handles wrap-around in src case only!
488 copy_from_ring_buffer(
489 hv_vmbus_ring_buffer_info* ring_info,
492 uint32_t start_read_offset)
495 char *ring_buffer = get_ring_buffer(ring_info);
496 uint32_t ring_buffer_size = get_ring_buffer_size(ring_info);
498 if (dest_len > ring_buffer_size - start_read_offset) {
499 /* wrap-around detected at the src */
500 fragLen = ring_buffer_size - start_read_offset;
501 memcpy(dest, ring_buffer + start_read_offset, fragLen);
502 memcpy(dest + fragLen, ring_buffer, dest_len - fragLen);
504 memcpy(dest, ring_buffer + start_read_offset, dest_len);
507 start_read_offset += dest_len;
508 start_read_offset %= ring_buffer_size;
510 return (start_read_offset);