sys/dev/hyperv/vmbus/hv_ring_buffer.c

   1 /*-
   2  * Copyright (c) 2009-2012 Microsoft Corp.
   3  * Copyright (c) 2012 NetApp Inc.
   4  * Copyright (c) 2012 Citrix Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/lock.h>
  34 #include <sys/mutex.h>
  35
  36 #include "hv_vmbus_priv.h"
  37
  38 /* Amount of space to write to */
  39 #define HV_BYTES_AVAIL_TO_WRITE(r, w, z) ((w) >= (r))? \
  40                                 ((z) - ((w) - (r))):((r) - (w))
  41
  42 /**
  43  * @brief Get number of bytes available to read and to write to
  44  * for the specified ring buffer
  45  */
  46 static inline void
  47 get_ring_buffer_avail_bytes(
  48             hv_vmbus_ring_buffer_info*  rbi,
  49             uint32_t*                   read,
  50             uint32_t*                   write)
  51 {
  52         uint32_t read_loc, write_loc;
  53
  54         /*
  55          * Capture the read/write indices before they changed
  56          */
  57         read_loc = rbi->ring_buffer->read_index;
  58         write_loc = rbi->ring_buffer->write_index;
  59
  60         *write = HV_BYTES_AVAIL_TO_WRITE(
  61                 read_loc, write_loc, rbi->ring_data_size);
  62         *read = rbi->ring_data_size - *write;
  63 }
  64
  65 /**
  66  * @brief Get the next write location for the specified ring buffer
  67  */
  68 static inline uint32_t
  69 get_next_write_location(hv_vmbus_ring_buffer_info* ring_info)
  70 {
  71         uint32_t next = ring_info->ring_buffer->write_index;
  72         return (next);
  73 }
  74
  75 /**
  76  * @brief Set the next write location for the specified ring buffer
  77  */
  78 static inline void
  79 set_next_write_location(
  80         hv_vmbus_ring_buffer_info*      ring_info,
  81         uint32_t                        next_write_location)
  82 {
  83         ring_info->ring_buffer->write_index = next_write_location;
  84 }
  85
  86 /**
  87  * @brief Get the next read location for the specified ring buffer
  88  */
  89 static inline uint32_t
  90 get_next_read_location(hv_vmbus_ring_buffer_info* ring_info)
  91 {
  92         uint32_t next = ring_info->ring_buffer->read_index;
  93         return (next);
  94 }
  95
  96 /**
  97  * @brief Get the next read location + offset for the specified ring buffer.
  98  * This allows the caller to skip.
  99  */
 100 static inline uint32_t
 101 get_next_read_location_with_offset(
 102         hv_vmbus_ring_buffer_info*      ring_info,
 103         uint32_t                        offset)
 104 {
 105         uint32_t next = ring_info->ring_buffer->read_index;
 106         next += offset;
 107         next %= ring_info->ring_data_size;
 108         return (next);
 109 }
 110
 111 /**
 112  * @brief Set the next read location for the specified ring buffer
 113  */
 114 static inline void
 115 set_next_read_location(
 116         hv_vmbus_ring_buffer_info*      ring_info,
 117         uint32_t                        next_read_location)
 118 {
 119         ring_info->ring_buffer->read_index = next_read_location;
 120 }
 121
 122 /**
 123  * @brief Get the start of the ring buffer
 124  */
 125 static inline void *
 126 get_ring_buffer(hv_vmbus_ring_buffer_info* ring_info)
 127 {
 128         return (void *) ring_info->ring_buffer->buffer;
 129 }
 130
 131 /**
 132  * @brief Get the size of the ring buffer.
 133  */
 134 static inline uint32_t
 135 get_ring_buffer_size(hv_vmbus_ring_buffer_info* ring_info)
 136 {
 137         return ring_info->ring_data_size;
 138 }
 139
 140 /**
 141  * Get the read and write indices as uint64_t of the specified ring buffer.
 142  */
 143 static inline uint64_t
 144 get_ring_buffer_indices(hv_vmbus_ring_buffer_info* ring_info)
 145 {
 146         return (uint64_t) ring_info->ring_buffer->write_index << 32;
 147 }
 148
 149 void
 150 hv_ring_buffer_read_begin(
 151         hv_vmbus_ring_buffer_info*      ring_info)
 152 {
 153         ring_info->ring_buffer->interrupt_mask = 1;
 154         mb();
 155 }
 156
 157 uint32_t
 158 hv_ring_buffer_read_end(
 159         hv_vmbus_ring_buffer_info*      ring_info)
 160 {
 161         uint32_t read, write;
 162
 163         ring_info->ring_buffer->interrupt_mask = 0;
 164         mb();
 165
 166         /*
 167          * Now check to see if the ring buffer is still empty.
 168          * If it is not, we raced and we need to process new
 169          * incoming messages.
 170          */
 171         get_ring_buffer_avail_bytes(ring_info, &read, &write);
 172
 173         return (read);
 174 }
 175
 176 /*
 177  * When we write to the ring buffer, check if the host needs to
 178  * be signaled. Here is the details of this protocol:
 179  *
 180  *      1. The host guarantees that while it is draining the
 181  *         ring buffer, it will set the interrupt_mask to
 182  *         indicate it does not need to be interrupted when
 183  *         new data is placed.
 184  *
 185  *      2. The host guarantees that it will completely drain
 186  *         the ring buffer before exiting the read loop. Further,
 187  *         once the ring buffer is empty, it will clear the
 188  *         interrupt_mask and re-check to see if new data has
 189  *         arrived.
 190  */
 191 static boolean_t
 192 hv_ring_buffer_needsig_on_write(
 193         uint32_t                        old_write_location,
 194         hv_vmbus_ring_buffer_info*      rbi)
 195 {
 196         mb();
 197         if (rbi->ring_buffer->interrupt_mask)
 198                 return (FALSE);
 199
 200         /* Read memory barrier */
 201         rmb();
 202         /*
 203          * This is the only case we need to signal when the
 204          * ring transitions from being empty to non-empty.
 205          */
 206         if (old_write_location == rbi->ring_buffer->read_index)
 207                 return (TRUE);
 208
 209         return (FALSE);
 210 }
 211
 212 static uint32_t copy_to_ring_buffer(
 213                         hv_vmbus_ring_buffer_info*      ring_info,
 214                         uint32_t                        start_write_offset,
 215                         char*                           src,
 216                         uint32_t                        src_len);
 217
 218 static uint32_t copy_from_ring_buffer(
 219                         hv_vmbus_ring_buffer_info*      ring_info,
 220                         char*                           dest,
 221                         uint32_t                        dest_len,
 222                         uint32_t                        start_read_offset);
 223
 224
 225 /**
 226  * @brief Get the interrupt mask for the specified ring buffer.
 227  */
 228 uint32_t
 229 hv_vmbus_get_ring_buffer_interrupt_mask(hv_vmbus_ring_buffer_info *rbi)
 230 {
 231         return rbi->ring_buffer->interrupt_mask;
 232 }
 233
 234 /**
 235  * @brief Initialize the ring buffer.
 236  */
 237 int
 238 hv_vmbus_ring_buffer_init(
 239         hv_vmbus_ring_buffer_info*      ring_info,
 240         void*                           buffer,
 241         uint32_t                        buffer_len)
 242 {
 243         memset(ring_info, 0, sizeof(hv_vmbus_ring_buffer_info));
 244
 245         ring_info->ring_buffer = (hv_vmbus_ring_buffer*) buffer;
 246         ring_info->ring_buffer->read_index =
 247             ring_info->ring_buffer->write_index = 0;
 248
 249         ring_info->ring_size = buffer_len;
 250         ring_info->ring_data_size = buffer_len - sizeof(hv_vmbus_ring_buffer);
 251
 252         mtx_init(&ring_info->ring_lock, "vmbus ring buffer", NULL, MTX_SPIN);
 253
 254         return (0);
 255 }
 256
 257 /**
 258  * @brief Cleanup the ring buffer.
 259  */
 260 void hv_ring_buffer_cleanup(hv_vmbus_ring_buffer_info* ring_info)
 261 {
 262         mtx_destroy(&ring_info->ring_lock);
 263 }
 264
 265 /**
 266  * @brief Write to the ring buffer.
 267  */
 268 int
 269 hv_ring_buffer_write(
 270         hv_vmbus_ring_buffer_info*      out_ring_info,
 271         hv_vmbus_sg_buffer_list         sg_buffers[],
 272         uint32_t                        sg_buffer_count,
 273         boolean_t                       *need_sig)
 274 {
 275         int i = 0;
 276         uint32_t byte_avail_to_write;
 277         uint32_t byte_avail_to_read;
 278         uint32_t old_write_location;
 279         uint32_t total_bytes_to_write = 0;
 280
 281         volatile uint32_t next_write_location;
 282         uint64_t prev_indices = 0;
 283
 284         for (i = 0; i < sg_buffer_count; i++) {
 285             total_bytes_to_write += sg_buffers[i].length;
 286         }
 287
 288         total_bytes_to_write += sizeof(uint64_t);
 289
 290         mtx_lock_spin(&out_ring_info->ring_lock);
 291
 292         get_ring_buffer_avail_bytes(out_ring_info, &byte_avail_to_read,
 293             &byte_avail_to_write);
 294
 295         /*
 296          * If there is only room for the packet, assume it is full.
 297          * Otherwise, the next time around, we think the ring buffer
 298          * is empty since the read index == write index
 299          */
 300
 301         if (byte_avail_to_write <= total_bytes_to_write) {
 302
 303             mtx_unlock_spin(&out_ring_info->ring_lock);
 304             return (EAGAIN);
 305         }
 306
 307         /*
 308          * Write to the ring buffer
 309          */
 310         next_write_location = get_next_write_location(out_ring_info);
 311
 312         old_write_location = next_write_location;
 313
 314         for (i = 0; i < sg_buffer_count; i++) {
 315             next_write_location = copy_to_ring_buffer(out_ring_info,
 316                 next_write_location, (char *) sg_buffers[i].data,
 317                 sg_buffers[i].length);
 318         }
 319
 320         /*
 321          * Set previous packet start
 322          */
 323         prev_indices = get_ring_buffer_indices(out_ring_info);
 324
 325         next_write_location = copy_to_ring_buffer(
 326                 out_ring_info, next_write_location,
 327                 (char *) &prev_indices, sizeof(uint64_t));
 328
 329         /*
 330          * Full memory barrier before upding the write index.
 331          */
 332         mb();
 333
 334         /*
 335          * Now, update the write location
 336          */
 337         set_next_write_location(out_ring_info, next_write_location);
 338
 339         mtx_unlock_spin(&out_ring_info->ring_lock);
 340
 341         *need_sig = hv_ring_buffer_needsig_on_write(old_write_location,
 342             out_ring_info);
 343
 344         return (0);
 345 }
 346
 347 /**
 348  * @brief Read without advancing the read index.
 349  */
 350 int
 351 hv_ring_buffer_peek(
 352         hv_vmbus_ring_buffer_info*      in_ring_info,
 353         void*                           buffer,
 354         uint32_t                        buffer_len)
 355 {
 356         uint32_t bytesAvailToWrite;
 357         uint32_t bytesAvailToRead;
 358         uint32_t nextReadLocation = 0;
 359
 360         mtx_lock_spin(&in_ring_info->ring_lock);
 361
 362         get_ring_buffer_avail_bytes(in_ring_info, &bytesAvailToRead,
 363                 &bytesAvailToWrite);
 364
 365         /*
 366          * Make sure there is something to read
 367          */
 368         if (bytesAvailToRead < buffer_len) {
 369             mtx_unlock_spin(&in_ring_info->ring_lock);
 370             return (EAGAIN);
 371         }
 372
 373         /*
 374          * Convert to byte offset
 375          */
 376         nextReadLocation = get_next_read_location(in_ring_info);
 377
 378         nextReadLocation = copy_from_ring_buffer(
 379                 in_ring_info, (char *)buffer, buffer_len, nextReadLocation);
 380
 381         mtx_unlock_spin(&in_ring_info->ring_lock);
 382
 383         return (0);
 384 }
 385
 386 /**
 387  * @brief Read and advance the read index.
 388  */
 389 int
 390 hv_ring_buffer_read(
 391         hv_vmbus_ring_buffer_info*      in_ring_info,
 392         void*                           buffer,
 393         uint32_t                        buffer_len,
 394         uint32_t                        offset)
 395 {
 396         uint32_t bytes_avail_to_write;
 397         uint32_t bytes_avail_to_read;
 398         uint32_t next_read_location = 0;
 399         uint64_t prev_indices = 0;
 400
 401         if (buffer_len <= 0)
 402             return (EINVAL);
 403
 404         mtx_lock_spin(&in_ring_info->ring_lock);
 405
 406         get_ring_buffer_avail_bytes(
 407             in_ring_info, &bytes_avail_to_read,
 408             &bytes_avail_to_write);
 409
 410         /*
 411          * Make sure there is something to read
 412          */
 413         if (bytes_avail_to_read < buffer_len) {
 414             mtx_unlock_spin(&in_ring_info->ring_lock);
 415             return (EAGAIN);
 416         }
 417
 418         next_read_location = get_next_read_location_with_offset(
 419             in_ring_info,
 420             offset);
 421
 422         next_read_location = copy_from_ring_buffer(
 423             in_ring_info,
 424             (char *) buffer,
 425             buffer_len,
 426             next_read_location);
 427
 428         next_read_location = copy_from_ring_buffer(
 429             in_ring_info,
 430             (char *) &prev_indices,
 431             sizeof(uint64_t),
 432             next_read_location);
 433
 434         /*
 435          * Make sure all reads are done before we update the read index since
 436          * the writer may start writing to the read area once the read index
 437          * is updated.
 438          */
 439         wmb();
 440
 441         /*
 442          * Update the read index
 443          */
 444         set_next_read_location(in_ring_info, next_read_location);
 445
 446         mtx_unlock_spin(&in_ring_info->ring_lock);
 447
 448         return (0);
 449 }
 450
 451 /**
 452  * @brief Helper routine to copy from source to ring buffer.
 453  *
 454  * Assume there is enough room. Handles wrap-around in dest case only!
 455  */
 456 uint32_t
 457 copy_to_ring_buffer(
 458         hv_vmbus_ring_buffer_info*      ring_info,
 459         uint32_t                        start_write_offset,
 460         char*                           src,
 461         uint32_t                        src_len)
 462 {
 463         char *ring_buffer = get_ring_buffer(ring_info);
 464         uint32_t ring_buffer_size = get_ring_buffer_size(ring_info);
 465         uint32_t fragLen;
 466
 467         if (src_len > ring_buffer_size - start_write_offset)  {
 468             /* wrap-around detected! */
 469             fragLen = ring_buffer_size - start_write_offset;
 470             memcpy(ring_buffer + start_write_offset, src, fragLen);
 471             memcpy(ring_buffer, src + fragLen, src_len - fragLen);
 472         } else {
 473             memcpy(ring_buffer + start_write_offset, src, src_len);
 474         }
 475
 476         start_write_offset += src_len;
 477         start_write_offset %= ring_buffer_size;
 478
 479         return (start_write_offset);
 480 }
 481
 482 /**
 483  * @brief Helper routine to copy to source from ring buffer.
 484  *
 485  * Assume there is enough room. Handles wrap-around in src case only!
 486  */
 487 uint32_t
 488 copy_from_ring_buffer(
 489         hv_vmbus_ring_buffer_info*      ring_info,
 490         char*                           dest,
 491         uint32_t                        dest_len,
 492         uint32_t                        start_read_offset)
 493 {
 494         uint32_t fragLen;
 495         char *ring_buffer = get_ring_buffer(ring_info);
 496         uint32_t ring_buffer_size = get_ring_buffer_size(ring_info);
 497
 498         if (dest_len > ring_buffer_size - start_read_offset) {
 499             /*  wrap-around detected at the src */
 500             fragLen = ring_buffer_size - start_read_offset;
 501             memcpy(dest, ring_buffer + start_read_offset, fragLen);
 502             memcpy(dest + fragLen, ring_buffer, dest_len - fragLen);
 503         } else {
 504             memcpy(dest, ring_buffer + start_read_offset, dest_len);
 505         }
 506
 507         start_read_offset += dest_len;
 508         start_read_offset %= ring_buffer_size;
 509
 510         return (start_read_offset);
 511 }
 512