sys/dev/drm2/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <sys/cdefs.h>
  26 __FBSDID("$FreeBSD$");
  27
  28 #include <dev/drm2/drmP.h>
  29 #include <dev/drm2/drm.h>
  30 #include <dev/drm2/i915/i915_drm.h>
  31 #include <dev/drm2/i915/i915_drv.h>
  32 #include <dev/drm2/i915/intel_drv.h>
  33 #include <sys/sched.h>
  34 #include <sys/sf_buf.h>
  35
  36 /* PPGTT support for Sandybdrige/Gen6 and later */
  37 static void
  38 i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
  39     unsigned first_entry, unsigned num_entries)
  40 {
  41         uint32_t *pt_vaddr;
  42         uint32_t scratch_pte;
  43         struct sf_buf *sf;
  44         unsigned act_pd, first_pte, last_pte, i;
  45
  46         act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
  47         first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  48
  49         scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr);
  50         scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC;
  51
  52         while (num_entries) {
  53                 last_pte = first_pte + num_entries;
  54                 if (last_pte > I915_PPGTT_PT_ENTRIES)
  55                         last_pte = I915_PPGTT_PT_ENTRIES;
  56
  57                 sched_pin();
  58                 sf = sf_buf_alloc(ppgtt->pt_pages[act_pd], SFB_CPUPRIVATE);
  59                 pt_vaddr = (uint32_t *)(uintptr_t)sf_buf_kva(sf);
  60
  61                 for (i = first_pte; i < last_pte; i++)
  62                         pt_vaddr[i] = scratch_pte;
  63
  64                 sf_buf_free(sf);
  65                 sched_unpin();
  66
  67                 num_entries -= last_pte - first_pte;
  68                 first_pte = 0;
  69                 act_pd++;
  70         }
  71
  72 }
  73
  74 int
  75 i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  76 {
  77         struct drm_i915_private *dev_priv;
  78         struct i915_hw_ppgtt *ppgtt;
  79         u_int first_pd_entry_in_global_pt, i;
  80
  81         dev_priv = dev->dev_private;
  82
  83         /*
  84          * ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  85          * entries. For aliasing ppgtt support we just steal them at the end for
  86          * now.
  87          */
  88         first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES;
  89
  90         ppgtt = malloc(sizeof(*ppgtt), DRM_I915_GEM, M_WAITOK | M_ZERO);
  91
  92         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
  93         ppgtt->pt_pages = malloc(sizeof(vm_page_t) * ppgtt->num_pd_entries,
  94             DRM_I915_GEM, M_WAITOK | M_ZERO);
  95
  96         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  97                 ppgtt->pt_pages[i] = vm_page_alloc(NULL, 0,
  98                     VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
  99                     VM_ALLOC_ZERO);
 100                 if (ppgtt->pt_pages[i] == NULL) {
 101                         dev_priv->mm.aliasing_ppgtt = ppgtt;
 102                         i915_gem_cleanup_aliasing_ppgtt(dev);
 103                         return (-ENOMEM);
 104                 }
 105         }
 106
 107         ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt.scratch_page_dma;
 108
 109         i915_ppgtt_clear_range(ppgtt, 0, ppgtt->num_pd_entries *
 110             I915_PPGTT_PT_ENTRIES);
 111         ppgtt->pd_offset = (first_pd_entry_in_global_pt) * sizeof(uint32_t);
 112         dev_priv->mm.aliasing_ppgtt = ppgtt;
 113         return (0);
 114 }
 115
 116 static void
 117 i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt, unsigned first_entry,
 118     unsigned num_entries, vm_page_t *pages, uint32_t pte_flags)
 119 {
 120         uint32_t *pt_vaddr, pte;
 121         struct sf_buf *sf;
 122         unsigned act_pd, first_pte;
 123         unsigned last_pte, i;
 124         vm_paddr_t page_addr;
 125
 126         act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 127         first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 128
 129         while (num_entries) {
 130                 last_pte = first_pte + num_entries;
 131                 if (last_pte > I915_PPGTT_PT_ENTRIES)
 132                         last_pte = I915_PPGTT_PT_ENTRIES;
 133
 134                 sched_pin();
 135                 sf = sf_buf_alloc(ppgtt->pt_pages[act_pd], SFB_CPUPRIVATE);
 136                 pt_vaddr = (uint32_t *)(uintptr_t)sf_buf_kva(sf);
 137
 138                 for (i = first_pte; i < last_pte; i++) {
 139                         page_addr = VM_PAGE_TO_PHYS(*pages);
 140                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 141                         pt_vaddr[i] = pte | pte_flags;
 142
 143                         pages++;
 144                 }
 145
 146                 sf_buf_free(sf);
 147                 sched_unpin();
 148
 149                 num_entries -= last_pte - first_pte;
 150                 first_pte = 0;
 151                 act_pd++;
 152         }
 153 }
 154
 155 void
 156 i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 157     struct drm_i915_gem_object *obj, enum i915_cache_level cache_level)
 158 {
 159         struct drm_device *dev;
 160         struct drm_i915_private *dev_priv;
 161         uint32_t pte_flags;
 162
 163         dev = obj->base.dev;
 164         dev_priv = dev->dev_private;
 165         pte_flags = GEN6_PTE_VALID;
 166
 167         switch (cache_level) {
 168         case I915_CACHE_LLC_MLC:
 169                 pte_flags |= GEN6_PTE_CACHE_LLC_MLC;
 170                 break;
 171         case I915_CACHE_LLC:
 172                 pte_flags |= GEN6_PTE_CACHE_LLC;
 173                 break;
 174         case I915_CACHE_NONE:
 175                 pte_flags |= GEN6_PTE_UNCACHED;
 176                 break;
 177         default:
 178                 panic("cache mode");
 179         }
 180
 181         i915_ppgtt_insert_pages(ppgtt, obj->gtt_space->start >> PAGE_SHIFT,
 182             obj->base.size >> PAGE_SHIFT, obj->pages, pte_flags);
 183 }
 184
 185 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
 186                               struct drm_i915_gem_object *obj)
 187 {
 188         i915_ppgtt_clear_range(ppgtt, obj->gtt_space->start >> PAGE_SHIFT,
 189             obj->base.size >> PAGE_SHIFT);
 190 }
 191
 192 void
 193 i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 194 {
 195         struct drm_i915_private *dev_priv;
 196         struct i915_hw_ppgtt *ppgtt;
 197         vm_page_t m;
 198         int i;
 199
 200         dev_priv = dev->dev_private;
 201         ppgtt = dev_priv->mm.aliasing_ppgtt;
 202         if (ppgtt == NULL)
 203                 return;
 204         dev_priv->mm.aliasing_ppgtt = NULL;
 205
 206         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 207                 m = ppgtt->pt_pages[i];
 208                 if (m != NULL) {
 209                         vm_page_unwire(m, 0);
 210                         vm_page_free(m);
 211                 }
 212         }
 213         free(ppgtt->pt_pages, DRM_I915_GEM);
 214         free(ppgtt, DRM_I915_GEM);
 215 }
 216
 217
 218 static unsigned int
 219 cache_level_to_agp_type(struct drm_device *dev, enum i915_cache_level
 220     cache_level)
 221 {
 222
 223         switch (cache_level) {
 224         case I915_CACHE_LLC_MLC:
 225                 if (INTEL_INFO(dev)->gen >= 6)
 226                         return (AGP_USER_CACHED_MEMORY_LLC_MLC);
 227                 /*
 228                  * Older chipsets do not have this extra level of CPU
 229                  * cacheing, so fallthrough and request the PTE simply
 230                  * as cached.
 231                  */
 232         case I915_CACHE_LLC:
 233                 return (AGP_USER_CACHED_MEMORY);
 234
 235         default:
 236         case I915_CACHE_NONE:
 237                 return (AGP_USER_MEMORY);
 238         }
 239 }
 240
 241 static bool
 242 do_idling(struct drm_i915_private *dev_priv)
 243 {
 244         bool ret = dev_priv->mm.interruptible;
 245
 246         if (dev_priv->mm.gtt.do_idle_maps) {
 247                 dev_priv->mm.interruptible = false;
 248                 if (i915_gpu_idle(dev_priv->dev, false)) {
 249                         DRM_ERROR("Couldn't idle GPU\n");
 250                         /* Wait a bit, in hopes it avoids the hang */
 251                         DELAY(10);
 252                 }
 253         }
 254
 255         return ret;
 256 }
 257
 258 static void
 259 undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 260 {
 261
 262         if (dev_priv->mm.gtt.do_idle_maps)
 263                 dev_priv->mm.interruptible = interruptible;
 264 }
 265
 266 void
 267 i915_gem_restore_gtt_mappings(struct drm_device *dev)
 268 {
 269         struct drm_i915_private *dev_priv;
 270         struct drm_i915_gem_object *obj;
 271
 272         dev_priv = dev->dev_private;
 273
 274         /* First fill our portion of the GTT with scratch pages */
 275         intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
 276             (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
 277
 278         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
 279                 i915_gem_clflush_object(obj);
 280                 i915_gem_gtt_rebind_object(obj, obj->cache_level);
 281         }
 282
 283         intel_gtt_chipset_flush();
 284 }
 285
 286 int
 287 i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj)
 288 {
 289         unsigned int agp_type;
 290
 291         agp_type = cache_level_to_agp_type(obj->base.dev, obj->cache_level);
 292         intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
 293             obj->base.size >> PAGE_SHIFT, obj->pages, agp_type);
 294         return (0);
 295 }
 296
 297 void
 298 i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj,
 299     enum i915_cache_level cache_level)
 300 {
 301         struct drm_device *dev;
 302         struct drm_i915_private *dev_priv;
 303         unsigned int agp_type;
 304
 305         dev = obj->base.dev;
 306         dev_priv = dev->dev_private;
 307         agp_type = cache_level_to_agp_type(dev, cache_level);
 308
 309         intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
 310             obj->base.size >> PAGE_SHIFT, obj->pages, agp_type);
 311 }
 312
 313 void
 314 i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 315 {
 316         struct drm_device *dev = obj->base.dev;
 317         struct drm_i915_private *dev_priv = dev->dev_private;
 318         bool interruptible;
 319
 320         dev = obj->base.dev;
 321         dev_priv = dev->dev_private;
 322
 323         interruptible = do_idling(dev_priv);
 324
 325         intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
 326             obj->base.size >> PAGE_SHIFT);
 327
 328         undo_idling(dev_priv, interruptible);
 329 }