--- zzzz-none-000/linux-3.10.107/drivers/gpu/drm/nouveau/nouveau_bo.c 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/drivers/gpu/drm/nouveau/nouveau_bo.c 2021-02-04 17:41:59.000000000 +0000 @@ -27,13 +27,9 @@ * Jeremy Kolb */ -#include +#include #include -#include -#include -#include - #include "nouveau_drm.h" #include "nouveau_dma.h" #include "nouveau_fence.h" @@ -52,24 +48,19 @@ { struct nouveau_drm *drm = nouveau_drm(dev); int i = reg - drm->tile.reg; - struct nouveau_fb *pfb = nouveau_fb(drm->device); - struct nouveau_fb_tile *tile = &pfb->tile.region[i]; - struct nouveau_engine *engine; + struct nvkm_device *device = nvxx_device(&drm->device); + struct nvkm_fb *fb = device->fb; + struct nvkm_fb_tile *tile = &fb->tile.region[i]; nouveau_fence_unref(®->fence); if (tile->pitch) - pfb->tile.fini(pfb, i, tile); + nvkm_fb_tile_fini(fb, i, tile); if (pitch) - pfb->tile.init(pfb, i, addr, size, pitch, flags, tile); - - pfb->tile.prog(pfb, i, tile); + nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile); - if ((engine = nouveau_engine(pfb, NVDEV_ENGINE_GR))) - engine->tile_prog(engine, i); - if ((engine = nouveau_engine(pfb, NVDEV_ENGINE_MPEG))) - engine->tile_prog(engine, i); + nvkm_fb_tile_prog(fb, i, tile); } static struct nouveau_drm_tile * @@ -92,18 +83,13 @@ static void nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile, - struct nouveau_fence *fence) + struct fence *fence) { struct nouveau_drm *drm = nouveau_drm(dev); if (tile) { spin_lock(&drm->tile.lock); - if (fence) { - /* Mark it as pending. */ - tile->fence = fence; - nouveau_fence_ref(fence); - } - + tile->fence = (struct nouveau_fence *)fence_get(fence); tile->used = false; spin_unlock(&drm->tile.lock); } @@ -114,18 +100,18 @@ u32 size, u32 pitch, u32 flags) { struct nouveau_drm *drm = nouveau_drm(dev); - struct nouveau_fb *pfb = nouveau_fb(drm->device); + struct nvkm_fb *fb = nvxx_fb(&drm->device); struct nouveau_drm_tile *tile, *found = NULL; int i; - for (i = 0; i < pfb->tile.regions; i++) { + for (i = 0; i < fb->tile.regions; i++) { tile = nv10_bo_get_tile_region(dev, i); if (pitch && !found) { found = tile; continue; - } else if (tile && pfb->tile.region[i].pitch) { + } else if (tile && fb->tile.region[i].pitch) { /* Kill an unused tile region. */ nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0); } @@ -146,8 +132,9 @@ struct drm_device *dev = drm->dev; struct nouveau_bo *nvbo = nouveau_bo(bo); - if (unlikely(nvbo->gem)) + if (unlikely(nvbo->gem.filp)) DRM_ERROR("bo %p still attached to GEM object\n", bo); + WARN_ON(nvbo->pin_refcnt > 0); nv10_bo_put_tile_region(dev, nvbo->tile, NULL); kfree(nvbo); } @@ -157,23 +144,23 @@ int *align, int *size) { struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); - struct nouveau_device *device = nv_device(drm->device); + struct nvif_device *device = &drm->device; - if (device->card_type < NV_50) { + if (device->info.family < NV_DEVICE_INFO_V0_TESLA) { if (nvbo->tile_mode) { - if (device->chipset >= 0x40) { + if (device->info.chipset >= 0x40) { *align = 65536; *size = roundup(*size, 64 * nvbo->tile_mode); - } else if (device->chipset >= 0x30) { + } else if (device->info.chipset >= 0x30) { *align = 32768; *size = roundup(*size, 64 * nvbo->tile_mode); - } else if (device->chipset >= 0x20) { + } else if (device->info.chipset >= 0x20) { *align = 16384; *size = roundup(*size, 64 * nvbo->tile_mode); - } else if (device->chipset >= 0x10) { + } else if (device->info.chipset >= 0x10) { *align = 16384; *size = roundup(*size, 32 * nvbo->tile_mode); } @@ -189,7 +176,7 @@ int nouveau_bo_new(struct drm_device *dev, int size, int align, uint32_t flags, uint32_t tile_mode, uint32_t tile_flags, - struct sg_table *sg, + struct sg_table *sg, struct reservation_object *robj, struct nouveau_bo **pnvbo) { struct nouveau_drm *drm = nouveau_drm(dev); @@ -197,6 +184,17 @@ size_t acc_size; int ret; int type = ttm_bo_type_device; + int lpg_shift = 12; + int max_size; + + if (drm->client.vm) + lpg_shift = drm->client.vm->mmu->lpg_shift; + max_size = INT_MAX & ~((1 << lpg_shift) - 1); + + if (size <= 0 || size > max_size) { + NV_WARN(drm, "skipped size %x\n", (u32)size); + return -EINVAL; + } if (sg) type = ttm_bo_type_sg; @@ -211,10 +209,13 @@ nvbo->tile_flags = tile_flags; nvbo->bo.bdev = &drm->ttm.bdev; + if (!nvxx_device(&drm->device)->func->cpu_coherent) + nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED; + nvbo->page_shift = 12; - if (drm->client.base.vm) { + if (drm->client.vm) { if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024) - nvbo->page_shift = drm->client.base.vm->vmm->lpg_shift; + nvbo->page_shift = drm->client.vm->mmu->lpg_shift; } nouveau_bo_fixup_align(nvbo, flags, &align, &size); @@ -227,7 +228,7 @@ ret = ttm_bo_init(&drm->ttm.bdev, &nvbo->bo, size, type, &nvbo->placement, align >> PAGE_SHIFT, false, NULL, acc_size, sg, - nouveau_bo_del_ttm); + robj, nouveau_bo_del_ttm); if (ret) { /* ttm will call nouveau_bo_del_ttm if it fails.. */ return ret; @@ -238,26 +239,26 @@ } static void -set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags) +set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t type, uint32_t flags) { *n = 0; if (type & TTM_PL_FLAG_VRAM) - pl[(*n)++] = TTM_PL_FLAG_VRAM | flags; + pl[(*n)++].flags = TTM_PL_FLAG_VRAM | flags; if (type & TTM_PL_FLAG_TT) - pl[(*n)++] = TTM_PL_FLAG_TT | flags; + pl[(*n)++].flags = TTM_PL_FLAG_TT | flags; if (type & TTM_PL_FLAG_SYSTEM) - pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags; + pl[(*n)++].flags = TTM_PL_FLAG_SYSTEM | flags; } static void set_placement_range(struct nouveau_bo *nvbo, uint32_t type) { struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); - struct nouveau_fb *pfb = nouveau_fb(drm->device); - u32 vram_pages = pfb->ram.size >> PAGE_SHIFT; + u32 vram_pages = drm->device.info.ram_size >> PAGE_SHIFT; + unsigned i, fpfn, lpfn; - if (nv_device(drm->device)->card_type == NV_10 && + if (drm->device.info.family == NV_DEVICE_INFO_V0_CELSIUS && nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) && nvbo->bo.mem.num_pages < vram_pages / 4) { /* @@ -267,11 +268,19 @@ * at the same time. */ if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) { - nvbo->placement.fpfn = vram_pages / 2; - nvbo->placement.lpfn = ~0; + fpfn = vram_pages / 2; + lpfn = ~0; } else { - nvbo->placement.fpfn = 0; - nvbo->placement.lpfn = vram_pages / 2; + fpfn = 0; + lpfn = vram_pages / 2; + } + for (i = 0; i < nvbo->placement.num_placement; ++i) { + nvbo->placements[i].fpfn = fpfn; + nvbo->placements[i].lpfn = lpfn; + } + for (i = 0; i < nvbo->placement.num_busy_placement; ++i) { + nvbo->busy_placements[i].fpfn = fpfn; + nvbo->busy_placements[i].lpfn = lpfn; } } } @@ -280,8 +289,9 @@ nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy) { struct ttm_placement *pl = &nvbo->placement; - uint32_t flags = TTM_PL_MASK_CACHING | - (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0); + uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED : + TTM_PL_MASK_CACHING) | + (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0); pl->placement = nvbo->placements; set_placement_list(nvbo->placements, &pl->num_placement, @@ -295,42 +305,75 @@ } int -nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype) +nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig) { struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); struct ttm_buffer_object *bo = &nvbo->bo; + bool force = false, evict = false; int ret; - ret = ttm_bo_reserve(bo, false, false, false, 0); + ret = ttm_bo_reserve(bo, false, false, false, NULL); if (ret) - goto out; + return ret; - if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) { - NV_ERROR(drm, "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo, - 1 << bo->mem.mem_type, memtype); - ret = -EINVAL; - goto out; + if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA && + memtype == TTM_PL_FLAG_VRAM && contig) { + if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) { + if (bo->mem.mem_type == TTM_PL_VRAM) { + struct nvkm_mem *mem = bo->mem.mm_node; + if (!list_is_singular(&mem->regions)) + evict = true; + } + nvbo->tile_flags &= ~NOUVEAU_GEM_TILE_NONCONTIG; + force = true; + } } - if (nvbo->pin_refcnt++) + if (nvbo->pin_refcnt) { + if (!(memtype & (1 << bo->mem.mem_type)) || evict) { + NV_ERROR(drm, "bo %p pinned elsewhere: " + "0x%08x vs 0x%08x\n", bo, + 1 << bo->mem.mem_type, memtype); + ret = -EBUSY; + } + nvbo->pin_refcnt++; goto out; + } + + if (evict) { + nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 0); + ret = nouveau_bo_validate(nvbo, false, false); + if (ret) + goto out; + } + nvbo->pin_refcnt++; nouveau_bo_placement_set(nvbo, memtype, 0); + /* drop pin_refcnt temporarily, so we don't trip the assertion + * in nouveau_bo_move() that makes sure we're not trying to + * move a pinned buffer + */ + nvbo->pin_refcnt--; ret = nouveau_bo_validate(nvbo, false, false); - if (ret == 0) { - switch (bo->mem.mem_type) { - case TTM_PL_VRAM: - drm->gem.vram_available -= bo->mem.size; - break; - case TTM_PL_TT: - drm->gem.gart_available -= bo->mem.size; - break; - default: - break; - } + if (ret) + goto out; + nvbo->pin_refcnt++; + + switch (bo->mem.mem_type) { + case TTM_PL_VRAM: + drm->gem.vram_available -= bo->mem.size; + break; + case TTM_PL_TT: + drm->gem.gart_available -= bo->mem.size; + break; + default: + break; } + out: + if (force && ret) + nvbo->tile_flags |= NOUVEAU_GEM_TILE_NONCONTIG; ttm_bo_unreserve(bo); return ret; } @@ -340,13 +383,15 @@ { struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); struct ttm_buffer_object *bo = &nvbo->bo; - int ret; + int ret, ref; - ret = ttm_bo_reserve(bo, false, false, false, 0); + ret = ttm_bo_reserve(bo, false, false, false, NULL); if (ret) return ret; - if (--nvbo->pin_refcnt) + ref = --nvbo->pin_refcnt; + WARN_ON_ONCE(ref < 0); + if (ref) goto out; nouveau_bo_placement_set(nvbo, bo->mem.placement, 0); @@ -375,11 +420,18 @@ { int ret; - ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); + ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL); if (ret) return ret; - ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap); + /* + * TTM buffers allocated using the DMA API already have a mapping, let's + * use it instead. + */ + if (!nvbo->force_coherent) + ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, + &nvbo->kmap); + ttm_bo_unreserve(&nvbo->bo); return ret; } @@ -387,10 +439,57 @@ void nouveau_bo_unmap(struct nouveau_bo *nvbo) { - if (nvbo) + if (!nvbo) + return; + + /* + * TTM buffers allocated using the DMA API already had a coherent + * mapping which we used, no need to unmap. + */ + if (!nvbo->force_coherent) ttm_bo_kunmap(&nvbo->kmap); } +void +nouveau_bo_sync_for_device(struct nouveau_bo *nvbo) +{ + struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); + struct nvkm_device *device = nvxx_device(&drm->device); + struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm; + int i; + + if (!ttm_dma) + return; + + /* Don't waste time looping if the object is coherent */ + if (nvbo->force_coherent) + return; + + for (i = 0; i < ttm_dma->ttm.num_pages; i++) + dma_sync_single_for_device(device->dev, ttm_dma->dma_address[i], + PAGE_SIZE, DMA_TO_DEVICE); +} + +void +nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo) +{ + struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); + struct nvkm_device *device = nvxx_device(&drm->device); + struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm; + int i; + + if (!ttm_dma) + return; + + /* Don't waste time looping if the object is coherent */ + if (nvbo->force_coherent) + return; + + for (i = 0; i < ttm_dma->ttm.num_pages; i++) + dma_sync_single_for_cpu(device->dev, ttm_dma->dma_address[i], + PAGE_SIZE, DMA_FROM_DEVICE); +} + int nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, bool no_wait_gpu) @@ -402,27 +501,41 @@ if (ret) return ret; + nouveau_bo_sync_for_device(nvbo); + return 0; } -u16 -nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index) +static inline void * +_nouveau_bo_mem_index(struct nouveau_bo *nvbo, unsigned index, void *mem, u8 sz) { - bool is_iomem; - u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); - mem = &mem[index]; - if (is_iomem) - return ioread16_native((void __force __iomem *)mem); - else - return *mem; + struct ttm_dma_tt *dma_tt; + u8 *m = mem; + + index *= sz; + + if (m) { + /* kmap'd address, return the corresponding offset */ + m += index; + } else { + /* DMA-API mapping, lookup the right address */ + dma_tt = (struct ttm_dma_tt *)nvbo->bo.ttm; + m = dma_tt->cpu_address[index / PAGE_SIZE]; + m += index % PAGE_SIZE; + } + + return m; } +#define nouveau_bo_mem_index(o, i, m) _nouveau_bo_mem_index(o, i, m, sizeof(*m)) void nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) { bool is_iomem; u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); - mem = &mem[index]; + + mem = nouveau_bo_mem_index(nvbo, index, mem); + if (is_iomem) iowrite16_native(val, (void __force __iomem *)mem); else @@ -434,7 +547,9 @@ { bool is_iomem; u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); - mem = &mem[index]; + + mem = nouveau_bo_mem_index(nvbo, index, mem); + if (is_iomem) return ioread32_native((void __force __iomem *)mem); else @@ -446,7 +561,9 @@ { bool is_iomem; u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); - mem = &mem[index]; + + mem = nouveau_bo_mem_index(nvbo, index, mem); + if (is_iomem) iowrite32_native(val, (void __force __iomem *)mem); else @@ -457,12 +574,11 @@ nouveau_ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size, uint32_t page_flags, struct page *dummy_read) { -#if __OS_HAS_AGP +#if IS_ENABLED(CONFIG_AGP) struct nouveau_drm *drm = nouveau_bdev(bdev); - struct drm_device *dev = drm->dev; - if (drm->agp.stat == ENABLED) { - return ttm_agp_tt_create(bdev, dev->agp->bridge, size, + if (drm->agp.bridge) { + return ttm_agp_tt_create(bdev, drm->agp.bridge, size, page_flags, dummy_read); } #endif @@ -490,29 +606,36 @@ man->default_caching = TTM_PL_FLAG_CACHED; break; case TTM_PL_VRAM: - if (nv_device(drm->device)->card_type >= NV_50) { + man->flags = TTM_MEMTYPE_FLAG_FIXED | + TTM_MEMTYPE_FLAG_MAPPABLE; + man->available_caching = TTM_PL_FLAG_UNCACHED | + TTM_PL_FLAG_WC; + man->default_caching = TTM_PL_FLAG_WC; + + if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { + /* Some BARs do not support being ioremapped WC */ + if (nvxx_bar(&drm->device)->iomap_uncached) { + man->available_caching = TTM_PL_FLAG_UNCACHED; + man->default_caching = TTM_PL_FLAG_UNCACHED; + } + man->func = &nouveau_vram_manager; man->io_reserve_fastpath = false; man->use_io_reserve_lru = true; } else { man->func = &ttm_bo_manager_func; } - man->flags = TTM_MEMTYPE_FLAG_FIXED | - TTM_MEMTYPE_FLAG_MAPPABLE; - man->available_caching = TTM_PL_FLAG_UNCACHED | - TTM_PL_FLAG_WC; - man->default_caching = TTM_PL_FLAG_WC; break; case TTM_PL_TT: - if (nv_device(drm->device)->card_type >= NV_50) + if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) man->func = &nouveau_gart_manager; else - if (drm->agp.stat != ENABLED) + if (!drm->agp.bridge) man->func = &nv04_gart_manager; else man->func = &ttm_bo_manager_func; - if (drm->agp.stat == ENABLED) { + if (drm->agp.bridge) { man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; @@ -550,35 +673,13 @@ } -/* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access - * TTM_PL_{VRAM,TT} directly. - */ - -static int -nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan, - struct nouveau_bo *nvbo, bool evict, - bool no_wait_gpu, struct ttm_mem_reg *new_mem) -{ - struct nouveau_fence *fence = NULL; - int ret; - - ret = nouveau_fence_new(chan, false, &fence); - if (ret) - return ret; - - ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, evict, - no_wait_gpu, new_mem); - nouveau_fence_unref(&fence); - return ret; -} - static int nve0_bo_move_init(struct nouveau_channel *chan, u32 handle) { int ret = RING_SPACE(chan, 2); if (ret == 0) { BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1); - OUT_RING (chan, handle); + OUT_RING (chan, handle & 0x0000ffff); FIRE_RING (chan); } return ret; @@ -588,7 +689,7 @@ nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) { - struct nouveau_mem *node = old_mem->mm_node; + struct nvkm_mem *node = old_mem->mm_node; int ret = RING_SPACE(chan, 10); if (ret == 0) { BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8); @@ -620,7 +721,7 @@ nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) { - struct nouveau_mem *node = old_mem->mm_node; + struct nvkm_mem *node = old_mem->mm_node; u64 src_offset = node->vma[0].offset; u64 dst_offset = node->vma[1].offset; u32 page_count = new_mem->num_pages; @@ -658,7 +759,7 @@ nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) { - struct nouveau_mem *node = old_mem->mm_node; + struct nvkm_mem *node = old_mem->mm_node; u64 src_offset = node->vma[0].offset; u64 dst_offset = node->vma[1].offset; u32 page_count = new_mem->num_pages; @@ -697,7 +798,7 @@ nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) { - struct nouveau_mem *node = old_mem->mm_node; + struct nvkm_mem *node = old_mem->mm_node; u64 src_offset = node->vma[0].offset; u64 dst_offset = node->vma[1].offset; u32 page_count = new_mem->num_pages; @@ -735,7 +836,7 @@ nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) { - struct nouveau_mem *node = old_mem->mm_node; + struct nvkm_mem *node = old_mem->mm_node; int ret = RING_SPACE(chan, 7); if (ret == 0) { BEGIN_NV04(chan, NvSubCopy, 0x0320, 6); @@ -753,7 +854,7 @@ nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) { - struct nouveau_mem *node = old_mem->mm_node; + struct nvkm_mem *node = old_mem->mm_node; int ret = RING_SPACE(chan, 7); if (ret == 0) { BEGIN_NV04(chan, NvSubCopy, 0x0304, 6); @@ -775,9 +876,9 @@ BEGIN_NV04(chan, NvSubCopy, 0x0000, 1); OUT_RING (chan, handle); BEGIN_NV04(chan, NvSubCopy, 0x0180, 3); - OUT_RING (chan, NvNotify0); - OUT_RING (chan, NvDmaFB); - OUT_RING (chan, NvDmaFB); + OUT_RING (chan, chan->drm->ntfy.handle); + OUT_RING (chan, chan->vram.handle); + OUT_RING (chan, chan->vram.handle); } return ret; @@ -787,12 +888,12 @@ nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) { - struct nouveau_mem *node = old_mem->mm_node; + struct nvkm_mem *node = old_mem->mm_node; u64 length = (new_mem->num_pages << PAGE_SHIFT); u64 src_offset = node->vma[0].offset; u64 dst_offset = node->vma[1].offset; int src_tiled = !!node->memtype; - int dst_tiled = !!((struct nouveau_mem *)new_mem->mm_node)->memtype; + int dst_tiled = !!((struct nvkm_mem *)new_mem->mm_node)->memtype; int ret; while (length) { @@ -864,7 +965,7 @@ BEGIN_NV04(chan, NvSubCopy, 0x0000, 1); OUT_RING (chan, handle); BEGIN_NV04(chan, NvSubCopy, 0x0180, 1); - OUT_RING (chan, NvNotify0); + OUT_RING (chan, chan->drm->ntfy.handle); } return ret; @@ -876,7 +977,7 @@ { if (mem->mem_type == TTM_PL_TT) return NvDmaTT; - return NvDmaFB; + return chan->vram.handle; } static int @@ -926,23 +1027,28 @@ } static int -nouveau_vma_getmap(struct nouveau_channel *chan, struct nouveau_bo *nvbo, - struct ttm_mem_reg *mem, struct nouveau_vma *vma) +nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, + struct ttm_mem_reg *mem) { - struct nouveau_mem *node = mem->mm_node; + struct nvkm_mem *old_node = bo->mem.mm_node; + struct nvkm_mem *new_node = mem->mm_node; + u64 size = (u64)mem->num_pages << PAGE_SHIFT; int ret; - ret = nouveau_vm_get(nv_client(chan->cli)->vm, mem->num_pages << - PAGE_SHIFT, node->page_shift, - NV_MEM_ACCESS_RW, vma); + ret = nvkm_vm_get(drm->client.vm, size, old_node->page_shift, + NV_MEM_ACCESS_RW, &old_node->vma[0]); if (ret) return ret; - if (mem->mem_type == TTM_PL_VRAM) - nouveau_vm_map(vma, node); - else - nouveau_vm_map_sg(vma, 0, mem->num_pages << PAGE_SHIFT, node); + ret = nvkm_vm_get(drm->client.vm, size, new_node->page_shift, + NV_MEM_ACCESS_RW, &old_node->vma[1]); + if (ret) { + nvkm_vm_put(&old_node->vma[0]); + return ret; + } + nvkm_vm_map(&old_node->vma[0], old_node); + nvkm_vm_map(&old_node->vma[1], new_node); return 0; } @@ -951,37 +1057,38 @@ bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct nouveau_drm *drm = nouveau_bdev(bo->bdev); - struct nouveau_channel *chan = chan = drm->channel; - struct nouveau_bo *nvbo = nouveau_bo(bo); - struct ttm_mem_reg *old_mem = &bo->mem; + struct nouveau_channel *chan = drm->ttm.chan; + struct nouveau_cli *cli = (void *)chan->user.client; + struct nouveau_fence *fence; int ret; - mutex_lock(&chan->cli->mutex); - /* create temporary vmas for the transfer and attach them to the - * old nouveau_mem node, these will get cleaned up after ttm has + * old nvkm_mem node, these will get cleaned up after ttm has * destroyed the ttm_mem_reg */ - if (nv_device(drm->device)->card_type >= NV_50) { - struct nouveau_mem *node = old_mem->mm_node; - - ret = nouveau_vma_getmap(chan, nvbo, old_mem, &node->vma[0]); + if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { + ret = nouveau_bo_move_prep(drm, bo, new_mem); if (ret) - goto out; - - ret = nouveau_vma_getmap(chan, nvbo, new_mem, &node->vma[1]); - if (ret) - goto out; + return ret; } - ret = drm->ttm.move(chan, bo, &bo->mem, new_mem); + mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING); + ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, intr); if (ret == 0) { - ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict, - no_wait_gpu, new_mem); + ret = drm->ttm.move(chan, bo, &bo->mem, new_mem); + if (ret == 0) { + ret = nouveau_fence_new(chan, false, &fence); + if (ret == 0) { + ret = ttm_bo_move_accel_cleanup(bo, + &fence->base, + evict, + no_wait_gpu, + new_mem); + nouveau_fence_unref(&fence); + } + } } - -out: - mutex_unlock(&chan->cli->mutex); + mutex_unlock(&cli->mutex); return ret; } @@ -991,13 +1098,15 @@ static const struct { const char *name; int engine; - u32 oclass; + s32 oclass; int (*exec)(struct nouveau_channel *, struct ttm_buffer_object *, struct ttm_mem_reg *, struct ttm_mem_reg *); int (*init)(struct nouveau_channel *, u32 handle); } _methods[] = { - { "COPY", 0, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init }, + { "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init }, + { "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init }, + { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init }, { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init }, { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init }, { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init }, @@ -1013,28 +1122,28 @@ int ret; do { - struct nouveau_object *object; struct nouveau_channel *chan; - u32 handle = (mthd->engine << 16) | mthd->oclass; - if (mthd->init == nve0_bo_move_init) + if (mthd->engine) chan = drm->cechan; else chan = drm->channel; if (chan == NULL) continue; - ret = nouveau_object_new(nv_object(drm), chan->handle, handle, - mthd->oclass, NULL, 0, &object); + ret = nvif_object_init(&chan->user, + mthd->oclass | (mthd->engine << 16), + mthd->oclass, NULL, 0, + &drm->ttm.copy); if (ret == 0) { - ret = mthd->init(chan, handle); + ret = mthd->init(chan, drm->ttm.copy.handle); if (ret) { - nouveau_object_del(nv_object(drm), - chan->handle, handle); + nvif_object_fini(&drm->ttm.copy); continue; } drm->ttm.move = mthd->exec; + drm->ttm.chan = chan; name = mthd->name; break; } @@ -1047,12 +1156,15 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { - u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; + struct ttm_place placement_memtype = { + .fpfn = 0, + .lpfn = 0, + .flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING + }; struct ttm_placement placement; struct ttm_mem_reg tmp_mem; int ret; - placement.fpfn = placement.lpfn = 0; placement.num_placement = placement.num_busy_placement = 1; placement.placement = placement.busy_placement = &placement_memtype; @@ -1080,12 +1192,15 @@ nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { - u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; + struct ttm_place placement_memtype = { + .fpfn = 0, + .lpfn = 0, + .flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING + }; struct ttm_placement placement; struct ttm_mem_reg tmp_mem; int ret; - placement.fpfn = placement.lpfn = 0; placement.num_placement = placement.num_busy_placement = 1; placement.placement = placement.busy_placement = &placement_memtype; @@ -1112,28 +1227,19 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem) { struct nouveau_bo *nvbo = nouveau_bo(bo); - struct nouveau_vma *vma; + struct nvkm_vma *vma; /* ttm can now (stupidly) pass the driver bos it didn't create... */ if (bo->destroy != nouveau_bo_del_ttm) return; list_for_each_entry(vma, &nvbo->vma_list, head) { - if (new_mem && new_mem->mem_type == TTM_PL_VRAM) { - nouveau_vm_map(vma, new_mem->mm_node); - } else - if (new_mem && new_mem->mem_type == TTM_PL_TT && - nvbo->page_shift == vma->vm->vmm->spg_shift) { - if (((struct nouveau_mem *)new_mem->mm_node)->sg) - nouveau_vm_map_sg_table(vma, 0, new_mem-> - num_pages << PAGE_SHIFT, - new_mem->mm_node); - else - nouveau_vm_map_sg(vma, 0, new_mem-> - num_pages << PAGE_SHIFT, - new_mem->mm_node); + if (new_mem && new_mem->mem_type != TTM_PL_SYSTEM && + (new_mem->mem_type == TTM_PL_VRAM || + nvbo->page_shift != vma->vm->mmu->lpg_shift)) { + nvkm_vm_map(vma, new_mem->mm_node); } else { - nouveau_vm_unmap(vma); + nvkm_vm_unmap(vma); } } } @@ -1151,7 +1257,7 @@ if (new_mem->mem_type != TTM_PL_VRAM) return 0; - if (nv_device(drm->device)->card_type >= NV_10) { + if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) { *new_tile = nv10_bo_set_tiling(dev, offset, new_mem->size, nvbo->tile_mode, nvbo->tile_flags); @@ -1167,8 +1273,9 @@ { struct nouveau_drm *drm = nouveau_bdev(bo->bdev); struct drm_device *dev = drm->dev; + struct fence *fence = reservation_object_get_excl(bo->resv); - nv10_bo_put_tile_region(dev, *old_tile, bo->sync_obj); + nv10_bo_put_tile_region(dev, *old_tile, fence); *old_tile = new_tile; } @@ -1182,7 +1289,10 @@ struct nouveau_drm_tile *new_tile = NULL; int ret = 0; - if (nv_device(drm->device)->card_type < NV_50) { + if (nvbo->pin_refcnt) + NV_WARN(drm, "Moving pinned object %p!\n", nvbo); + + if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) { ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile); if (ret) return ret; @@ -1196,31 +1306,28 @@ goto out; } - /* CPU copy if we have no accelerated method available */ - if (!drm->ttm.move) { - ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); - goto out; - } - /* Hardware assisted copy. */ - if (new_mem->mem_type == TTM_PL_SYSTEM) - ret = nouveau_bo_move_flipd(bo, evict, intr, - no_wait_gpu, new_mem); - else if (old_mem->mem_type == TTM_PL_SYSTEM) - ret = nouveau_bo_move_flips(bo, evict, intr, - no_wait_gpu, new_mem); - else - ret = nouveau_bo_move_m2mf(bo, evict, intr, - no_wait_gpu, new_mem); - - if (!ret) - goto out; + if (drm->ttm.move) { + if (new_mem->mem_type == TTM_PL_SYSTEM) + ret = nouveau_bo_move_flipd(bo, evict, intr, + no_wait_gpu, new_mem); + else if (old_mem->mem_type == TTM_PL_SYSTEM) + ret = nouveau_bo_move_flips(bo, evict, intr, + no_wait_gpu, new_mem); + else + ret = nouveau_bo_move_m2mf(bo, evict, intr, + no_wait_gpu, new_mem); + if (!ret) + goto out; + } /* Fallback to software copy. */ - ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); + ret = ttm_bo_wait(bo, true, intr, no_wait_gpu); + if (ret == 0) + ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); out: - if (nv_device(drm->device)->card_type < NV_50) { + if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) { if (ret) nouveau_bo_vm_cleanup(bo, NULL, &new_tile); else @@ -1233,7 +1340,9 @@ static int nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp) { - return 0; + struct nouveau_bo *nvbo = nouveau_bo(bo); + + return drm_vma_node_verify_access(&nvbo->gem.vma_node, filp); } static int @@ -1241,7 +1350,8 @@ { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; struct nouveau_drm *drm = nouveau_bdev(bdev); - struct drm_device *dev = drm->dev; + struct nvkm_device *device = nvxx_device(&drm->device); + struct nvkm_mem *node = mem->mm_node; int ret; mem->bus.addr = NULL; @@ -1256,27 +1366,33 @@ /* System memory */ return 0; case TTM_PL_TT: -#if __OS_HAS_AGP - if (drm->agp.stat == ENABLED) { +#if IS_ENABLED(CONFIG_AGP) + if (drm->agp.bridge) { mem->bus.offset = mem->start << PAGE_SHIFT; mem->bus.base = drm->agp.base; - mem->bus.is_iomem = !dev->agp->cant_use_aperture; + mem->bus.is_iomem = !drm->agp.cma; } #endif - break; + if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA || !node->memtype) + /* untiled */ + break; + /* fallthrough, tiled memory */ case TTM_PL_VRAM: mem->bus.offset = mem->start << PAGE_SHIFT; - mem->bus.base = pci_resource_start(dev->pdev, 1); + mem->bus.base = device->func->resource_addr(device, 1); mem->bus.is_iomem = true; - if (nv_device(drm->device)->card_type >= NV_50) { - struct nouveau_bar *bar = nouveau_bar(drm->device); - struct nouveau_mem *node = mem->mm_node; + if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { + struct nvkm_bar *bar = nvxx_bar(&drm->device); + int page_shift = 12; + if (drm->device.info.family >= NV_DEVICE_INFO_V0_FERMI) + page_shift = node->page_shift; - ret = bar->umap(bar, node, NV_MEM_ACCESS_RW, - &node->bar_vma); + ret = nvkm_bar_umap(bar, node->size << 12, page_shift, + &node->bar_vma); if (ret) return ret; + nvkm_vm_map(&node->bar_vma, node); mem->bus.offset = node->bar_vma.offset; } break; @@ -1289,14 +1405,13 @@ static void nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) { - struct nouveau_drm *drm = nouveau_bdev(bdev); - struct nouveau_bar *bar = nouveau_bar(drm->device); - struct nouveau_mem *node = mem->mm_node; + struct nvkm_mem *node = mem->mm_node; if (!node->bar_vma.node) return; - bar->unmap(bar, &node->bar_vma); + nvkm_vm_unmap(&node->bar_vma); + nvkm_vm_put(&node->bar_vma); } static int @@ -1304,25 +1419,43 @@ { struct nouveau_drm *drm = nouveau_bdev(bo->bdev); struct nouveau_bo *nvbo = nouveau_bo(bo); - struct nouveau_device *device = nv_device(drm->device); - u32 mappable = pci_resource_len(device->pdev, 1) >> PAGE_SHIFT; + struct nvkm_device *device = nvxx_device(&drm->device); + u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT; + int i, ret; /* as long as the bo isn't in vram, and isn't tiled, we've got * nothing to do here. */ if (bo->mem.mem_type != TTM_PL_VRAM) { - if (nv_device(drm->device)->card_type < NV_50 || + if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA || !nouveau_bo_tile_layout(nvbo)) return 0; + + if (bo->mem.mem_type == TTM_PL_SYSTEM) { + nouveau_bo_placement_set(nvbo, TTM_PL_TT, 0); + + ret = nouveau_bo_validate(nvbo, false, false); + if (ret) + return ret; + } + return 0; } /* make sure bo is in mappable vram */ - if (bo->mem.start + bo->mem.num_pages < mappable) + if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA || + bo->mem.start + bo->mem.num_pages < mappable) return 0; + for (i = 0; i < nvbo->placement.num_placement; ++i) { + nvbo->placements[i].fpfn = 0; + nvbo->placements[i].lpfn = mappable; + } + + for (i = 0; i < nvbo->placement.num_busy_placement; ++i) { + nvbo->busy_placements[i].fpfn = 0; + nvbo->busy_placements[i].lpfn = mappable; + } - nvbo->placement.fpfn = 0; - nvbo->placement.lpfn = mappable; nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0); return nouveau_bo_validate(nvbo, false, false); } @@ -1332,7 +1465,9 @@ { struct ttm_dma_tt *ttm_dma = (void *)ttm; struct nouveau_drm *drm; + struct nvkm_device *device; struct drm_device *dev; + struct device *pdev; unsigned i; int r; bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); @@ -1349,10 +1484,20 @@ } drm = nouveau_bdev(ttm->bdev); + device = nvxx_device(&drm->device); dev = drm->dev; + pdev = device->dev; -#if __OS_HAS_AGP - if (drm->agp.stat == ENABLED) { + /* + * Objects matching this condition have been marked as force_coherent, + * so use the DMA API for them. + */ + if (!nvxx_device(&drm->device)->func->cpu_coherent && + ttm->caching_state == tt_uncached) + return ttm_dma_populate(ttm_dma, dev->dev); + +#if IS_ENABLED(CONFIG_AGP) + if (drm->agp.bridge) { return ttm_agp_tt_populate(ttm); } #endif @@ -1369,18 +1514,22 @@ } for (i = 0; i < ttm->num_pages; i++) { - ttm_dma->dma_address[i] = pci_map_page(dev->pdev, ttm->pages[i], - 0, PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); - if (pci_dma_mapping_error(dev->pdev, ttm_dma->dma_address[i])) { + dma_addr_t addr; + + addr = dma_map_page(pdev, ttm->pages[i], 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + + if (dma_mapping_error(pdev, addr)) { while (--i) { - pci_unmap_page(dev->pdev, ttm_dma->dma_address[i], - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + dma_unmap_page(pdev, ttm_dma->dma_address[i], + PAGE_SIZE, DMA_BIDIRECTIONAL); ttm_dma->dma_address[i] = 0; } ttm_pool_unpopulate(ttm); return -EFAULT; } + + ttm_dma->dma_address[i] = addr; } return 0; } @@ -1390,7 +1539,9 @@ { struct ttm_dma_tt *ttm_dma = (void *)ttm; struct nouveau_drm *drm; + struct nvkm_device *device; struct drm_device *dev; + struct device *pdev; unsigned i; bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); @@ -1398,10 +1549,22 @@ return; drm = nouveau_bdev(ttm->bdev); + device = nvxx_device(&drm->device); dev = drm->dev; + pdev = device->dev; + + /* + * Objects matching this condition have been marked as force_coherent, + * so use the DMA API for them. + */ + if (!nvxx_device(&drm->device)->func->cpu_coherent && + ttm->caching_state == tt_uncached) { + ttm_dma_unpopulate(ttm_dma, dev->dev); + return; + } -#if __OS_HAS_AGP - if (drm->agp.stat == ENABLED) { +#if IS_ENABLED(CONFIG_AGP) + if (drm->agp.bridge) { ttm_agp_tt_unpopulate(ttm); return; } @@ -1416,8 +1579,8 @@ for (i = 0; i < ttm->num_pages; i++) { if (ttm_dma->dma_address[i]) { - pci_unmap_page(dev->pdev, ttm_dma->dma_address[i], - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + dma_unmap_page(pdev, ttm_dma->dma_address[i], PAGE_SIZE, + DMA_BIDIRECTIONAL); } } @@ -1425,49 +1588,14 @@ } void -nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence) +nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive) { - struct nouveau_fence *old_fence = NULL; + struct reservation_object *resv = nvbo->bo.resv; - if (likely(fence)) - nouveau_fence_ref(fence); - - spin_lock(&nvbo->bo.bdev->fence_lock); - old_fence = nvbo->bo.sync_obj; - nvbo->bo.sync_obj = fence; - spin_unlock(&nvbo->bo.bdev->fence_lock); - - nouveau_fence_unref(&old_fence); -} - -static void -nouveau_bo_fence_unref(void **sync_obj) -{ - nouveau_fence_unref((struct nouveau_fence **)sync_obj); -} - -static void * -nouveau_bo_fence_ref(void *sync_obj) -{ - return nouveau_fence_ref(sync_obj); -} - -static bool -nouveau_bo_fence_signalled(void *sync_obj) -{ - return nouveau_fence_done(sync_obj); -} - -static int -nouveau_bo_fence_wait(void *sync_obj, bool lazy, bool intr) -{ - return nouveau_fence_wait(sync_obj, lazy, intr); -} - -static int -nouveau_bo_fence_flush(void *sync_obj) -{ - return 0; + if (exclusive) + reservation_object_add_excl_fence(resv, &fence->base); + else if (fence) + reservation_object_add_shared_fence(resv, &fence->base); } struct ttm_bo_driver nouveau_bo_driver = { @@ -1480,20 +1608,15 @@ .move_notify = nouveau_bo_move_ntfy, .move = nouveau_bo_move, .verify_access = nouveau_bo_verify_access, - .sync_obj_signaled = nouveau_bo_fence_signalled, - .sync_obj_wait = nouveau_bo_fence_wait, - .sync_obj_flush = nouveau_bo_fence_flush, - .sync_obj_unref = nouveau_bo_fence_unref, - .sync_obj_ref = nouveau_bo_fence_ref, .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify, .io_mem_reserve = &nouveau_ttm_io_mem_reserve, .io_mem_free = &nouveau_ttm_io_mem_free, }; -struct nouveau_vma * -nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nouveau_vm *vm) +struct nvkm_vma * +nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nvkm_vm *vm) { - struct nouveau_vma *vma; + struct nvkm_vma *vma; list_for_each_entry(vma, &nvbo->vma_list, head) { if (vma->vm == vm) return vma; @@ -1503,26 +1626,21 @@ } int -nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nouveau_vm *vm, - struct nouveau_vma *vma) +nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nvkm_vm *vm, + struct nvkm_vma *vma) { const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT; - struct nouveau_mem *node = nvbo->bo.mem.mm_node; int ret; - ret = nouveau_vm_get(vm, size, nvbo->page_shift, + ret = nvkm_vm_get(vm, size, nvbo->page_shift, NV_MEM_ACCESS_RW, vma); if (ret) return ret; - if (nvbo->bo.mem.mem_type == TTM_PL_VRAM) - nouveau_vm_map(vma, nvbo->bo.mem.mm_node); - else if (nvbo->bo.mem.mem_type == TTM_PL_TT) { - if (node->sg) - nouveau_vm_map_sg_table(vma, 0, size, node); - else - nouveau_vm_map_sg(vma, 0, size, node); - } + if ( nvbo->bo.mem.mem_type != TTM_PL_SYSTEM && + (nvbo->bo.mem.mem_type == TTM_PL_VRAM || + nvbo->page_shift != vma->vm->mmu->lpg_shift)) + nvkm_vm_map(vma, nvbo->bo.mem.mm_node); list_add_tail(&vma->head, &nvbo->vma_list); vma->refcount = 1; @@ -1530,17 +1648,12 @@ } void -nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nouveau_vma *vma) +nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nvkm_vma *vma) { if (vma->node) { - if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM) { - spin_lock(&nvbo->bo.bdev->fence_lock); - ttm_bo_wait(&nvbo->bo, false, false, false); - spin_unlock(&nvbo->bo.bdev->fence_lock); - nouveau_vm_unmap(vma); - } - - nouveau_vm_put(vma); + if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM) + nvkm_vm_unmap(vma); + nvkm_vm_put(vma); list_del(&vma->head); } }