xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
انشعاب 0
کد مسائل تقاضاهای واکشی Actions Packages پروژه‌ها انتشارها دانشنامه فعالیت

Merge tag 'drm-for-v4.15' of git://people.freedesktop.org/~airlied/linux

فهرست منبع 
Pull drm updates from Dave Airlie:
 "This is the main drm pull request for v4.15.

  Core:
   - Atomic object lifetime fixes
   - Atomic iterator improvements
   - Sparse/smatch fixes
   - Legacy kms ioctls to be interruptible
   - EDID override improvements
   - fb/gem helper cleanups
   - Simple outreachy patches
   - Documentation improvements
   - Fix dma-buf rcu races
   - DRM mode object leasing for improving VR use cases.
   - vgaarb improvements for non-x86 platforms.

  New driver:
   - tve200: Faraday Technology TVE200 block.

     This "TV Encoder" encodes a ITU-T BT.656 stream and can be found in
     the StorLink SL3516 (later Cortina Systems CS3516) as well as the
     Grain Media GM8180.

  New bridges:
   - SiI9234 support

  New panels:
   - S6E63J0X03, OTM8009A, Seiko 43WVF1G, 7" rpi touch panel, Toshiba
     LT089AC19000, Innolux AT043TN24

  i915:
   - Remove Coffeelake from alpha support
   - Cannonlake workarounds
   - Infoframe refactoring for DisplayPort
   - VBT updates
   - DisplayPort vswing/emph/buffer translation refactoring
   - CCS fixes
   - Restore GPU clock boost on missed vblanks
   - Scatter list updates for userptr allocations
   - Gen9+ transition watermarks
   - Display IPC (Isochronous Priority Control)
   - Private PAT management
   - GVT: improved error handling and pci config sanitizing
   - Execlist refactoring
   - Transparent Huge Page support
   - User defined priorities support
   - HuC/GuC firmware refactoring
   - DP MST fixes
   - eDP power sequencing fixes
   - Use RCU instead of stop_machine
   - PSR state tracking support
   - Eviction fixes
   - BDW DP aux channel timeout fixes
   - LSPCON fixes
   - Cannonlake PLL fixes

  amdgpu:
   - Per VM BO support
   - Powerplay cleanups
   - CI powerplay support
   - PASID mgr for kfd
   - SR-IOV fixes
   - initial GPU reset for vega10
   - Prime mmap support
   - TTM updates
   - Clock query interface for Raven
   - Fence to handle ioctl
   - UVD encode ring support on Polaris
   - Transparent huge page DMA support
   - Compute LRU pipe tweaks
   - BO flag to allow buffers to opt out of implicit sync
   - CTX priority setting API
   - VRAM lost infrastructure plumbing

  qxl:
   - fix flicker since atomic rework

  amdkfd:
   - Further improvements from internal AMD tree
   - Usermode events
   - Drop radeon support

  nouveau:
   - Pascal temperature sensor support
   - Improved BAR2 handling
   - MMU rework to support Pascal MMU

  exynos:
   - Improved HDMI/mixer support
   - HDMI audio interface support

  tegra:
   - Prep work for tegra186
   - Cleanup/fixes

  msm:
   - Preemption support for a5xx
   - Display fixes for 8x96 (snapdragon 820)
   - Async cursor plane fixes
   - FW loading rework
   - GPU debugging improvements

  vc4:
   - Prep for DSI panels
   - fix T-format tiling scanout
   - New madvise ioctl

  Rockchip:
   - LVDS support

  omapdrm:
   - omap4 HDMI CEC support

  etnaviv:
   - GPU performance counters groundwork

  sun4i:
   - refactor driver load + TCON backend
   - HDMI improvements
   - A31 support
   - Misc fixes

  udl:
   - Probe/EDID read fixes.

  tilcdc:
   - Misc fixes.

  pl111:
   - Support more variants

  adv7511:
   - Improve EDID handling.
   - HDMI CEC support

  sii8620:
   - Add remote control support"

* tag 'drm-for-v4.15' of git://people.freedesktop.org/~airlied/linux: (1480 commits)
  drm/rockchip: analogix_dp: Use mutex rather than spinlock
  drm/mode_object: fix documentation for object lookups.
  drm/i915: Reorder context-close to avoid calling i915_vma_close() under RCU
  drm/i915: Move init_clock_gating() back to where it was
  drm/i915: Prune the reservation shared fence array
  drm/i915: Idle the GPU before shinking everything
  drm/i915: Lock llist_del_first() vs llist_del_all()
  drm/i915: Calculate ironlake intermediate watermarks correctly, v2.
  drm/i915: Disable lazy PPGTT page table optimization for vGPU
  drm/i915/execlists: Remove the priority "optimisation"
  drm/i915: Filter out spurious execlists context-switch interrupts
  drm/amdgpu: use irq-safe lock for kiq->ring_lock
  drm/amdgpu: bypass lru touch for KIQ ring submission
  drm/amdgpu: Potential uninitialized variable in amdgpu_vm_update_directories()
  drm/amdgpu: potential uninitialized variable in amdgpu_vce_ring_parse_cs()
  drm/amd/powerplay: initialize a variable before using it
  drm/amd/powerplay: suppress KASAN out of bounds warning in vega10_populate_all_memory_levels
  drm/amd/amdgpu: fix evicted VRAM bo adjudgement condition
  drm/vblank: Tune drm_crtc_accurate_vblank_count() WARN down to a debug
  drm/rockchip: add CONFIG_OF dependency for lvds
  ...
This commit is contained in:
Linus Torvalds
2017-11-15 20:42:10 -08:00
والد 5d352e69c6 f150891fd9
کامیت e60e1ee606
1040فایلهای تغییر یافته به همراه65899 افزوده شده و 56430 حذف شده
دانلود پرونده وصله دانلود فایل تغییرات diff Expand all files Collapse all files

43
drivers/gpu/drm/ttm/ttm_bo.c
مشاهده پرونده

@@ -150,8 +150,7 @@ static void ttm_bo_release_list(struct kref *list_kref)
ttm_tt_destroy(bo->ttm);
atomic_dec(&bo->glob->bo_count);
dma_fence_put(bo->moving);
if (bo->resv == &bo->ttm_resv)
reservation_object_fini(&bo->ttm_resv);
reservation_object_fini(&bo->ttm_resv);
mutex_destroy(&bo->wu_mutex);
if (bo->destroy)
bo->destroy(bo);
@@ -402,14 +401,11 @@ static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
if (bo->resv == &bo->ttm_resv)
return 0;
reservation_object_init(&bo->ttm_resv);
BUG_ON(!reservation_object_trylock(&bo->ttm_resv));
r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv);
if (r) {
if (r)
reservation_object_unlock(&bo->ttm_resv);
reservation_object_fini(&bo->ttm_resv);
}
return r;
}
@@ -440,28 +436,30 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
struct ttm_bo_global *glob = bo->glob;
int ret;
ret = ttm_bo_individualize_resv(bo);
if (ret) {
/* Last resort, if we fail to allocate memory for the
* fences block for the BO to become idle
*/
reservation_object_wait_timeout_rcu(bo->resv, true, false,
30 * HZ);
spin_lock(&glob->lru_lock);
goto error;
}
spin_lock(&glob->lru_lock);
ret = __ttm_bo_reserve(bo, false, true, NULL);
if (!ret) {
if (!ttm_bo_wait(bo, false, true)) {
if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) {
ttm_bo_del_from_lru(bo);
spin_unlock(&glob->lru_lock);
ttm_bo_cleanup_memtype_use(bo);
if (bo->resv != &bo->ttm_resv)
reservation_object_unlock(&bo->ttm_resv);
return;
}
ret = ttm_bo_individualize_resv(bo);
if (ret) {
/* Last resort, if we fail to allocate memory for the
* fences block for the BO to become idle and free it.
*/
spin_unlock(&glob->lru_lock);
ttm_bo_wait(bo, true, true);
ttm_bo_cleanup_memtype_use(bo);
return;
}
ttm_bo_flush_all_fences(bo);
/*
@@ -474,11 +472,12 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
ttm_bo_add_to_lru(bo);
}
if (bo->resv != &bo->ttm_resv)
reservation_object_unlock(&bo->ttm_resv);
__ttm_bo_unreserve(bo);
}
if (bo->resv != &bo->ttm_resv)
reservation_object_unlock(&bo->ttm_resv);
error:
kref_get(&bo->list_kref);
list_add_tail(&bo->ddestroy, &bdev->ddestroy);
spin_unlock(&glob->lru_lock);
@@ -1203,8 +1202,8 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
lockdep_assert_held(&bo->resv->lock.base);
} else {
bo->resv = &bo->ttm_resv;
reservation_object_init(&bo->ttm_resv);
}
reservation_object_init(&bo->ttm_resv);
atomic_inc(&bo->glob->bo_count);
drm_vma_node_reset(&bo->vma_node);
bo->priority = 0;

2
drivers/gpu/drm/ttm/ttm_bo_util.c
مشاهده پرونده

@@ -474,6 +474,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
INIT_LIST_HEAD(&fbo->lru);
INIT_LIST_HEAD(&fbo->swap);
INIT_LIST_HEAD(&fbo->io_reserve_lru);
mutex_init(&fbo->wu_mutex);
fbo->moving = NULL;
drm_vma_node_reset(&fbo->vma_node);
atomic_set(&fbo->cpu_writers, 0);
@@ -587,7 +588,6 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo,
unsigned long offset, size;
int ret;
BUG_ON(!list_empty(&bo->swap));
map->virtual = NULL;
map->bo = bo;
if (num_pages > bo->num_pages)

12
drivers/gpu/drm/ttm/ttm_memory.c
مشاهده پرونده

@@ -546,8 +546,7 @@ int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
EXPORT_SYMBOL(ttm_mem_global_alloc);
int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
struct page *page,
bool no_wait, bool interruptible)
struct page *page, uint64_t size)
{
struct ttm_mem_zone *zone = NULL;
@@ -564,11 +563,11 @@ int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
zone = glob->zone_kernel;
#endif
return ttm_mem_global_alloc_zone(glob, zone, PAGE_SIZE, no_wait,
interruptible);
return ttm_mem_global_alloc_zone(glob, zone, size, false, false);
}
void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page)
void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page,
uint64_t size)
{
struct ttm_mem_zone *zone = NULL;
@@ -579,10 +578,9 @@ void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page)
if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
zone = glob->zone_kernel;
#endif
ttm_mem_global_free_zone(glob, zone, PAGE_SIZE);
ttm_mem_global_free_zone(glob, zone, size);
}
size_t ttm_round_pot(size_t size)
{
if ((size & (size - 1)) == 0)

407
drivers/gpu/drm/ttm/ttm_page_alloc.c
مشاهده پرونده

@@ -95,7 +95,7 @@ struct ttm_pool_opts {
unsigned small;
};
#define NUM_POOLS 4
#define NUM_POOLS 6
/**
* struct ttm_pool_manager - Holds memory pools for fst allocation
@@ -122,6 +122,8 @@ struct ttm_pool_manager {
struct ttm_page_pool uc_pool;
struct ttm_page_pool wc_pool_dma32;
struct ttm_page_pool uc_pool_dma32;
struct ttm_page_pool wc_pool_huge;
struct ttm_page_pool uc_pool_huge;
} ;
};
};
@@ -256,8 +258,8 @@ static int set_pages_array_uc(struct page **pages, int addrinarray)
/**
* Select the right pool or requested caching state and ttm flags. */
static struct ttm_page_pool *ttm_get_pool(int flags,
enum ttm_caching_state cstate)
static struct ttm_page_pool *ttm_get_pool(int flags, bool huge,
enum ttm_caching_state cstate)
{
int pool_index;
@@ -269,9 +271,15 @@ static struct ttm_page_pool *ttm_get_pool(int flags,
else
pool_index = 0x1;
if (flags & TTM_PAGE_FLAG_DMA32)
if (flags & TTM_PAGE_FLAG_DMA32) {
if (huge)
return NULL;
pool_index |= 0x2;
} else if (huge) {
pool_index |= 0x4;
}
return &_manager->pools[pool_index];
}
@@ -321,7 +329,7 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free,
pages_to_free = kmalloc(npages_to_free * sizeof(struct page *),
GFP_KERNEL);
if (!pages_to_free) {
pr_err("Failed to allocate memory for pool free operation\n");
pr_debug("Failed to allocate memory for pool free operation\n");
return 0;
}
@@ -494,12 +502,14 @@ static void ttm_handle_caching_state_failure(struct list_head *pages,
* pages returned in pages array.
*/
static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
int ttm_flags, enum ttm_caching_state cstate, unsigned count)
int ttm_flags, enum ttm_caching_state cstate,
unsigned count, unsigned order)
{
struct page **caching_array;
struct page *p;
int r = 0;
unsigned i, cpages;
unsigned i, j, cpages;
unsigned npages = 1 << order;
unsigned max_cpages = min(count,
(unsigned)(PAGE_SIZE/sizeof(struct page *)));
@@ -507,15 +517,15 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL);
if (!caching_array) {
pr_err("Unable to allocate table for new pages\n");
pr_debug("Unable to allocate table for new pages\n");
return -ENOMEM;
}
for (i = 0, cpages = 0; i < count; ++i) {
p = alloc_page(gfp_flags);
p = alloc_pages(gfp_flags, order);
if (!p) {
pr_err("Unable to get page %u\n", i);
pr_debug("Unable to get page %u\n", i);
/* store already allocated pages in the pool after
* setting the caching state */
@@ -531,14 +541,18 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
goto out;
}
list_add(&p->lru, pages);
#ifdef CONFIG_HIGHMEM
/* gfp flags of highmem page should never be dma32 so we
* we should be fine in such case
*/
if (!PageHighMem(p))
if (PageHighMem(p))
continue;
#endif
{
caching_array[cpages++] = p;
for (j = 0; j < npages; ++j) {
caching_array[cpages++] = p++;
if (cpages == max_cpages) {
r = ttm_set_pages_caching(caching_array,
@@ -552,8 +566,6 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
cpages = 0;
}
}
list_add(&p->lru, pages);
}
if (cpages) {
@@ -573,9 +585,9 @@ out:
* Fill the given pool if there aren't enough pages and the requested number of
* pages is small.
*/
static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
int ttm_flags, enum ttm_caching_state cstate, unsigned count,
unsigned long *irq_flags)
static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags,
enum ttm_caching_state cstate,
unsigned count, unsigned long *irq_flags)
{
struct page *p;
int r;
@@ -605,7 +617,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
INIT_LIST_HEAD(&new_pages);
r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
cstate, alloc_size);
cstate, alloc_size, 0);
spin_lock_irqsave(&pool->lock, *irq_flags);
if (!r) {
@@ -613,7 +625,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
++pool->nrefills;
pool->npages += alloc_size;
} else {
pr_err("Failed to fill pool (%p)\n", pool);
pr_debug("Failed to fill pool (%p)\n", pool);
/* If we have any pages left put them to the pool. */
list_for_each_entry(p, &new_pages, lru) {
++cpages;
@@ -627,22 +639,25 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
}
/**
* Cut 'count' number of pages from the pool and put them on the return list.
* Allocate pages from the pool and put them on the return list.
*
* @return count of pages still required to fulfill the request.
* @return zero for success or negative error code.
*/
static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
struct list_head *pages,
int ttm_flags,
enum ttm_caching_state cstate,
unsigned count)
static int ttm_page_pool_get_pages(struct ttm_page_pool *pool,
struct list_head *pages,
int ttm_flags,
enum ttm_caching_state cstate,
unsigned count, unsigned order)
{
unsigned long irq_flags;
struct list_head *p;
unsigned i;
int r = 0;
spin_lock_irqsave(&pool->lock, irq_flags);
ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
if (!order)
ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count,
&irq_flags);
if (count >= pool->npages) {
/* take all pages from the pool */
@@ -672,32 +687,126 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
count = 0;
out:
spin_unlock_irqrestore(&pool->lock, irq_flags);
return count;
/* clear the pages coming from the pool if requested */
if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
struct page *page;
list_for_each_entry(page, pages, lru) {
if (PageHighMem(page))
clear_highpage(page);
else
clear_page(page_address(page));
}
}
/* If pool didn't have enough pages allocate new one. */
if (count) {
gfp_t gfp_flags = pool->gfp_flags;
/* set zero flag for page allocation if required */
if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC)
gfp_flags |= __GFP_ZERO;
/* ttm_alloc_new_pages doesn't reference pool so we can run
* multiple requests in parallel.
**/
r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate,
count, order);
}
return r;
}
/* Put all pages in pages list to correct pool to wait for reuse */
static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
enum ttm_caching_state cstate)
{
struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate);
#endif
unsigned long irq_flags;
struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
unsigned i;
if (pool == NULL) {
/* No pool for this memory type so free the pages */
for (i = 0; i < npages; i++) {
if (pages[i]) {
if (page_count(pages[i]) != 1)
pr_err("Erroneous page count. Leaking pages.\n");
__free_page(pages[i]);
pages[i] = NULL;
i = 0;
while (i < npages) {
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
struct page *p = pages[i];
#endif
unsigned order = 0, j;
if (!pages[i]) {
++i;
continue;
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
for (j = 0; j < HPAGE_PMD_NR; ++j)
if (p++ != pages[i + j])
break;
if (j == HPAGE_PMD_NR)
order = HPAGE_PMD_ORDER;
#endif
if (page_count(pages[i]) != 1)
pr_err("Erroneous page count. Leaking pages.\n");
__free_pages(pages[i], order);
j = 1 << order;
while (j) {
pages[i++] = NULL;
--j;
}
}
return;
}
i = 0;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
if (huge) {
unsigned max_size, n2free;
spin_lock_irqsave(&huge->lock, irq_flags);
while (i < npages) {
struct page *p = pages[i];
unsigned j;
if (!p)
break;
for (j = 0; j < HPAGE_PMD_NR; ++j)
if (p++ != pages[i + j])
break;
if (j != HPAGE_PMD_NR)
break;
list_add_tail(&pages[i]->lru, &huge->list);
for (j = 0; j < HPAGE_PMD_NR; ++j)
pages[i++] = NULL;
huge->npages++;
}
/* Check that we don't go over the pool limit */
max_size = _manager->options.max_size;
max_size /= HPAGE_PMD_NR;
if (huge->npages > max_size)
n2free = huge->npages - max_size;
else
n2free = 0;
spin_unlock_irqrestore(&huge->lock, irq_flags);
if (n2free)
ttm_page_pool_free(huge, n2free, false);
}
#endif
spin_lock_irqsave(&pool->lock, irq_flags);
for (i = 0; i < npages; i++) {
while (i < npages) {
if (pages[i]) {
if (page_count(pages[i]) != 1)
pr_err("Erroneous page count. Leaking pages.\n");
@@ -705,6 +814,7 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
pages[i] = NULL;
pool->npages++;
}
++i;
}
/* Check that we don't go over the pool limit */
npages = 0;
@@ -727,76 +837,97 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
enum ttm_caching_state cstate)
{
struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate);
#endif
struct list_head plist;
struct page *p = NULL;
gfp_t gfp_flags = GFP_USER;
unsigned count;
int r;
/* set zero flag for page allocation if required */
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
gfp_flags |= __GFP_ZERO;
/* No pool for cached pages */
if (pool == NULL) {
gfp_t gfp_flags = GFP_USER;
unsigned i;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
unsigned j;
#endif
/* set zero flag for page allocation if required */
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
gfp_flags |= __GFP_ZERO;
if (flags & TTM_PAGE_FLAG_DMA32)
gfp_flags |= GFP_DMA32;
else
gfp_flags |= GFP_HIGHUSER;
for (r = 0; r < npages; ++r) {
i = 0;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
while (npages >= HPAGE_PMD_NR) {
gfp_t huge_flags = gfp_flags;
huge_flags |= GFP_TRANSHUGE;
huge_flags &= ~__GFP_MOVABLE;
huge_flags &= ~__GFP_COMP;
p = alloc_pages(huge_flags, HPAGE_PMD_ORDER);
if (!p)
break;
for (j = 0; j < HPAGE_PMD_NR; ++j)
pages[i++] = p++;
npages -= HPAGE_PMD_NR;
}
#endif
while (npages) {
p = alloc_page(gfp_flags);
if (!p) {
pr_err("Unable to allocate page\n");
pr_debug("Unable to allocate page\n");
return -ENOMEM;
}
pages[r] = p;
pages[i++] = p;
--npages;
}
return 0;
}
/* combine zero flag to pool flags */
gfp_flags |= pool->gfp_flags;
/* First we take pages from the pool */
INIT_LIST_HEAD(&plist);
npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
count = 0;
list_for_each_entry(p, &plist, lru) {
pages[count++] = p;
}
/* clear the pages coming from the pool if requested */
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
list_for_each_entry(p, &plist, lru) {
if (PageHighMem(p))
clear_highpage(p);
else
clear_page(page_address(p));
}
}
/* If pool didn't have enough pages allocate new one. */
if (npages > 0) {
/* ttm_alloc_new_pages doesn't reference pool so we can run
* multiple requests in parallel.
**/
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
if (huge && npages >= HPAGE_PMD_NR) {
INIT_LIST_HEAD(&plist);
r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages);
ttm_page_pool_get_pages(huge, &plist, flags, cstate,
npages / HPAGE_PMD_NR,
HPAGE_PMD_ORDER);
list_for_each_entry(p, &plist, lru) {
pages[count++] = p;
}
if (r) {
/* If there is any pages in the list put them back to
* the pool. */
pr_err("Failed to allocate extra pages for large request\n");
ttm_put_pages(pages, count, flags, cstate);
return r;
unsigned j;
for (j = 0; j < HPAGE_PMD_NR; ++j)
pages[count++] = &p[j];
}
}
#endif
INIT_LIST_HEAD(&plist);
r = ttm_page_pool_get_pages(pool, &plist, flags, cstate,
npages - count, 0);
list_for_each_entry(p, &plist, lru)
pages[count++] = p;
if (r) {
/* If there is any pages in the list put them back to
* the pool.
*/
pr_debug("Failed to allocate extra pages for large request\n");
ttm_put_pages(pages, count, flags, cstate);
return r;
}
return 0;
}
@@ -832,6 +963,14 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
GFP_USER | GFP_DMA32, "uc dma");
ttm_page_pool_init_locked(&_manager->wc_pool_huge,
GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP),
"wc huge");
ttm_page_pool_init_locked(&_manager->uc_pool_huge,
GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP)
, "uc huge");
_manager->options.max_size = max_pages;
_manager->options.small = SMALL_ALLOCATION;
_manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
@@ -873,17 +1012,16 @@ int ttm_pool_populate(struct ttm_tt *ttm)
if (ttm->state != tt_unpopulated)
return 0;
for (i = 0; i < ttm->num_pages; ++i) {
ret = ttm_get_pages(&ttm->pages[i], 1,
ttm->page_flags,
ttm->caching_state);
if (ret != 0) {
ttm_pool_unpopulate(ttm);
return -ENOMEM;
}
ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
ttm->caching_state);
if (unlikely(ret != 0)) {
ttm_pool_unpopulate(ttm);
return ret;
}
for (i = 0; i < ttm->num_pages; ++i) {
ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
false, false);
PAGE_SIZE);
if (unlikely(ret != 0)) {
ttm_pool_unpopulate(ttm);
return -ENOMEM;
@@ -908,18 +1046,91 @@ void ttm_pool_unpopulate(struct ttm_tt *ttm)
unsigned i;
for (i = 0; i < ttm->num_pages; ++i) {
if (ttm->pages[i]) {
ttm_mem_global_free_page(ttm->glob->mem_glob,
ttm->pages[i]);
ttm_put_pages(&ttm->pages[i], 1,
ttm->page_flags,
ttm->caching_state);
}
if (!ttm->pages[i])
continue;
ttm_mem_global_free_page(ttm->glob->mem_glob, ttm->pages[i],
PAGE_SIZE);
}
ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
ttm->caching_state);
ttm->state = tt_unpopulated;
}
EXPORT_SYMBOL(ttm_pool_unpopulate);
#if defined(CONFIG_SWIOTLB) || defined(CONFIG_INTEL_IOMMU)
int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt)
{
unsigned i, j;
int r;
r = ttm_pool_populate(&tt->ttm);
if (r)
return r;
for (i = 0; i < tt->ttm.num_pages; ++i) {
struct page *p = tt->ttm.pages[i];
size_t num_pages = 1;
for (j = i + 1; j < tt->ttm.num_pages; ++j) {
if (++p != tt->ttm.pages[j])
break;
++num_pages;
}
tt->dma_address[i] = dma_map_page(dev, tt->ttm.pages[i],
0, num_pages * PAGE_SIZE,
DMA_BIDIRECTIONAL);
if (dma_mapping_error(dev, tt->dma_address[i])) {
while (i--) {
dma_unmap_page(dev, tt->dma_address[i],
PAGE_SIZE, DMA_BIDIRECTIONAL);
tt->dma_address[i] = 0;
}
ttm_pool_unpopulate(&tt->ttm);
return -EFAULT;
}
for (j = 1; j < num_pages; ++j) {
tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE;
++i;
}
}
return 0;
}
EXPORT_SYMBOL(ttm_populate_and_map_pages);
void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt)
{
unsigned i, j;
for (i = 0; i < tt->ttm.num_pages;) {
struct page *p = tt->ttm.pages[i];
size_t num_pages = 1;
if (!tt->dma_address[i] || !tt->ttm.pages[i]) {
++i;
continue;
}
for (j = i + 1; j < tt->ttm.num_pages; ++j) {
if (++p != tt->ttm.pages[j])
break;
++num_pages;
}
dma_unmap_page(dev, tt->dma_address[i], num_pages * PAGE_SIZE,
DMA_BIDIRECTIONAL);
i += num_pages;
}
ttm_pool_unpopulate(&tt->ttm);
}
EXPORT_SYMBOL(ttm_unmap_and_unpopulate_pages);
#endif
int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
{
struct ttm_page_pool *p;
@@ -929,12 +1140,12 @@ int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
seq_printf(m, "No pool allocator running.\n");
return 0;
}
seq_printf(m, "%6s %12s %13s %8s\n",
seq_printf(m, "%7s %12s %13s %8s\n",
h[0], h[1], h[2], h[3]);
for (i = 0; i < NUM_POOLS; ++i) {
p = &_manager->pools[i];
seq_printf(m, "%6s %12ld %13ld %8d\n",
seq_printf(m, "%7s %12ld %13ld %8d\n",
p->name, p->nrefills,
p->nfrees, p->npages);
}

256
drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
مشاهده پرونده

@@ -60,37 +60,32 @@
#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *))
#define SMALL_ALLOCATION 4
#define FREE_ALL_PAGES (~0U)
/* times are in msecs */
#define IS_UNDEFINED (0)
#define IS_WC (1<<1)
#define IS_UC (1<<2)
#define IS_CACHED (1<<3)
#define IS_DMA32 (1<<4)
#define VADDR_FLAG_HUGE_POOL 1UL
enum pool_type {
POOL_IS_UNDEFINED,
POOL_IS_WC = IS_WC,
POOL_IS_UC = IS_UC,
POOL_IS_CACHED = IS_CACHED,
POOL_IS_WC_DMA32 = IS_WC | IS_DMA32,
POOL_IS_UC_DMA32 = IS_UC | IS_DMA32,
POOL_IS_CACHED_DMA32 = IS_CACHED | IS_DMA32,
IS_UNDEFINED = 0,
IS_WC = 1 << 1,
IS_UC = 1 << 2,
IS_CACHED = 1 << 3,
IS_DMA32 = 1 << 4,
IS_HUGE = 1 << 5
};
/*
* The pool structure. There are usually six pools:
* The pool structure. There are up to nine pools:
* - generic (not restricted to DMA32):
* - write combined, uncached, cached.
* - dma32 (up to 2^32 - so up 4GB):
* - write combined, uncached, cached.
* - huge (not restricted to DMA32):
* - write combined, uncached, cached.
* for each 'struct device'. The 'cached' is for pages that are actively used.
* The other ones can be shrunk by the shrinker API if neccessary.
* @pools: The 'struct device->dma_pools' link.
* @type: Type of the pool
* @lock: Protects the inuse_list and free_list from concurrnet access. Must be
* @lock: Protects the free_list from concurrnet access. Must be
* used with irqsave/irqrestore variants because pool allocator maybe called
* from delayed work.
* @inuse_list: Pool of pages that are in use. The order is very important and
* it is in the order that the TTM pages that are put back are in.
* @free_list: Pool of pages that are free to be used. No order requirements.
* @dev: The device that is associated with these pools.
* @size: Size used during DMA allocation.
@@ -107,7 +102,6 @@ struct dma_pool {
struct list_head pools; /* The 'struct device->dma_pools link */
enum pool_type type;
spinlock_t lock;
struct list_head inuse_list;
struct list_head free_list;
struct device *dev;
unsigned size;
@@ -124,13 +118,14 @@ struct dma_pool {
* The accounting page keeping track of the allocated page along with
* the DMA address.
* @page_list: The link to the 'page_list' in 'struct dma_pool'.
* @vaddr: The virtual address of the page
* @vaddr: The virtual address of the page and a flag if the page belongs to a
* huge pool
* @dma: The bus address of the page. If the page is not allocated
* via the DMA API, it will be -1.
*/
struct dma_page {
struct list_head page_list;
void *vaddr;
unsigned long vaddr;
struct page *p;
dma_addr_t dma;
};
@@ -329,7 +324,8 @@ static int ttm_set_pages_caching(struct dma_pool *pool,
static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page)
{
dma_addr_t dma = d_page->dma;
dma_free_coherent(pool->dev, pool->size, d_page->vaddr, dma);
d_page->vaddr &= ~VADDR_FLAG_HUGE_POOL;
dma_free_coherent(pool->dev, pool->size, (void *)d_page->vaddr, dma);
kfree(d_page);
d_page = NULL;
@@ -337,19 +333,22 @@ static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page)
static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool)
{
struct dma_page *d_page;
void *vaddr;
d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL);
if (!d_page)
return NULL;
d_page->vaddr = dma_alloc_coherent(pool->dev, pool->size,
&d_page->dma,
pool->gfp_flags);
if (d_page->vaddr) {
if (is_vmalloc_addr(d_page->vaddr))
d_page->p = vmalloc_to_page(d_page->vaddr);
vaddr = dma_alloc_coherent(pool->dev, pool->size, &d_page->dma,
pool->gfp_flags);
if (vaddr) {
if (is_vmalloc_addr(vaddr))
d_page->p = vmalloc_to_page(vaddr);
else
d_page->p = virt_to_page(d_page->vaddr);
d_page->p = virt_to_page(vaddr);
d_page->vaddr = (unsigned long)vaddr;
if (pool->type & IS_HUGE)
d_page->vaddr |= VADDR_FLAG_HUGE_POOL;
} else {
kfree(d_page);
d_page = NULL;
@@ -381,11 +380,40 @@ static void ttm_pool_update_free_locked(struct dma_pool *pool,
}
/* set memory back to wb and free the pages. */
static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page)
{
struct page *page = d_page->p;
unsigned i, num_pages;
int ret;
/* Don't set WB on WB page pool. */
if (!(pool->type & IS_CACHED)) {
num_pages = pool->size / PAGE_SIZE;
for (i = 0; i < num_pages; ++i, ++page) {
ret = set_pages_array_wb(&page, 1);
if (ret) {
pr_err("%s: Failed to set %d pages to wb!\n",
pool->dev_name, 1);
}
}
}
list_del(&d_page->page_list);
__ttm_dma_free_page(pool, d_page);
}
static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages,
struct page *pages[], unsigned npages)
{
struct dma_page *d_page, *tmp;
if (pool->type & IS_HUGE) {
list_for_each_entry_safe(d_page, tmp, d_pages, page_list)
ttm_dma_page_put(pool, d_page);
return;
}
/* Don't set WB on WB page pool. */
if (npages && !(pool->type & IS_CACHED) &&
set_pages_array_wb(pages, npages))
@@ -398,17 +426,6 @@ static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages,
}
}
static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page)
{
/* Don't set WB on WB page pool. */
if (!(pool->type & IS_CACHED) && set_pages_array_wb(&d_page->p, 1))
pr_err("%s: Failed to set %d pages to wb!\n",
pool->dev_name, 1);
list_del(&d_page->page_list);
__ttm_dma_free_page(pool, d_page);
}
/*
* Free pages from pool.
*
@@ -446,7 +463,7 @@ static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free,
GFP_KERNEL);
if (!pages_to_free) {
pr_err("%s: Failed to allocate memory for pool free operation\n",
pr_debug("%s: Failed to allocate memory for pool free operation\n",
pool->dev_name);
return 0;
}
@@ -577,8 +594,8 @@ static int ttm_dma_pool_match(struct device *dev, void *res, void *match_data)
static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags,
enum pool_type type)
{
char *n[] = {"wc", "uc", "cached", " dma32", "unknown",};
enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_UNDEFINED};
const char *n[] = {"wc", "uc", "cached", " dma32", "huge"};
enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_HUGE};
struct device_pools *sec_pool = NULL;
struct dma_pool *pool = NULL, **ptr;
unsigned i;
@@ -609,18 +626,24 @@ static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags,
sec_pool->pool = pool;
INIT_LIST_HEAD(&pool->free_list);
INIT_LIST_HEAD(&pool->inuse_list);
INIT_LIST_HEAD(&pool->pools);
spin_lock_init(&pool->lock);
pool->dev = dev;
pool->npages_free = pool->npages_in_use = 0;
pool->nfrees = 0;
pool->gfp_flags = flags;
pool->size = PAGE_SIZE;
if (type & IS_HUGE)
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
pool->size = HPAGE_PMD_SIZE;
#else
BUG();
#endif
else
pool->size = PAGE_SIZE;
pool->type = type;
pool->nrefills = 0;
p = pool->name;
for (i = 0; i < 5; i++) {
for (i = 0; i < ARRAY_SIZE(t); i++) {
if (type & t[i]) {
p += snprintf(p, sizeof(pool->name) - (p - pool->name),
"%s", n[i]);
@@ -724,7 +747,7 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool,
struct dma_page *dma_p;
struct page *p;
int r = 0;
unsigned i, cpages;
unsigned i, j, npages, cpages;
unsigned max_cpages = min(count,
(unsigned)(PAGE_SIZE/sizeof(struct page *)));
@@ -732,7 +755,7 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool,
caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL);
if (!caching_array) {
pr_err("%s: Unable to allocate table for new pages\n",
pr_debug("%s: Unable to allocate table for new pages\n",
pool->dev_name);
return -ENOMEM;
}
@@ -745,8 +768,8 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool,
for (i = 0, cpages = 0; i < count; ++i) {
dma_p = __ttm_dma_alloc_page(pool);
if (!dma_p) {
pr_err("%s: Unable to get page %u\n",
pool->dev_name, i);
pr_debug("%s: Unable to get page %u\n",
pool->dev_name, i);
/* store already allocated pages in the pool after
* setting the caching state */
@@ -762,28 +785,32 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool,
goto out;
}
p = dma_p->p;
list_add(&dma_p->page_list, d_pages);
#ifdef CONFIG_HIGHMEM
/* gfp flags of highmem page should never be dma32 so we
* we should be fine in such case
*/
if (!PageHighMem(p))
if (PageHighMem(p))
continue;
#endif
{
caching_array[cpages++] = p;
npages = pool->size / PAGE_SIZE;
for (j = 0; j < npages; ++j) {
caching_array[cpages++] = p + j;
if (cpages == max_cpages) {
/* Note: Cannot hold the spinlock */
r = ttm_set_pages_caching(pool, caching_array,
cpages);
cpages);
if (r) {
ttm_dma_handle_caching_state_failure(
pool, d_pages, caching_array,
cpages);
pool, d_pages, caching_array,
cpages);
goto out;
}
cpages = 0;
}
}
list_add(&dma_p->page_list, d_pages);
}
if (cpages) {
@@ -828,8 +855,8 @@ static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool,
struct dma_page *d_page;
unsigned cpages = 0;
pr_err("%s: Failed to fill %s pool (r:%d)!\n",
pool->dev_name, pool->name, r);
pr_debug("%s: Failed to fill %s pool (r:%d)!\n",
pool->dev_name, pool->name, r);
list_for_each_entry(d_page, &d_pages, page_list) {
cpages++;
@@ -871,6 +898,27 @@ static int ttm_dma_pool_get_pages(struct dma_pool *pool,
return r;
}
static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge)
{
struct ttm_tt *ttm = &ttm_dma->ttm;
gfp_t gfp_flags;
if (ttm->page_flags & TTM_PAGE_FLAG_DMA32)
gfp_flags = GFP_USER | GFP_DMA32;
else
gfp_flags = GFP_HIGHUSER;
if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)
gfp_flags |= __GFP_ZERO;
if (huge) {
gfp_flags |= GFP_TRANSHUGE;
gfp_flags &= ~__GFP_MOVABLE;
gfp_flags &= ~__GFP_COMP;
}
return gfp_flags;
}
/*
* On success pages list will hold count number of correctly
* cached pages. On failure will hold the negative return value (-ENOMEM, etc).
@@ -879,33 +927,70 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev)
{
struct ttm_tt *ttm = &ttm_dma->ttm;
struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
unsigned long num_pages = ttm->num_pages;
struct dma_pool *pool;
enum pool_type type;
unsigned i;
gfp_t gfp_flags;
int ret;
if (ttm->state != tt_unpopulated)
return 0;
INIT_LIST_HEAD(&ttm_dma->pages_list);
i = 0;
type = ttm_to_type(ttm->page_flags, ttm->caching_state);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
if (ttm->page_flags & TTM_PAGE_FLAG_DMA32)
gfp_flags = GFP_USER | GFP_DMA32;
else
gfp_flags = GFP_HIGHUSER;
if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)
gfp_flags |= __GFP_ZERO;
goto skip_huge;
pool = ttm_dma_find_pool(dev, type | IS_HUGE);
if (!pool) {
gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm_dma, true);
pool = ttm_dma_pool_init(dev, gfp_flags, type | IS_HUGE);
if (IS_ERR_OR_NULL(pool))
goto skip_huge;
}
while (num_pages >= HPAGE_PMD_NR) {
unsigned j;
ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
if (ret != 0)
break;
ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
pool->size);
if (unlikely(ret != 0)) {
ttm_dma_unpopulate(ttm_dma, dev);
return -ENOMEM;
}
for (j = i + 1; j < (i + HPAGE_PMD_NR); ++j) {
ttm->pages[j] = ttm->pages[j - 1] + 1;
ttm_dma->dma_address[j] = ttm_dma->dma_address[j - 1] +
PAGE_SIZE;
}
i += HPAGE_PMD_NR;
num_pages -= HPAGE_PMD_NR;
}
skip_huge:
#endif
pool = ttm_dma_find_pool(dev, type);
if (!pool) {
gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm_dma, false);
pool = ttm_dma_pool_init(dev, gfp_flags, type);
if (IS_ERR_OR_NULL(pool)) {
if (IS_ERR_OR_NULL(pool))
return -ENOMEM;
}
}
INIT_LIST_HEAD(&ttm_dma->pages_list);
for (i = 0; i < ttm->num_pages; ++i) {
while (num_pages) {
ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
if (ret != 0) {
ttm_dma_unpopulate(ttm_dma, dev);
@@ -913,11 +998,14 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev)
}
ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
false, false);
pool->size);
if (unlikely(ret != 0)) {
ttm_dma_unpopulate(ttm_dma, dev);
return -ENOMEM;
}
++i;
--num_pages;
}
if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
@@ -941,10 +1029,33 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
struct dma_page *d_page, *next;
enum pool_type type;
bool is_cached = false;
unsigned count = 0, i, npages = 0;
unsigned count, i, npages = 0;
unsigned long irq_flags;
type = ttm_to_type(ttm->page_flags, ttm->caching_state);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
pool = ttm_dma_find_pool(dev, type | IS_HUGE);
if (pool) {
count = 0;
list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list,
page_list) {
if (!(d_page->vaddr & VADDR_FLAG_HUGE_POOL))
continue;
count++;
ttm_mem_global_free_page(ttm->glob->mem_glob,
d_page->p, pool->size);
ttm_dma_page_put(pool, d_page);
}
spin_lock_irqsave(&pool->lock, irq_flags);
pool->npages_in_use -= count;
pool->nfrees += count;
spin_unlock_irqrestore(&pool->lock, irq_flags);
}
#endif
pool = ttm_dma_find_pool(dev, type);
if (!pool)
return;
@@ -953,6 +1064,7 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
ttm_to_type(ttm->page_flags, tt_cached)) == pool);
/* make sure pages array match list and count number of pages */
count = 0;
list_for_each_entry(d_page, &ttm_dma->pages_list, page_list) {
ttm->pages[count] = d_page->p;
count++;
@@ -978,13 +1090,13 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
if (is_cached) {
list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) {
ttm_mem_global_free_page(ttm->glob->mem_glob,
d_page->p);
d_page->p, pool->size);
ttm_dma_page_put(pool, d_page);
}
} else {
for (i = 0; i < count; i++) {
ttm_mem_global_free_page(ttm->glob->mem_glob,
ttm->pages[i]);
ttm->pages[i], pool->size);
}
}