xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
포크 0
코드 이슈 풀 리퀘스트 Actions Packages Projects 릴리즈 위키 활동

drm/i915: Move more GEM objects under gem/

소스 검색 
Continuing the theme of separating out the GEM clutter.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Mika Kuoppala <mika.kuoppala@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20190528092956.14910-8-chris@chris-wilson.co.uk
This commit is contained in:
Chris Wilson
2019-05-28 10:29:49 +01:00
부모 f0e4a06397
커밋 10be98a77c
77개의 변경된 파일과 339개의 추가작업 그리고 698개의 파일을 삭제
Download Patch File Download Diff File Expand all files Collapse all files

160
drivers/gpu/drm/i915/gem/i915_gem_clflush.c Normal file
파일 보기

@@ -0,0 +1,160 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#include "i915_drv.h"
#include "i915_gem_clflush.h"
#include "intel_frontbuffer.h"
static DEFINE_SPINLOCK(clflush_lock);
struct clflush {
struct dma_fence dma; /* Must be first for dma_fence_free() */
struct i915_sw_fence wait;
struct work_struct work;
struct drm_i915_gem_object *obj;
};
static const char *i915_clflush_get_driver_name(struct dma_fence *fence)
{
return DRIVER_NAME;
}
static const char *i915_clflush_get_timeline_name(struct dma_fence *fence)
{
return "clflush";
}
static void i915_clflush_release(struct dma_fence *fence)
{
struct clflush *clflush = container_of(fence, typeof(*clflush), dma);
i915_sw_fence_fini(&clflush->wait);
BUILD_BUG_ON(offsetof(typeof(*clflush), dma));
dma_fence_free(&clflush->dma);
}
static const struct dma_fence_ops i915_clflush_ops = {
.get_driver_name = i915_clflush_get_driver_name,
.get_timeline_name = i915_clflush_get_timeline_name,
.release = i915_clflush_release,
};
static void __i915_do_clflush(struct drm_i915_gem_object *obj)
{
GEM_BUG_ON(!i915_gem_object_has_pages(obj));
drm_clflush_sg(obj->mm.pages);
intel_fb_obj_flush(obj, ORIGIN_CPU);
}
static void i915_clflush_work(struct work_struct *work)
{
struct clflush *clflush = container_of(work, typeof(*clflush), work);
struct drm_i915_gem_object *obj = clflush->obj;
if (i915_gem_object_pin_pages(obj)) {
DRM_ERROR("Failed to acquire obj->pages for clflushing\n");
goto out;
}
__i915_do_clflush(obj);
i915_gem_object_unpin_pages(obj);
out:
i915_gem_object_put(obj);
dma_fence_signal(&clflush->dma);
dma_fence_put(&clflush->dma);
}
static int __i915_sw_fence_call
i915_clflush_notify(struct i915_sw_fence *fence,
enum i915_sw_fence_notify state)
{
struct clflush *clflush = container_of(fence, typeof(*clflush), wait);
switch (state) {
case FENCE_COMPLETE:
schedule_work(&clflush->work);
break;
case FENCE_FREE:
dma_fence_put(&clflush->dma);
break;
}
return NOTIFY_DONE;
}
bool i915_gem_clflush_object(struct drm_i915_gem_object *obj,
unsigned int flags)
{
struct clflush *clflush;
/*
* Stolen memory is always coherent with the GPU as it is explicitly
* marked as wc by the system, or the system is cache-coherent.
* Similarly, we only access struct pages through the CPU cache, so
* anything not backed by physical memory we consider to be always
* coherent and not need clflushing.
*/
if (!i915_gem_object_has_struct_page(obj)) {
obj->cache_dirty = false;
return false;
}
/* If the GPU is snooping the contents of the CPU cache,
* we do not need to manually clear the CPU cache lines. However,
* the caches are only snooped when the render cache is
* flushed/invalidated. As we always have to emit invalidations
* and flushes when moving into and out of the RENDER domain, correct
* snooping behaviour occurs naturally as the result of our domain
* tracking.
*/
if (!(flags & I915_CLFLUSH_FORCE) &&
obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)
return false;
trace_i915_gem_object_clflush(obj);
clflush = NULL;
if (!(flags & I915_CLFLUSH_SYNC))
clflush = kmalloc(sizeof(*clflush), GFP_KERNEL);
if (clflush) {
GEM_BUG_ON(!obj->cache_dirty);
dma_fence_init(&clflush->dma,
&i915_clflush_ops,
&clflush_lock,
to_i915(obj->base.dev)->mm.unordered_timeline,
0);
i915_sw_fence_init(&clflush->wait, i915_clflush_notify);
clflush->obj = i915_gem_object_get(obj);
INIT_WORK(&clflush->work, i915_clflush_work);
dma_fence_get(&clflush->dma);
i915_sw_fence_await_reservation(&clflush->wait,
obj->resv, NULL,
true, I915_FENCE_TIMEOUT,
I915_FENCE_GFP);
reservation_object_lock(obj->resv, NULL);
reservation_object_add_excl_fence(obj->resv, &clflush->dma);
reservation_object_unlock(obj->resv);
i915_sw_fence_commit(&clflush->wait);
} else if (obj->mm.pages) {
__i915_do_clflush(obj);
} else {
GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
}
obj->cache_dirty = false;
return true;
}

20
drivers/gpu/drm/i915/gem/i915_gem_clflush.h Normal file
파일 보기

@@ -0,0 +1,20 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#ifndef __I915_GEM_CLFLUSH_H__
#define __I915_GEM_CLFLUSH_H__
#include <linux/types.h>
struct drm_i915_private;
struct drm_i915_gem_object;
bool i915_gem_clflush_object(struct drm_i915_gem_object *obj,
unsigned int flags);
#define I915_CLFLUSH_FORCE BIT(0)
#define I915_CLFLUSH_SYNC BIT(1)
#endif /* __I915_GEM_CLFLUSH_H__ */

2453
drivers/gpu/drm/i915/gem/i915_gem_context.c Normal file
파일 보기

파일 크기가 너무 크기때문에 변경 상태를 표시하지 않습니다. Load Diff

240
drivers/gpu/drm/i915/gem/i915_gem_context.h Normal file
파일 보기

@@ -0,0 +1,240 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#ifndef __I915_GEM_CONTEXT_H__
#define __I915_GEM_CONTEXT_H__
#include "i915_gem_context_types.h"
#include "gt/intel_context.h"
#include "i915_gem.h"
#include "i915_scheduler.h"
#include "intel_device_info.h"
struct drm_device;
struct drm_file;
static inline bool i915_gem_context_is_closed(const struct i915_gem_context *ctx)
{
return test_bit(CONTEXT_CLOSED, &ctx->flags);
}
static inline void i915_gem_context_set_closed(struct i915_gem_context *ctx)
{
GEM_BUG_ON(i915_gem_context_is_closed(ctx));
set_bit(CONTEXT_CLOSED, &ctx->flags);
}
static inline bool i915_gem_context_no_error_capture(const struct i915_gem_context *ctx)
{
return test_bit(UCONTEXT_NO_ERROR_CAPTURE, &ctx->user_flags);
}
static inline void i915_gem_context_set_no_error_capture(struct i915_gem_context *ctx)
{
set_bit(UCONTEXT_NO_ERROR_CAPTURE, &ctx->user_flags);
}
static inline void i915_gem_context_clear_no_error_capture(struct i915_gem_context *ctx)
{
clear_bit(UCONTEXT_NO_ERROR_CAPTURE, &ctx->user_flags);
}
static inline bool i915_gem_context_is_bannable(const struct i915_gem_context *ctx)
{
return test_bit(UCONTEXT_BANNABLE, &ctx->user_flags);
}
static inline void i915_gem_context_set_bannable(struct i915_gem_context *ctx)
{
set_bit(UCONTEXT_BANNABLE, &ctx->user_flags);
}
static inline void i915_gem_context_clear_bannable(struct i915_gem_context *ctx)
{
clear_bit(UCONTEXT_BANNABLE, &ctx->user_flags);
}
static inline bool i915_gem_context_is_recoverable(const struct i915_gem_context *ctx)
{
return test_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags);
}
static inline void i915_gem_context_set_recoverable(struct i915_gem_context *ctx)
{
set_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags);
}
static inline void i915_gem_context_clear_recoverable(struct i915_gem_context *ctx)
{
clear_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags);
}
static inline bool i915_gem_context_is_banned(const struct i915_gem_context *ctx)
{
return test_bit(CONTEXT_BANNED, &ctx->flags);
}
static inline void i915_gem_context_set_banned(struct i915_gem_context *ctx)
{
set_bit(CONTEXT_BANNED, &ctx->flags);
}
static inline bool i915_gem_context_force_single_submission(const struct i915_gem_context *ctx)
{
return test_bit(CONTEXT_FORCE_SINGLE_SUBMISSION, &ctx->flags);
}
static inline void i915_gem_context_set_force_single_submission(struct i915_gem_context *ctx)
{
__set_bit(CONTEXT_FORCE_SINGLE_SUBMISSION, &ctx->flags);
}
static inline bool
i915_gem_context_user_engines(const struct i915_gem_context *ctx)
{
return test_bit(CONTEXT_USER_ENGINES, &ctx->flags);
}
static inline void
i915_gem_context_set_user_engines(struct i915_gem_context *ctx)
{
set_bit(CONTEXT_USER_ENGINES, &ctx->flags);
}
static inline void
i915_gem_context_clear_user_engines(struct i915_gem_context *ctx)
{
clear_bit(CONTEXT_USER_ENGINES, &ctx->flags);
}
int __i915_gem_context_pin_hw_id(struct i915_gem_context *ctx);
static inline int i915_gem_context_pin_hw_id(struct i915_gem_context *ctx)
{
if (atomic_inc_not_zero(&ctx->hw_id_pin_count))
return 0;
return __i915_gem_context_pin_hw_id(ctx);
}
static inline void i915_gem_context_unpin_hw_id(struct i915_gem_context *ctx)
{
GEM_BUG_ON(atomic_read(&ctx->hw_id_pin_count) == 0u);
atomic_dec(&ctx->hw_id_pin_count);
}
static inline bool i915_gem_context_is_kernel(struct i915_gem_context *ctx)
{
return !ctx->file_priv;
}
/* i915_gem_context.c */
int __must_check i915_gem_contexts_init(struct drm_i915_private *dev_priv);
void i915_gem_contexts_lost(struct drm_i915_private *dev_priv);
void i915_gem_contexts_fini(struct drm_i915_private *dev_priv);
int i915_gem_context_open(struct drm_i915_private *i915,
struct drm_file *file);
void i915_gem_context_close(struct drm_file *file);
void i915_gem_context_release(struct kref *ctx_ref);
struct i915_gem_context *
i915_gem_context_create_gvt(struct drm_device *dev);
int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data,
struct drm_file *file);
int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data,
struct drm_file *file);
int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
struct drm_file *file);
int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
struct drm_file *file);
int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv);
int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv);
int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data,
struct drm_file *file);
struct i915_gem_context *
i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio);
static inline struct i915_gem_context *
i915_gem_context_get(struct i915_gem_context *ctx)
{
kref_get(&ctx->ref);
return ctx;
}
static inline void i915_gem_context_put(struct i915_gem_context *ctx)
{
kref_put(&ctx->ref, i915_gem_context_release);
}
static inline struct i915_gem_engines *
i915_gem_context_engines(struct i915_gem_context *ctx)
{
return rcu_dereference_protected(ctx->engines,
lockdep_is_held(&ctx->engines_mutex));
}
static inline struct i915_gem_engines *
i915_gem_context_lock_engines(struct i915_gem_context *ctx)
__acquires(&ctx->engines_mutex)
{
mutex_lock(&ctx->engines_mutex);
return i915_gem_context_engines(ctx);
}
static inline void
i915_gem_context_unlock_engines(struct i915_gem_context *ctx)
__releases(&ctx->engines_mutex)
{
mutex_unlock(&ctx->engines_mutex);
}
static inline struct intel_context *
i915_gem_context_lookup_engine(struct i915_gem_context *ctx, unsigned int idx)
{
return i915_gem_context_engines(ctx)->engines[idx];
}
static inline struct intel_context *
i915_gem_context_get_engine(struct i915_gem_context *ctx, unsigned int idx)
{
struct intel_context *ce = ERR_PTR(-EINVAL);
rcu_read_lock(); {
struct i915_gem_engines *e = rcu_dereference(ctx->engines);
if (likely(idx < e->num_engines && e->engines[idx]))
ce = intel_context_get(e->engines[idx]);
} rcu_read_unlock();
return ce;
}
static inline void
i915_gem_engines_iter_init(struct i915_gem_engines_iter *it,
struct i915_gem_engines *engines)
{
GEM_BUG_ON(!engines);
it->engines = engines;
it->idx = 0;
}
struct intel_context *
i915_gem_engines_iter_next(struct i915_gem_engines_iter *it);
#define for_each_gem_engine(ce, engines, it) \
for (i915_gem_engines_iter_init(&(it), (engines)); \
((ce) = i915_gem_engines_iter_next(&(it)));)
struct i915_lut_handle *i915_lut_handle_alloc(void);
void i915_lut_handle_free(struct i915_lut_handle *lut);
#endif /* !__I915_GEM_CONTEXT_H__ */

208
drivers/gpu/drm/i915/gem/i915_gem_context_types.h Normal file
파일 보기

@@ -0,0 +1,208 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2019 Intel Corporation
*/
#ifndef __I915_GEM_CONTEXT_TYPES_H__
#define __I915_GEM_CONTEXT_TYPES_H__
#include <linux/atomic.h>
#include <linux/list.h>
#include <linux/llist.h>
#include <linux/kref.h>
#include <linux/mutex.h>
#include <linux/radix-tree.h>
#include <linux/rbtree.h>
#include <linux/rcupdate.h>
#include <linux/types.h>
#include "gt/intel_context_types.h"
#include "i915_scheduler.h"
struct pid;
struct drm_i915_private;
struct drm_i915_file_private;
struct i915_hw_ppgtt;
struct i915_timeline;
struct intel_ring;
struct i915_gem_engines {
struct rcu_work rcu;
struct drm_i915_private *i915;
unsigned int num_engines;
struct intel_context *engines[];
};
struct i915_gem_engines_iter {
unsigned int idx;
const struct i915_gem_engines *engines;
};
/**
* struct i915_gem_context - client state
*
* The struct i915_gem_context represents the combined view of the driver and
* logical hardware state for a particular client.
*/
struct i915_gem_context {
/** i915: i915 device backpointer */
struct drm_i915_private *i915;
/** file_priv: owning file descriptor */
struct drm_i915_file_private *file_priv;
/**
* @engines: User defined engines for this context
*
* Various uAPI offer the ability to lookup up an
* index from this array to select an engine operate on.
*
* Multiple logically distinct instances of the same engine
* may be defined in the array, as well as composite virtual
* engines.
*
* Execbuf uses the I915_EXEC_RING_MASK as an index into this
* array to select which HW context + engine to execute on. For
* the default array, the user_ring_map[] is used to translate
* the legacy uABI onto the approprate index (e.g. both
* I915_EXEC_DEFAULT and I915_EXEC_RENDER select the same
* context, and I915_EXEC_BSD is weird). For a use defined
* array, execbuf uses I915_EXEC_RING_MASK as a plain index.
*
* User defined by I915_CONTEXT_PARAM_ENGINE (when the
* CONTEXT_USER_ENGINES flag is set).
*/
struct i915_gem_engines __rcu *engines;
struct mutex engines_mutex; /* guards writes to engines */
struct i915_timeline *timeline;
/**
* @ppgtt: unique address space (GTT)
*
* In full-ppgtt mode, each context has its own address space ensuring
* complete seperation of one client from all others.
*
* In other modes, this is a NULL pointer with the expectation that
* the caller uses the shared global GTT.
*/
struct i915_hw_ppgtt *ppgtt;
/**
* @pid: process id of creator
*
* Note that who created the context may not be the principle user,
* as the context may be shared across a local socket. However,
* that should only affect the default context, all contexts created
* explicitly by the client are expected to be isolated.
*/
struct pid *pid;
/**
* @name: arbitrary name
*
* A name is constructed for the context from the creator's process
* name, pid and user handle in order to uniquely identify the
* context in messages.
*/
const char *name;
/** link: place with &drm_i915_private.context_list */
struct list_head link;
struct llist_node free_link;
/**
* @ref: reference count
*
* A reference to a context is held by both the client who created it
* and on each request submitted to the hardware using the request
* (to ensure the hardware has access to the state until it has
* finished all pending writes). See i915_gem_context_get() and
* i915_gem_context_put() for access.
*/
struct kref ref;
/**
* @rcu: rcu_head for deferred freeing.
*/
struct rcu_head rcu;
/**
* @user_flags: small set of booleans controlled by the user
*/
unsigned long user_flags;
#define UCONTEXT_NO_ZEROMAP 0
#define UCONTEXT_NO_ERROR_CAPTURE 1
#define UCONTEXT_BANNABLE 2
#define UCONTEXT_RECOVERABLE 3
/**
* @flags: small set of booleans
*/
unsigned long flags;
#define CONTEXT_BANNED 0
#define CONTEXT_CLOSED 1
#define CONTEXT_FORCE_SINGLE_SUBMISSION 2
#define CONTEXT_USER_ENGINES 3
/**
* @hw_id: - unique identifier for the context
*
* The hardware needs to uniquely identify the context for a few
* functions like fault reporting, PASID, scheduling. The
* &drm_i915_private.context_hw_ida is used to assign a unqiue
* id for the lifetime of the context.
*
* @hw_id_pin_count: - number of times this context had been pinned
* for use (should be, at most, once per engine).
*
* @hw_id_link: - all contexts with an assigned id are tracked
* for possible repossession.
*/
unsigned int hw_id;
atomic_t hw_id_pin_count;
struct list_head hw_id_link;
struct mutex mutex;
struct i915_sched_attr sched;
/** ring_size: size for allocating the per-engine ring buffer */
u32 ring_size;
/** desc_template: invariant fields for the HW context descriptor */
u32 desc_template;
/** guilty_count: How many times this context has caused a GPU hang. */
atomic_t guilty_count;
/**
* @active_count: How many times this context was active during a GPU
* hang, but did not cause it.
*/
atomic_t active_count;
/**
* @hang_timestamp: The last time(s) this context caused a GPU hang
*/
unsigned long hang_timestamp[2];
#define CONTEXT_FAST_HANG_JIFFIES (120 * HZ) /* 3 hangs within 120s? Banned! */
/** remap_slice: Bitmask of cache lines that need remapping */
u8 remap_slice;
/** handles_vma: rbtree to look up our context specific obj/vma for
* the user handle. (user handles are per fd, but the binding is
* per vm, which may be one per context or shared with the global GTT)
*/
struct radix_tree_root handles_vma;
/** handles_list: reverse list of all the rbtree entries in use for
* this context, which allows us to free all the allocations on
* context close.
*/
struct list_head handles_list;
};
#endif /* __I915_GEM_CONTEXT_TYPES_H__ */

318
drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c Normal file
파일 보기

@@ -0,0 +1,318 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright 2012 Red Hat Inc
*/
#include <linux/dma-buf.h>
#include <linux/highmem.h>
#include <linux/reservation.h>
#include "i915_drv.h"
#include "i915_gem_object.h"
static struct drm_i915_gem_object *dma_buf_to_obj(struct dma_buf *buf)
{
return to_intel_bo(buf->priv);
}
static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachment,
enum dma_data_direction dir)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(attachment->dmabuf);
struct sg_table *st;
struct scatterlist *src, *dst;
int ret, i;
ret = i915_gem_object_pin_pages(obj);
if (ret)
goto err;
/* Copy sg so that we make an independent mapping */
st = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
if (st == NULL) {
ret = -ENOMEM;
goto err_unpin_pages;
}
ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL);
if (ret)
goto err_free;
src = obj->mm.pages->sgl;
dst = st->sgl;
for (i = 0; i < obj->mm.pages->nents; i++) {
sg_set_page(dst, sg_page(src), src->length, 0);
dst = sg_next(dst);
src = sg_next(src);
}
if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) {
ret = -ENOMEM;
goto err_free_sg;
}
return st;
err_free_sg:
sg_free_table(st);
err_free:
kfree(st);
err_unpin_pages:
i915_gem_object_unpin_pages(obj);
err:
return ERR_PTR(ret);
}
static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment,
struct sg_table *sg,
enum dma_data_direction dir)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(attachment->dmabuf);
dma_unmap_sg(attachment->dev, sg->sgl, sg->nents, dir);
sg_free_table(sg);
kfree(sg);
i915_gem_object_unpin_pages(obj);
}
static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
return i915_gem_object_pin_map(obj, I915_MAP_WB);
}
static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
i915_gem_object_flush_map(obj);
i915_gem_object_unpin_map(obj);
}
static void *i915_gem_dmabuf_kmap(struct dma_buf *dma_buf, unsigned long page_num)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
struct page *page;
if (page_num >= obj->base.size >> PAGE_SHIFT)
return NULL;
if (!i915_gem_object_has_struct_page(obj))
return NULL;
if (i915_gem_object_pin_pages(obj))
return NULL;
/* Synchronisation is left to the caller (via .begin_cpu_access()) */
page = i915_gem_object_get_page(obj, page_num);
if (IS_ERR(page))
goto err_unpin;
return kmap(page);
err_unpin:
i915_gem_object_unpin_pages(obj);
return NULL;
}
static void i915_gem_dmabuf_kunmap(struct dma_buf *dma_buf, unsigned long page_num, void *addr)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
kunmap(virt_to_page(addr));
i915_gem_object_unpin_pages(obj);
}
static int i915_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
int ret;
if (obj->base.size < vma->vm_end - vma->vm_start)
return -EINVAL;
if (!obj->base.filp)
return -ENODEV;
ret = call_mmap(obj->base.filp, vma);
if (ret)
return ret;
fput(vma->vm_file);
vma->vm_file = get_file(obj->base.filp);
return 0;
}
static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
struct drm_device *dev = obj->base.dev;
bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE);
int err;
err = i915_gem_object_pin_pages(obj);
if (err)
return err;
err = i915_mutex_lock_interruptible(dev);
if (err)
goto out;
err = i915_gem_object_set_to_cpu_domain(obj, write);
mutex_unlock(&dev->struct_mutex);
out:
i915_gem_object_unpin_pages(obj);
return err;
}
static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
struct drm_device *dev = obj->base.dev;
int err;
err = i915_gem_object_pin_pages(obj);
if (err)
return err;
err = i915_mutex_lock_interruptible(dev);
if (err)
goto out;
err = i915_gem_object_set_to_gtt_domain(obj, false);
mutex_unlock(&dev->struct_mutex);
out:
i915_gem_object_unpin_pages(obj);
return err;
}
static const struct dma_buf_ops i915_dmabuf_ops = {
.map_dma_buf = i915_gem_map_dma_buf,
.unmap_dma_buf = i915_gem_unmap_dma_buf,
.release = drm_gem_dmabuf_release,
.map = i915_gem_dmabuf_kmap,
.unmap = i915_gem_dmabuf_kunmap,
.mmap = i915_gem_dmabuf_mmap,
.vmap = i915_gem_dmabuf_vmap,
.vunmap = i915_gem_dmabuf_vunmap,
.begin_cpu_access = i915_gem_begin_cpu_access,
.end_cpu_access = i915_gem_end_cpu_access,
};
struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
struct drm_gem_object *gem_obj, int flags)
{
struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
exp_info.ops = &i915_dmabuf_ops;
exp_info.size = gem_obj->size;
exp_info.flags = flags;
exp_info.priv = gem_obj;
exp_info.resv = obj->resv;
if (obj->ops->dmabuf_export) {
int ret = obj->ops->dmabuf_export(obj);
if (ret)
return ERR_PTR(ret);
}
return drm_gem_dmabuf_export(dev, &exp_info);
}
static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
{
struct sg_table *pages;
unsigned int sg_page_sizes;
pages = dma_buf_map_attachment(obj->base.import_attach,
DMA_BIDIRECTIONAL);
if (IS_ERR(pages))
return PTR_ERR(pages);
sg_page_sizes = i915_sg_page_sizes(pages->sgl);
__i915_gem_object_set_pages(obj, pages, sg_page_sizes);
return 0;
}
static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj,
struct sg_table *pages)
{
dma_buf_unmap_attachment(obj->base.import_attach, pages,
DMA_BIDIRECTIONAL);
}
static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {
.get_pages = i915_gem_object_get_pages_dmabuf,
.put_pages = i915_gem_object_put_pages_dmabuf,
};
struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
struct dma_buf *dma_buf)
{
struct dma_buf_attachment *attach;
struct drm_i915_gem_object *obj;
int ret;
/* is this one of own objects? */
if (dma_buf->ops == &i915_dmabuf_ops) {
obj = dma_buf_to_obj(dma_buf);
/* is it from our device? */
if (obj->base.dev == dev) {
/*
* Importing dmabuf exported from out own gem increases
* refcount on gem itself instead of f_count of dmabuf.
*/
return &i915_gem_object_get(obj)->base;
}
}
/* need to attach */
attach = dma_buf_attach(dma_buf, dev->dev);
if (IS_ERR(attach))
return ERR_CAST(attach);
get_dma_buf(dma_buf);
obj = i915_gem_object_alloc();
if (obj == NULL) {
ret = -ENOMEM;
goto fail_detach;
}
drm_gem_private_object_init(dev, &obj->base, dma_buf->size);
i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops);
obj->base.import_attach = attach;
obj->resv = dma_buf->resv;
/* We use GTT as shorthand for a coherent domain, one that is
* neither in the GPU cache nor in the CPU cache, where all
* writes are immediately visible in memory. (That's not strictly
* true, but it's close! There are internal buffers such as the
* write-combined buffer or a delay through the chipset for GTT
* writes that do require us to treat GTT as a separate cache domain.)
*/
obj->read_domains = I915_GEM_DOMAIN_GTT;
obj->write_domain = 0;
return &obj->base;
fail_detach:
dma_buf_detach(dma_buf, attach);
dma_buf_put(dma_buf);
return ERR_PTR(ret);
}
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/mock_dmabuf.c"
#include "selftests/i915_gem_dmabuf.c"
#endif

2768
drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c Normal file
파일 보기

파일 크기가 너무 크기때문에 변경 상태를 표시하지 않습니다. Load Diff

197
drivers/gpu/drm/i915/gem/i915_gem_internal.c Normal file
파일 보기

@@ -0,0 +1,197 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2014-2016 Intel Corporation
*/
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/swiotlb.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"
#include "i915_gem.h"
#include "i915_gem_object.h"
#include "i915_utils.h"
#define QUIET (__GFP_NORETRY | __GFP_NOWARN)
#define MAYFAIL (__GFP_RETRY_MAYFAIL | __GFP_NOWARN)
static void internal_free_pages(struct sg_table *st)
{
struct scatterlist *sg;
for (sg = st->sgl; sg; sg = __sg_next(sg)) {
if (sg_page(sg))
__free_pages(sg_page(sg), get_order(sg->length));
}
sg_free_table(st);
kfree(st);
}
static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
{
struct drm_i915_private *i915 = to_i915(obj->base.dev);
struct sg_table *st;
struct scatterlist *sg;
unsigned int sg_page_sizes;
unsigned int npages;
int max_order;
gfp_t gfp;
max_order = MAX_ORDER;
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
unsigned int max_segment;
max_segment = swiotlb_max_segment();
if (max_segment) {
max_segment = max_t(unsigned int, max_segment,
PAGE_SIZE) >> PAGE_SHIFT;
max_order = min(max_order, ilog2(max_segment));
}
}
#endif
gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE;
if (IS_I965GM(i915) || IS_I965G(i915)) {
/* 965gm cannot relocate objects above 4GiB. */
gfp &= ~__GFP_HIGHMEM;
gfp |= __GFP_DMA32;
}
create_st:
st = kmalloc(sizeof(*st), GFP_KERNEL);
if (!st)
return -ENOMEM;
npages = obj->base.size / PAGE_SIZE;
if (sg_alloc_table(st, npages, GFP_KERNEL)) {
kfree(st);
return -ENOMEM;
}
sg = st->sgl;
st->nents = 0;
sg_page_sizes = 0;
do {
int order = min(fls(npages) - 1, max_order);
struct page *page;
do {
page = alloc_pages(gfp | (order ? QUIET : MAYFAIL),
order);
if (page)
break;
if (!order--)
goto err;
/* Limit subsequent allocations as well */
max_order = order;
} while (1);
sg_set_page(sg, page, PAGE_SIZE << order, 0);
sg_page_sizes |= PAGE_SIZE << order;
st->nents++;
npages -= 1 << order;
if (!npages) {
sg_mark_end(sg);
break;
}
sg = __sg_next(sg);
} while (1);
if (i915_gem_gtt_prepare_pages(obj, st)) {
/* Failed to dma-map try again with single page sg segments */
if (get_order(st->sgl->length)) {
internal_free_pages(st);
max_order = 0;
goto create_st;
}
goto err;
}
/* Mark the pages as dontneed whilst they are still pinned. As soon
* as they are unpinned they are allowed to be reaped by the shrinker,
* and the caller is expected to repopulate - the contents of this
* object are only valid whilst active and pinned.
*/
obj->mm.madv = I915_MADV_DONTNEED;
__i915_gem_object_set_pages(obj, st, sg_page_sizes);
return 0;
err:
sg_set_page(sg, NULL, 0, 0);
sg_mark_end(sg);
internal_free_pages(st);
return -ENOMEM;
}
static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj,
struct sg_table *pages)
{
i915_gem_gtt_finish_pages(obj, pages);
internal_free_pages(pages);
obj->mm.dirty = false;
obj->mm.madv = I915_MADV_WILLNEED;
}
static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
I915_GEM_OBJECT_IS_SHRINKABLE,
.get_pages = i915_gem_object_get_pages_internal,
.put_pages = i915_gem_object_put_pages_internal,
};
/**
* i915_gem_object_create_internal: create an object with volatile pages
* @i915: the i915 device
* @size: the size in bytes of backing storage to allocate for the object
*
* Creates a new object that wraps some internal memory for private use.
* This object is not backed by swappable storage, and as such its contents
* are volatile and only valid whilst pinned. If the object is reaped by the
* shrinker, its pages and data will be discarded. Equally, it is not a full
* GEM object and so not valid for access from userspace. This makes it useful
* for hardware interfaces like ringbuffers (which are pinned from the time
* the request is written to the time the hardware stops accessing it), but
* not for contexts (which need to be preserved when not active for later
* reuse). Note that it is not cleared upon allocation.
*/
struct drm_i915_gem_object *
i915_gem_object_create_internal(struct drm_i915_private *i915,
phys_addr_t size)
{
struct drm_i915_gem_object *obj;
unsigned int cache_level;
GEM_BUG_ON(!size);
GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE));
if (overflows_type(size, obj->base.size))
return ERR_PTR(-E2BIG);
obj = i915_gem_object_alloc();
if (!obj)
return ERR_PTR(-ENOMEM);
drm_gem_private_object_init(&i915->drm, &obj->base, size);
i915_gem_object_init(obj, &i915_gem_object_internal_ops);
obj->read_domains = I915_GEM_DOMAIN_CPU;
obj->write_domain = I915_GEM_DOMAIN_CPU;
cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
i915_gem_object_set_cache_coherency(obj, cache_level);
return obj;
}

10
drivers/gpu/drm/i915/gem/i915_gem_object.c
파일 보기

@@ -23,8 +23,9 @@
*/
#include "i915_drv.h"
#include "i915_gem_object.h"
#include "i915_gem_clflush.h"
#include "i915_gem_context.h"
#include "i915_gem_object.h"
#include "i915_globals.h"
#include "intel_frontbuffer.h"
@@ -442,3 +443,10 @@ int __init i915_global_objects_init(void)
i915_global_register(&global.base);
return 0;
}
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/huge_gem_object.c"
#include "selftests/huge_pages.c"
#include "selftests/i915_gem_object.c"
#include "selftests/i915_gem_coherency.c"
#endif

251
drivers/gpu/drm/i915/gem/i915_gem_pm.c Normal file
파일 보기

@@ -0,0 +1,251 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2019 Intel Corporation
*/
#include "gem/i915_gem_pm.h"
#include "gt/intel_gt_pm.h"
#include "i915_drv.h"
#include "i915_globals.h"
static void i915_gem_park(struct drm_i915_private *i915)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
lockdep_assert_held(&i915->drm.struct_mutex);
for_each_engine(engine, i915, id)
i915_gem_batch_pool_fini(&engine->batch_pool);
i915_timelines_park(i915);
i915_vma_parked(i915);
i915_globals_park();
}
static void idle_work_handler(struct work_struct *work)
{
struct drm_i915_private *i915 =
container_of(work, typeof(*i915), gem.idle_work);
bool restart = true;
cancel_delayed_work(&i915->gem.retire_work);
mutex_lock(&i915->drm.struct_mutex);
intel_wakeref_lock(&i915->gt.wakeref);
if (!intel_wakeref_active(&i915->gt.wakeref) && !work_pending(work)) {
i915_gem_park(i915);
restart = false;
}
intel_wakeref_unlock(&i915->gt.wakeref);
mutex_unlock(&i915->drm.struct_mutex);
if (restart)
queue_delayed_work(i915->wq,
&i915->gem.retire_work,
round_jiffies_up_relative(HZ));
}
static void retire_work_handler(struct work_struct *work)
{
struct drm_i915_private *i915 =
container_of(work, typeof(*i915), gem.retire_work.work);
/* Come back later if the device is busy... */
if (mutex_trylock(&i915->drm.struct_mutex)) {
i915_retire_requests(i915);
mutex_unlock(&i915->drm.struct_mutex);
}
queue_delayed_work(i915->wq,
&i915->gem.retire_work,
round_jiffies_up_relative(HZ));
}
static int pm_notifier(struct notifier_block *nb,
unsigned long action,
void *data)
{
struct drm_i915_private *i915 =
container_of(nb, typeof(*i915), gem.pm_notifier);
switch (action) {
case INTEL_GT_UNPARK:
i915_globals_unpark();
queue_delayed_work(i915->wq,
&i915->gem.retire_work,
round_jiffies_up_relative(HZ));
break;
case INTEL_GT_PARK:
queue_work(i915->wq, &i915->gem.idle_work);
break;
}
return NOTIFY_OK;
}
static bool switch_to_kernel_context_sync(struct drm_i915_private *i915)
{
bool result = true;
do {
if (i915_gem_wait_for_idle(i915,
I915_WAIT_LOCKED |
I915_WAIT_FOR_IDLE_BOOST,
I915_GEM_IDLE_TIMEOUT) == -ETIME) {
/* XXX hide warning from gem_eio */
if (i915_modparams.reset) {
dev_err(i915->drm.dev,
"Failed to idle engines, declaring wedged!\n");
GEM_TRACE_DUMP();
}
/*
* Forcibly cancel outstanding work and leave
* the gpu quiet.
*/
i915_gem_set_wedged(i915);
result = false;
}
} while (i915_retire_requests(i915) && result);
GEM_BUG_ON(i915->gt.awake);
return result;
}
bool i915_gem_load_power_context(struct drm_i915_private *i915)
{
return switch_to_kernel_context_sync(i915);
}
void i915_gem_suspend(struct drm_i915_private *i915)
{
GEM_TRACE("\n");
intel_wakeref_auto(&i915->mm.userfault_wakeref, 0);
flush_workqueue(i915->wq);
mutex_lock(&i915->drm.struct_mutex);
/*
* We have to flush all the executing contexts to main memory so
* that they can saved in the hibernation image. To ensure the last
* context image is coherent, we have to switch away from it. That
* leaves the i915->kernel_context still active when
* we actually suspend, and its image in memory may not match the GPU
* state. Fortunately, the kernel_context is disposable and we do
* not rely on its state.
*/
switch_to_kernel_context_sync(i915);
mutex_unlock(&i915->drm.struct_mutex);
/*
* Assert that we successfully flushed all the work and
* reset the GPU back to its idle, low power state.
*/
GEM_BUG_ON(i915->gt.awake);
flush_work(&i915->gem.idle_work);
cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work);
i915_gem_drain_freed_objects(i915);
intel_uc_suspend(i915);
}
void i915_gem_suspend_late(struct drm_i915_private *i915)
{
struct drm_i915_gem_object *obj;
struct list_head *phases[] = {
&i915->mm.unbound_list,
&i915->mm.bound_list,
NULL
}, **phase;
/*
* Neither the BIOS, ourselves or any other kernel
* expects the system to be in execlists mode on startup,
* so we need to reset the GPU back to legacy mode. And the only
* known way to disable logical contexts is through a GPU reset.
*
* So in order to leave the system in a known default configuration,
* always reset the GPU upon unload and suspend. Afterwards we then
* clean up the GEM state tracking, flushing off the requests and
* leaving the system in a known idle state.
*
* Note that is of the upmost importance that the GPU is idle and
* all stray writes are flushed *before* we dismantle the backing
* storage for the pinned objects.
*
* However, since we are uncertain that resetting the GPU on older
* machines is a good idea, we don't - just in case it leaves the
* machine in an unusable condition.
*/
mutex_lock(&i915->drm.struct_mutex);
for (phase = phases; *phase; phase++) {
list_for_each_entry(obj, *phase, mm.link)
WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
}
mutex_unlock(&i915->drm.struct_mutex);
intel_uc_sanitize(i915);
i915_gem_sanitize(i915);
}
void i915_gem_resume(struct drm_i915_private *i915)
{
GEM_TRACE("\n");
WARN_ON(i915->gt.awake);
mutex_lock(&i915->drm.struct_mutex);
intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
i915_gem_restore_gtt_mappings(i915);
i915_gem_restore_fences(i915);
/*
* As we didn't flush the kernel context before suspend, we cannot
* guarantee that the context image is complete. So let's just reset
* it and start again.
*/
intel_gt_resume(i915);
if (i915_gem_init_hw(i915))
goto err_wedged;
intel_uc_resume(i915);
/* Always reload a context for powersaving. */
if (!i915_gem_load_power_context(i915))
goto err_wedged;
out_unlock:
intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
mutex_unlock(&i915->drm.struct_mutex);
return;
err_wedged:
if (!i915_reset_failed(i915)) {
dev_err(i915->drm.dev,
"Failed to re-initialize GPU, declaring it wedged!\n");
i915_gem_set_wedged(i915);
}
goto out_unlock;
}
void i915_gem_init__pm(struct drm_i915_private *i915)
{
INIT_WORK(&i915->gem.idle_work, idle_work_handler);
INIT_DELAYED_WORK(&i915->gem.retire_work, retire_work_handler);
i915->gem.pm_notifier.notifier_call = pm_notifier;
blocking_notifier_chain_register(&i915->gt.pm_notifications,
&i915->gem.pm_notifier);
}

25
drivers/gpu/drm/i915/gem/i915_gem_pm.h Normal file
파일 보기

@@ -0,0 +1,25 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2019 Intel Corporation
*/
#ifndef __I915_GEM_PM_H__
#define __I915_GEM_PM_H__
#include <linux/types.h>
struct drm_i915_private;
struct work_struct;
void i915_gem_init__pm(struct drm_i915_private *i915);
bool i915_gem_load_power_context(struct drm_i915_private *i915);
void i915_gem_resume(struct drm_i915_private *i915);
void i915_gem_idle_work_handler(struct work_struct *work);
void i915_gem_suspend(struct drm_i915_private *i915);
void i915_gem_suspend_late(struct drm_i915_private *i915);
#endif /* __I915_GEM_PM_H__ */

555
drivers/gpu/drm/i915/gem/i915_gem_shrinker.c Normal file
파일 보기

@@ -0,0 +1,555 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2008-2015 Intel Corporation
*/
#include <linux/oom.h>
#include <linux/sched/mm.h>
#include <linux/shmem_fs.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/pci.h>
#include <linux/dma-buf.h>
#include <linux/vmalloc.h>
#include <drm/i915_drm.h>
#include "i915_trace.h"
static bool shrinker_lock(struct drm_i915_private *i915,
unsigned int flags,
bool *unlock)
{
struct mutex *m = &i915->drm.struct_mutex;
switch (mutex_trylock_recursive(m)) {
case MUTEX_TRYLOCK_RECURSIVE:
*unlock = false;
return true;
case MUTEX_TRYLOCK_FAILED:
*unlock = false;
if (flags & I915_SHRINK_ACTIVE &&
mutex_lock_killable_nested(m, I915_MM_SHRINKER) == 0)
*unlock = true;
return *unlock;
case MUTEX_TRYLOCK_SUCCESS:
*unlock = true;
return true;
}
BUG();
}
static void shrinker_unlock(struct drm_i915_private *i915, bool unlock)
{
if (!unlock)
return;
mutex_unlock(&i915->drm.struct_mutex);
}
static bool swap_available(void)
{
return get_nr_swap_pages() > 0;
}
static bool can_release_pages(struct drm_i915_gem_object *obj)
{
/* Consider only shrinkable ojects. */
if (!i915_gem_object_is_shrinkable(obj))
return false;
/* Only report true if by unbinding the object and putting its pages
* we can actually make forward progress towards freeing physical
* pages.
*
* If the pages are pinned for any other reason than being bound
* to the GPU, simply unbinding from the GPU is not going to succeed
* in releasing our pin count on the pages themselves.
*/
if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count)
return false;
/* If any vma are "permanently" pinned, it will prevent us from
* reclaiming the obj->mm.pages. We only allow scanout objects to claim
* a permanent pin, along with a few others like the context objects.
* To simplify the scan, and to avoid walking the list of vma under the
* object, we just check the count of its permanently pinned.
*/
if (READ_ONCE(obj->pin_global))
return false;
/* We can only return physical pages to the system if we can either
* discard the contents (because the user has marked them as being
* purgeable) or if we can move their contents out to swap.
*/
return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
}
static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
{
if (i915_gem_object_unbind(obj) == 0)
__i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
return !i915_gem_object_has_pages(obj);
}
static void try_to_writeback(struct drm_i915_gem_object *obj,
unsigned int flags)
{
switch (obj->mm.madv) {
case I915_MADV_DONTNEED:
i915_gem_object_truncate(obj);
case __I915_MADV_PURGED:
return;
}
if (flags & I915_SHRINK_WRITEBACK)
i915_gem_object_writeback(obj);
}
/**
* i915_gem_shrink - Shrink buffer object caches
* @i915: i915 device
* @target: amount of memory to make available, in pages
* @nr_scanned: optional output for number of pages scanned (incremental)
* @flags: control flags for selecting cache types
*
* This function is the main interface to the shrinker. It will try to release
* up to @target pages of main memory backing storage from buffer objects.
* Selection of the specific caches can be done with @flags. This is e.g. useful
* when purgeable objects should be removed from caches preferentially.
*
* Note that it's not guaranteed that released amount is actually available as
* free system memory - the pages might still be in-used to due to other reasons
* (like cpu mmaps) or the mm core has reused them before we could grab them.
* Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
* avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
*
* Also note that any kind of pinning (both per-vma address space pins and
* backing storage pins at the buffer object level) result in the shrinker code
* having to skip the object.
*
* Returns:
* The number of pages of backing storage actually released.
*/
unsigned long
i915_gem_shrink(struct drm_i915_private *i915,
unsigned long target,
unsigned long *nr_scanned,
unsigned flags)
{
const struct {
struct list_head *list;
unsigned int bit;
} phases[] = {
{ &i915->mm.unbound_list, I915_SHRINK_UNBOUND },
{ &i915->mm.bound_list, I915_SHRINK_BOUND },
{ NULL, 0 },
}, *phase;
intel_wakeref_t wakeref = 0;
unsigned long count = 0;
unsigned long scanned = 0;
bool unlock;
if (!shrinker_lock(i915, flags, &unlock))
return 0;
/*
* When shrinking the active list, also consider active contexts.
* Active contexts are pinned until they are retired, and so can
* not be simply unbound to retire and unpin their pages. To shrink
* the contexts, we must wait until the gpu is idle.
*
* We don't care about errors here; if we cannot wait upon the GPU,
* we will free as much as we can and hope to get a second chance.
*/
if (flags & I915_SHRINK_ACTIVE)
i915_gem_wait_for_idle(i915,
I915_WAIT_LOCKED,
MAX_SCHEDULE_TIMEOUT);
trace_i915_gem_shrink(i915, target, flags);
i915_retire_requests(i915);
/*
* Unbinding of objects will require HW access; Let us not wake the
* device just to recover a little memory. If absolutely necessary,
* we will force the wake during oom-notifier.
*/
if (flags & I915_SHRINK_BOUND) {
wakeref = intel_runtime_pm_get_if_in_use(i915);
if (!wakeref)
flags &= ~I915_SHRINK_BOUND;
}
/*
* As we may completely rewrite the (un)bound list whilst unbinding
* (due to retiring requests) we have to strictly process only
* one element of the list at the time, and recheck the list
* on every iteration.
*
* In particular, we must hold a reference whilst removing the
* object as we may end up waiting for and/or retiring the objects.
* This might release the final reference (held by the active list)
* and result in the object being freed from under us. This is
* similar to the precautions the eviction code must take whilst
* removing objects.
*
* Also note that although these lists do not hold a reference to
* the object we can safely grab one here: The final object
* unreferencing and the bound_list are both protected by the
* dev->struct_mutex and so we won't ever be able to observe an
* object on the bound_list with a reference count equals 0.
*/
for (phase = phases; phase->list; phase++) {
struct list_head still_in_list;
struct drm_i915_gem_object *obj;
if ((flags & phase->bit) == 0)
continue;
INIT_LIST_HEAD(&still_in_list);
/*
* We serialize our access to unreferenced objects through
* the use of the struct_mutex. While the objects are not
* yet freed (due to RCU then a workqueue) we still want
* to be able to shrink their pages, so they remain on
* the unbound/bound list until actually freed.
*/
spin_lock(&i915->mm.obj_lock);
while (count < target &&
(obj = list_first_entry_or_null(phase->list,
typeof(*obj),
mm.link))) {
list_move_tail(&obj->mm.link, &still_in_list);
if (flags & I915_SHRINK_PURGEABLE &&
obj->mm.madv != I915_MADV_DONTNEED)
continue;
if (flags & I915_SHRINK_VMAPS &&
!is_vmalloc_addr(obj->mm.mapping))
continue;
if (!(flags & I915_SHRINK_ACTIVE) &&
(i915_gem_object_is_active(obj) ||
i915_gem_object_is_framebuffer(obj)))
continue;
if (!can_release_pages(obj))
continue;
spin_unlock(&i915->mm.obj_lock);
if (unsafe_drop_pages(obj)) {
/* May arrive from get_pages on another bo */
mutex_lock_nested(&obj->mm.lock,
I915_MM_SHRINKER);
if (!i915_gem_object_has_pages(obj)) {
try_to_writeback(obj, flags);
count += obj->base.size >> PAGE_SHIFT;
}
mutex_unlock(&obj->mm.lock);
}
scanned += obj->base.size >> PAGE_SHIFT;
spin_lock(&i915->mm.obj_lock);
}
list_splice_tail(&still_in_list, phase->list);
spin_unlock(&i915->mm.obj_lock);
}
if (flags & I915_SHRINK_BOUND)
intel_runtime_pm_put(i915, wakeref);
i915_retire_requests(i915);
shrinker_unlock(i915, unlock);
if (nr_scanned)
*nr_scanned += scanned;
return count;
}
/**
* i915_gem_shrink_all - Shrink buffer object caches completely
* @i915: i915 device
*
* This is a simple wraper around i915_gem_shrink() to aggressively shrink all
* caches completely. It also first waits for and retires all outstanding
* requests to also be able to release backing storage for active objects.
*
* This should only be used in code to intentionally quiescent the gpu or as a
* last-ditch effort when memory seems to have run out.
*
* Returns:
* The number of pages of backing storage actually released.
*/
unsigned long i915_gem_shrink_all(struct drm_i915_private *i915)
{
intel_wakeref_t wakeref;
unsigned long freed = 0;
with_intel_runtime_pm(i915, wakeref) {
freed = i915_gem_shrink(i915, -1UL, NULL,
I915_SHRINK_BOUND |
I915_SHRINK_UNBOUND |
I915_SHRINK_ACTIVE);
}
return freed;
}
static unsigned long
i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
{
struct drm_i915_private *i915 =
container_of(shrinker, struct drm_i915_private, mm.shrinker);
struct drm_i915_gem_object *obj;
unsigned long num_objects = 0;
unsigned long count = 0;
spin_lock(&i915->mm.obj_lock);
list_for_each_entry(obj, &i915->mm.unbound_list, mm.link)
if (can_release_pages(obj)) {
count += obj->base.size >> PAGE_SHIFT;
num_objects++;
}
list_for_each_entry(obj, &i915->mm.bound_list, mm.link)
if (!i915_gem_object_is_active(obj) && can_release_pages(obj)) {
count += obj->base.size >> PAGE_SHIFT;
num_objects++;
}
spin_unlock(&i915->mm.obj_lock);
/* Update our preferred vmscan batch size for the next pass.
* Our rough guess for an effective batch size is roughly 2
* available GEM objects worth of pages. That is we don't want
* the shrinker to fire, until it is worth the cost of freeing an
* entire GEM object.
*/
if (num_objects) {
unsigned long avg = 2 * count / num_objects;
i915->mm.shrinker.batch =
max((i915->mm.shrinker.batch + avg) >> 1,
128ul /* default SHRINK_BATCH */);
}
return count;
}
static unsigned long
i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
{
struct drm_i915_private *i915 =
container_of(shrinker, struct drm_i915_private, mm.shrinker);
unsigned long freed;
bool unlock;
sc->nr_scanned = 0;
if (!shrinker_lock(i915, 0, &unlock))
return SHRINK_STOP;
freed = i915_gem_shrink(i915,
sc->nr_to_scan,
&sc->nr_scanned,
I915_SHRINK_BOUND |
I915_SHRINK_UNBOUND |
I915_SHRINK_PURGEABLE |
I915_SHRINK_WRITEBACK);
if (sc->nr_scanned < sc->nr_to_scan)
freed += i915_gem_shrink(i915,
sc->nr_to_scan - sc->nr_scanned,
&sc->nr_scanned,
I915_SHRINK_BOUND |
I915_SHRINK_UNBOUND |
I915_SHRINK_WRITEBACK);
if (sc->nr_scanned < sc->nr_to_scan && current_is_kswapd()) {
intel_wakeref_t wakeref;
with_intel_runtime_pm(i915, wakeref) {
freed += i915_gem_shrink(i915,
sc->nr_to_scan - sc->nr_scanned,
&sc->nr_scanned,
I915_SHRINK_ACTIVE |
I915_SHRINK_BOUND |
I915_SHRINK_UNBOUND |
I915_SHRINK_WRITEBACK);
}
}
shrinker_unlock(i915, unlock);
return sc->nr_scanned ? freed : SHRINK_STOP;
}
static int
i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
{
struct drm_i915_private *i915 =
container_of(nb, struct drm_i915_private, mm.oom_notifier);
struct drm_i915_gem_object *obj;
unsigned long unevictable, bound, unbound, freed_pages;
intel_wakeref_t wakeref;
freed_pages = 0;
with_intel_runtime_pm(i915, wakeref)
freed_pages += i915_gem_shrink(i915, -1UL, NULL,
I915_SHRINK_BOUND |
I915_SHRINK_UNBOUND |
I915_SHRINK_WRITEBACK);
/* Because we may be allocating inside our own driver, we cannot
* assert that there are no objects with pinned pages that are not
* being pointed to by hardware.
*/
unbound = bound = unevictable = 0;
spin_lock(&i915->mm.obj_lock);
list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) {
if (!can_release_pages(obj))
unevictable += obj->base.size >> PAGE_SHIFT;
else
unbound += obj->base.size >> PAGE_SHIFT;
}
list_for_each_entry(obj, &i915->mm.bound_list, mm.link) {
if (!can_release_pages(obj))
unevictable += obj->base.size >> PAGE_SHIFT;
else
bound += obj->base.size >> PAGE_SHIFT;
}
spin_unlock(&i915->mm.obj_lock);
if (freed_pages || unbound || bound)
pr_info("Purging GPU memory, %lu pages freed, "
"%lu pages still pinned.\n",
freed_pages, unevictable);
*(unsigned long *)ptr += freed_pages;
return NOTIFY_DONE;
}
static int
i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
{
struct drm_i915_private *i915 =
container_of(nb, struct drm_i915_private, mm.vmap_notifier);
struct i915_vma *vma, *next;
unsigned long freed_pages = 0;
intel_wakeref_t wakeref;
bool unlock;
if (!shrinker_lock(i915, 0, &unlock))
return NOTIFY_DONE;
/* Force everything onto the inactive lists */
if (i915_gem_wait_for_idle(i915,
I915_WAIT_LOCKED,
MAX_SCHEDULE_TIMEOUT))
goto out;
with_intel_runtime_pm(i915, wakeref)
freed_pages += i915_gem_shrink(i915, -1UL, NULL,
I915_SHRINK_BOUND |
I915_SHRINK_UNBOUND |
I915_SHRINK_VMAPS);
/* We also want to clear any cached iomaps as they wrap vmap */
mutex_lock(&i915->ggtt.vm.mutex);
list_for_each_entry_safe(vma, next,
&i915->ggtt.vm.bound_list, vm_link) {
unsigned long count = vma->node.size >> PAGE_SHIFT;
if (!vma->iomap || i915_vma_is_active(vma))
continue;
mutex_unlock(&i915->ggtt.vm.mutex);
if (i915_vma_unbind(vma) == 0)
freed_pages += count;
mutex_lock(&i915->ggtt.vm.mutex);
}
mutex_unlock(&i915->ggtt.vm.mutex);
out:
shrinker_unlock(i915, unlock);
*(unsigned long *)ptr += freed_pages;
return NOTIFY_DONE;
}
/**
* i915_gem_shrinker_register - Register the i915 shrinker
* @i915: i915 device
*
* This function registers and sets up the i915 shrinker and OOM handler.
*/
void i915_gem_shrinker_register(struct drm_i915_private *i915)
{
i915->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
i915->mm.shrinker.count_objects = i915_gem_shrinker_count;
i915->mm.shrinker.seeks = DEFAULT_SEEKS;
i915->mm.shrinker.batch = 4096;
WARN_ON(register_shrinker(&i915->mm.shrinker));
i915->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
WARN_ON(register_oom_notifier(&i915->mm.oom_notifier));
i915->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
WARN_ON(register_vmap_purge_notifier(&i915->mm.vmap_notifier));
}
/**
* i915_gem_shrinker_unregister - Unregisters the i915 shrinker
* @i915: i915 device
*
* This function unregisters the i915 shrinker and OOM handler.
*/
void i915_gem_shrinker_unregister(struct drm_i915_private *i915)
{
WARN_ON(unregister_vmap_purge_notifier(&i915->mm.vmap_notifier));
WARN_ON(unregister_oom_notifier(&i915->mm.oom_notifier));
unregister_shrinker(&i915->mm.shrinker);
}
void i915_gem_shrinker_taints_mutex(struct drm_i915_private *i915,
struct mutex *mutex)
{
bool unlock = false;
if (!IS_ENABLED(CONFIG_LOCKDEP))
return;
if (!lockdep_is_held_type(&i915->drm.struct_mutex, -1)) {
mutex_acquire(&i915->drm.struct_mutex.dep_map,
I915_MM_NORMAL, 0, _RET_IP_);
unlock = true;
}
fs_reclaim_acquire(GFP_KERNEL);
/*
* As we invariably rely on the struct_mutex within the shrinker,
* but have a complicated recursion dance, taint all the mutexes used
* within the shrinker with the struct_mutex. For completeness, we
* taint with all subclass of struct_mutex, even though we should
* only need tainting by I915_MM_NORMAL to catch possible ABBA
* deadlocks from using struct_mutex inside @mutex.
*/
mutex_acquire(&i915->drm.struct_mutex.dep_map,
I915_MM_SHRINKER, 0, _RET_IP_);
mutex_acquire(&mutex->dep_map, 0, 0, _RET_IP_);
mutex_release(&mutex->dep_map, 0, _RET_IP_);
mutex_release(&i915->drm.struct_mutex.dep_map, 0, _RET_IP_);
fs_reclaim_release(GFP_KERNEL);
if (unlock)
mutex_release(&i915->drm.struct_mutex.dep_map, 0, _RET_IP_);
}

704
drivers/gpu/drm/i915/gem/i915_gem_stolen.c Normal file
파일 보기

@@ -0,0 +1,704 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2008-2012 Intel Corporation
*/
#include <linux/errno.h>
#include <linux/mutex.h>
#include <drm/drm_mm.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"
/*
* The BIOS typically reserves some of the system's memory for the exclusive
* use of the integrated graphics. This memory is no longer available for
* use by the OS and so the user finds that his system has less memory
* available than he put in. We refer to this memory as stolen.
*
* The BIOS will allocate its framebuffer from the stolen memory. Our
* goal is try to reuse that object for our own fbcon which must always
* be available for panics. Anything else we can reuse the stolen memory
* for is a boon.
*/
int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
struct drm_mm_node *node, u64 size,
unsigned alignment, u64 start, u64 end)
{
int ret;
if (!drm_mm_initialized(&dev_priv->mm.stolen))
return -ENODEV;
/* WaSkipStolenMemoryFirstPage:bdw+ */
if (INTEL_GEN(dev_priv) >= 8 && start < 4096)
start = 4096;
mutex_lock(&dev_priv->mm.stolen_lock);
ret = drm_mm_insert_node_in_range(&dev_priv->mm.stolen, node,
size, alignment, 0,
start, end, DRM_MM_INSERT_BEST);
mutex_unlock(&dev_priv->mm.stolen_lock);
return ret;
}
int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv,
struct drm_mm_node *node, u64 size,
unsigned alignment)
{
return i915_gem_stolen_insert_node_in_range(dev_priv, node, size,
alignment, 0, U64_MAX);
}
void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
struct drm_mm_node *node)
{
mutex_lock(&dev_priv->mm.stolen_lock);
drm_mm_remove_node(node);
mutex_unlock(&dev_priv->mm.stolen_lock);
}
static int i915_adjust_stolen(struct drm_i915_private *dev_priv,
struct resource *dsm)
{
struct i915_ggtt *ggtt = &dev_priv->ggtt;
struct resource *r;
if (dsm->start == 0 || dsm->end <= dsm->start)
return -EINVAL;
/*
* TODO: We have yet too encounter the case where the GTT wasn't at the
* end of stolen. With that assumption we could simplify this.
*/
/* Make sure we don't clobber the GTT if it's within stolen memory */
if (INTEL_GEN(dev_priv) <= 4 &&
!IS_G33(dev_priv) && !IS_PINEVIEW(dev_priv) && !IS_G4X(dev_priv)) {
struct resource stolen[2] = {*dsm, *dsm};
struct resource ggtt_res;
resource_size_t ggtt_start;
ggtt_start = I915_READ(PGTBL_CTL);
if (IS_GEN(dev_priv, 4))
ggtt_start = (ggtt_start & PGTBL_ADDRESS_LO_MASK) |
(ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28;
else
ggtt_start &= PGTBL_ADDRESS_LO_MASK;
ggtt_res =
(struct resource) DEFINE_RES_MEM(ggtt_start,
ggtt_total_entries(ggtt) * 4);
if (ggtt_res.start >= stolen[0].start && ggtt_res.start < stolen[0].end)
stolen[0].end = ggtt_res.start;
if (ggtt_res.end > stolen[1].start && ggtt_res.end <= stolen[1].end)
stolen[1].start = ggtt_res.end;
/* Pick the larger of the two chunks */
if (resource_size(&stolen[0]) > resource_size(&stolen[1]))
*dsm = stolen[0];
else
*dsm = stolen[1];
if (stolen[0].start != stolen[1].start ||
stolen[0].end != stolen[1].end) {
DRM_DEBUG_DRIVER("GTT within stolen memory at %pR\n", &ggtt_res);
DRM_DEBUG_DRIVER("Stolen memory adjusted to %pR\n", dsm);
}
}
/*
* Verify that nothing else uses this physical address. Stolen
* memory should be reserved by the BIOS and hidden from the
* kernel. So if the region is already marked as busy, something
* is seriously wrong.
*/
r = devm_request_mem_region(dev_priv->drm.dev, dsm->start,
resource_size(dsm),
"Graphics Stolen Memory");
if (r == NULL) {
/*
* One more attempt but this time requesting region from
* start + 1, as we have seen that this resolves the region
* conflict with the PCI Bus.
* This is a BIOS w/a: Some BIOS wrap stolen in the root
* PCI bus, but have an off-by-one error. Hence retry the
* reservation starting from 1 instead of 0.
* There's also BIOS with off-by-one on the other end.
*/
r = devm_request_mem_region(dev_priv->drm.dev, dsm->start + 1,
resource_size(dsm) - 2,
"Graphics Stolen Memory");
/*
* GEN3 firmware likes to smash pci bridges into the stolen
* range. Apparently this works.
*/
if (r == NULL && !IS_GEN(dev_priv, 3)) {
DRM_ERROR("conflict detected with stolen region: %pR\n",
dsm);
return -EBUSY;
}
}
return 0;
}
void i915_gem_cleanup_stolen(struct drm_i915_private *dev_priv)
{
if (!drm_mm_initialized(&dev_priv->mm.stolen))
return;
drm_mm_takedown(&dev_priv->mm.stolen);
}
static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
resource_size_t *base,
resource_size_t *size)
{
u32 reg_val = I915_READ(IS_GM45(dev_priv) ?
CTG_STOLEN_RESERVED :
ELK_STOLEN_RESERVED);
resource_size_t stolen_top = dev_priv->dsm.end + 1;
DRM_DEBUG_DRIVER("%s_STOLEN_RESERVED = %08x\n",
IS_GM45(dev_priv) ? "CTG" : "ELK", reg_val);
if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0)
return;
/*
* Whether ILK really reuses the ELK register for this is unclear.
* Let's see if we catch anyone with this supposedly enabled on ILK.
*/
WARN(IS_GEN(dev_priv, 5), "ILK stolen reserved found? 0x%08x\n",
reg_val);
if (!(reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK))
return;
*base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
WARN_ON((reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);
*size = stolen_top - *base;
}
static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
resource_size_t *base,
resource_size_t *size)
{
u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
return;
*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
switch (reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK) {
case GEN6_STOLEN_RESERVED_1M:
*size = 1024 * 1024;
break;
case GEN6_STOLEN_RESERVED_512K:
*size = 512 * 1024;
break;
case GEN6_STOLEN_RESERVED_256K:
*size = 256 * 1024;
break;
case GEN6_STOLEN_RESERVED_128K:
*size = 128 * 1024;
break;
default:
*size = 1024 * 1024;
MISSING_CASE(reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK);
}
}
static void vlv_get_stolen_reserved(struct drm_i915_private *dev_priv,
resource_size_t *base,
resource_size_t *size)
{
u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
resource_size_t stolen_top = dev_priv->dsm.end + 1;
DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
return;
switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
default:
MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
/* fall through */
case GEN7_STOLEN_RESERVED_1M:
*size = 1024 * 1024;
break;
}
/*
* On vlv, the ADDR_MASK portion is left as 0 and HW deduces the
* reserved location as (top - size).
*/
*base = stolen_top - *size;
}
static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
resource_size_t *base,
resource_size_t *size)
{
u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
return;
*base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;
switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
case GEN7_STOLEN_RESERVED_1M:
*size = 1024 * 1024;
break;
case GEN7_STOLEN_RESERVED_256K:
*size = 256 * 1024;
break;
default:
*size = 1024 * 1024;
MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
}
}
static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
resource_size_t *base,
resource_size_t *size)
{
u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
return;
*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
case GEN8_STOLEN_RESERVED_1M:
*size = 1024 * 1024;
break;
case GEN8_STOLEN_RESERVED_2M:
*size = 2 * 1024 * 1024;
break;
case GEN8_STOLEN_RESERVED_4M:
*size = 4 * 1024 * 1024;
break;
case GEN8_STOLEN_RESERVED_8M:
*size = 8 * 1024 * 1024;
break;
default:
*size = 8 * 1024 * 1024;
MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
}
}
static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
resource_size_t *base,
resource_size_t *size)
{
u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
resource_size_t stolen_top = dev_priv->dsm.end + 1;
DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
return;
if (!(reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK))
return;
*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
*size = stolen_top - *base;
}
static void icl_get_stolen_reserved(struct drm_i915_private *dev_priv,
resource_size_t *base,
resource_size_t *size)
{
u64 reg_val = I915_READ64(GEN6_STOLEN_RESERVED);
DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = 0x%016llx\n", reg_val);
*base = reg_val & GEN11_STOLEN_RESERVED_ADDR_MASK;
switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
case GEN8_STOLEN_RESERVED_1M:
*size = 1024 * 1024;
break;
case GEN8_STOLEN_RESERVED_2M:
*size = 2 * 1024 * 1024;
break;
case GEN8_STOLEN_RESERVED_4M:
*size = 4 * 1024 * 1024;
break;
case GEN8_STOLEN_RESERVED_8M:
*size = 8 * 1024 * 1024;
break;
default:
*size = 8 * 1024 * 1024;
MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
}
}
int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
{
resource_size_t reserved_base, stolen_top;
resource_size_t reserved_total, reserved_size;
mutex_init(&dev_priv->mm.stolen_lock);
if (intel_vgpu_active(dev_priv)) {
DRM_INFO("iGVT-g active, disabling use of stolen memory\n");
return 0;
}
if (intel_vtd_active() && INTEL_GEN(dev_priv) < 8) {
DRM_INFO("DMAR active, disabling use of stolen memory\n");
return 0;
}
if (resource_size(&intel_graphics_stolen_res) == 0)
return 0;
dev_priv->dsm = intel_graphics_stolen_res;
if (i915_adjust_stolen(dev_priv, &dev_priv->dsm))
return 0;
GEM_BUG_ON(dev_priv->dsm.start == 0);
GEM_BUG_ON(dev_priv->dsm.end <= dev_priv->dsm.start);
stolen_top = dev_priv->dsm.end + 1;
reserved_base = stolen_top;
reserved_size = 0;
switch (INTEL_GEN(dev_priv)) {
case 2:
case 3:
break;
case 4:
if (!IS_G4X(dev_priv))
break;
/* fall through */
case 5:
g4x_get_stolen_reserved(dev_priv,
&reserved_base, &reserved_size);
break;
case 6:
gen6_get_stolen_reserved(dev_priv,
&reserved_base, &reserved_size);
break;
case 7:
if (IS_VALLEYVIEW(dev_priv))
vlv_get_stolen_reserved(dev_priv,
&reserved_base, &reserved_size);
else
gen7_get_stolen_reserved(dev_priv,
&reserved_base, &reserved_size);
break;
case 8:
case 9:
case 10:
if (IS_LP(dev_priv))
chv_get_stolen_reserved(dev_priv,
&reserved_base, &reserved_size);
else
bdw_get_stolen_reserved(dev_priv,
&reserved_base, &reserved_size);
break;
case 11:
default:
icl_get_stolen_reserved(dev_priv, &reserved_base,
&reserved_size);
break;
}
/*
* Our expectation is that the reserved space is at the top of the
* stolen region and *never* at the bottom. If we see !reserved_base,
* it likely means we failed to read the registers correctly.
*/
if (!reserved_base) {
DRM_ERROR("inconsistent reservation %pa + %pa; ignoring\n",
&reserved_base, &reserved_size);
reserved_base = stolen_top;
reserved_size = 0;
}
dev_priv->dsm_reserved =
(struct resource) DEFINE_RES_MEM(reserved_base, reserved_size);
if (!resource_contains(&dev_priv->dsm, &dev_priv->dsm_reserved)) {
DRM_ERROR("Stolen reserved area %pR outside stolen memory %pR\n",
&dev_priv->dsm_reserved, &dev_priv->dsm);
return 0;
}
/* It is possible for the reserved area to end before the end of stolen
* memory, so just consider the start. */
reserved_total = stolen_top - reserved_base;
DRM_DEBUG_DRIVER("Memory reserved for graphics device: %lluK, usable: %lluK\n",
(u64)resource_size(&dev_priv->dsm) >> 10,
((u64)resource_size(&dev_priv->dsm) - reserved_total) >> 10);
dev_priv->stolen_usable_size =
resource_size(&dev_priv->dsm) - reserved_total;
/* Basic memrange allocator for stolen space. */
drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->stolen_usable_size);
return 0;
}
static struct sg_table *
i915_pages_create_for_stolen(struct drm_device *dev,
resource_size_t offset, resource_size_t size)
{
struct drm_i915_private *dev_priv = to_i915(dev);
struct sg_table *st;
struct scatterlist *sg;
GEM_BUG_ON(range_overflows(offset, size, resource_size(&dev_priv->dsm)));
/* We hide that we have no struct page backing our stolen object
* by wrapping the contiguous physical allocation with a fake
* dma mapping in a single scatterlist.
*/
st = kmalloc(sizeof(*st), GFP_KERNEL);
if (st == NULL)
return ERR_PTR(-ENOMEM);
if (sg_alloc_table(st, 1, GFP_KERNEL)) {
kfree(st);
return ERR_PTR(-ENOMEM);
}
sg = st->sgl;
sg->offset = 0;
sg->length = size;
sg_dma_address(sg) = (dma_addr_t)dev_priv->dsm.start + offset;
sg_dma_len(sg) = size;
return st;
}
static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
{
struct sg_table *pages =
i915_pages_create_for_stolen(obj->base.dev,
obj->stolen->start,
obj->stolen->size);
if (IS_ERR(pages))
return PTR_ERR(pages);
__i915_gem_object_set_pages(obj, pages, obj->stolen->size);
return 0;
}
static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj,
struct sg_table *pages)
{
/* Should only be called from i915_gem_object_release_stolen() */
sg_free_table(pages);
kfree(pages);
}
static void
i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
{
struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen);
GEM_BUG_ON(!stolen);
__i915_gem_object_unpin_pages(obj);
i915_gem_stolen_remove_node(dev_priv, stolen);
kfree(stolen);
}
static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = {
.get_pages = i915_gem_object_get_pages_stolen,
.put_pages = i915_gem_object_put_pages_stolen,
.release = i915_gem_object_release_stolen,
};
static struct drm_i915_gem_object *
_i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
struct drm_mm_node *stolen)
{
struct drm_i915_gem_object *obj;
unsigned int cache_level;
obj = i915_gem_object_alloc();
if (obj == NULL)
return NULL;
drm_gem_private_object_init(&dev_priv->drm, &obj->base, stolen->size);
i915_gem_object_init(obj, &i915_gem_object_stolen_ops);
obj->stolen = stolen;
obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
cache_level = HAS_LLC(dev_priv) ? I915_CACHE_LLC : I915_CACHE_NONE;
i915_gem_object_set_cache_coherency(obj, cache_level);
if (i915_gem_object_pin_pages(obj))
goto cleanup;
return obj;
cleanup:
i915_gem_object_free(obj);
return NULL;
}
struct drm_i915_gem_object *
i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
resource_size_t size)
{
struct drm_i915_gem_object *obj;
struct drm_mm_node *stolen;
int ret;
if (!drm_mm_initialized(&dev_priv->mm.stolen))
return NULL;
if (size == 0)
return NULL;
stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
if (!stolen)
return NULL;
ret = i915_gem_stolen_insert_node(dev_priv, stolen, size, 4096);
if (ret) {
kfree(stolen);
return NULL;
}
obj = _i915_gem_object_create_stolen(dev_priv, stolen);
if (obj)
return obj;
i915_gem_stolen_remove_node(dev_priv, stolen);
kfree(stolen);
return NULL;
}
struct drm_i915_gem_object *
i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv,
resource_size_t stolen_offset,
resource_size_t gtt_offset,
resource_size_t size)
{
struct i915_ggtt *ggtt = &dev_priv->ggtt;
struct drm_i915_gem_object *obj;
struct drm_mm_node *stolen;
struct i915_vma *vma;
int ret;
if (!drm_mm_initialized(&dev_priv->mm.stolen))
return NULL;
lockdep_assert_held(&dev_priv->drm.struct_mutex);
DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
&stolen_offset, &gtt_offset, &size);
/* KISS and expect everything to be page-aligned */
if (WARN_ON(size == 0) ||
WARN_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)) ||
WARN_ON(!IS_ALIGNED(stolen_offset, I915_GTT_MIN_ALIGNMENT)))
return NULL;
stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
if (!stolen)
return NULL;
stolen->start = stolen_offset;
stolen->size = size;
mutex_lock(&dev_priv->mm.stolen_lock);
ret = drm_mm_reserve_node(&dev_priv->mm.stolen, stolen);
mutex_unlock(&dev_priv->mm.stolen_lock);
if (ret) {
DRM_DEBUG_DRIVER("failed to allocate stolen space\n");
kfree(stolen);
return NULL;
}
obj = _i915_gem_object_create_stolen(dev_priv, stolen);
if (obj == NULL) {
DRM_DEBUG_DRIVER("failed to allocate stolen object\n");
i915_gem_stolen_remove_node(dev_priv, stolen);
kfree(stolen);
return NULL;
}
/* Some objects just need physical mem from stolen space */
if (gtt_offset == I915_GTT_OFFSET_NONE)
return obj;
ret = i915_gem_object_pin_pages(obj);
if (ret)
goto err;
vma = i915_vma_instance(obj, &ggtt->vm, NULL);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto err_pages;
}
/* To simplify the initialisation sequence between KMS and GTT,
* we allow construction of the stolen object prior to
* setting up the GTT space. The actual reservation will occur
* later.
*/
ret = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
size, gtt_offset, obj->cache_level,
0);
if (ret) {
DRM_DEBUG_DRIVER("failed to allocate stolen GTT space\n");
goto err_pages;
}
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
vma->pages = obj->mm.pages;
vma->flags |= I915_VMA_GLOBAL_BIND;
__i915_vma_set_map_and_fenceable(vma);
mutex_lock(&ggtt->vm.mutex);
list_move_tail(&vma->vm_link, &ggtt->vm.bound_list);
mutex_unlock(&ggtt->vm.mutex);
spin_lock(&dev_priv->mm.obj_lock);
list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list);
obj->bind_count++;
spin_unlock(&dev_priv->mm.obj_lock);
return obj;
err_pages:
i915_gem_object_unpin_pages(obj);
err:
i915_gem_object_put(obj);
return NULL;
}

440
drivers/gpu/drm/i915/gem/i915_gem_tiling.c Normal file
파일 보기

@@ -0,0 +1,440 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2008 Intel Corporation
*/
#include <linux/string.h>
#include <linux/bitops.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"
#include "i915_gem.h"
#include "i915_gem_ioctls.h"
#include "i915_gem_object.h"
/**
* DOC: buffer object tiling
*
* i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace
* interface to declare fence register requirements.
*
* In principle GEM doesn't care at all about the internal data layout of an
* object, and hence it also doesn't care about tiling or swizzling. There's two
* exceptions:
*
* - For X and Y tiling the hardware provides detilers for CPU access, so called
* fences. Since there's only a limited amount of them the kernel must manage
* these, and therefore userspace must tell the kernel the object tiling if it
* wants to use fences for detiling.
* - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which
* depends upon the physical page frame number. When swapping such objects the
* page frame number might change and the kernel must be able to fix this up
* and hence now the tiling. Note that on a subset of platforms with
* asymmetric memory channel population the swizzling pattern changes in an
* unknown way, and for those the kernel simply forbids swapping completely.
*
* Since neither of this applies for new tiling layouts on modern platforms like
* W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled.
* Anything else can be handled in userspace entirely without the kernel's
* invovlement.
*/
/**
* i915_gem_fence_size - required global GTT size for a fence
* @i915: i915 device
* @size: object size
* @tiling: tiling mode
* @stride: tiling stride
*
* Return the required global GTT size for a fence (view of a tiled object),
* taking into account potential fence register mapping.
*/
u32 i915_gem_fence_size(struct drm_i915_private *i915,
u32 size, unsigned int tiling, unsigned int stride)
{
u32 ggtt_size;
GEM_BUG_ON(!size);
if (tiling == I915_TILING_NONE)
return size;
GEM_BUG_ON(!stride);
if (INTEL_GEN(i915) >= 4) {
stride *= i915_gem_tile_height(tiling);
GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE));
return roundup(size, stride);
}
/* Previous chips need a power-of-two fence region when tiling */
if (IS_GEN(i915, 3))
ggtt_size = 1024*1024;
else
ggtt_size = 512*1024;
while (ggtt_size < size)
ggtt_size <<= 1;
return ggtt_size;
}
/**
* i915_gem_fence_alignment - required global GTT alignment for a fence
* @i915: i915 device
* @size: object size
* @tiling: tiling mode
* @stride: tiling stride
*
* Return the required global GTT alignment for a fence (a view of a tiled
* object), taking into account potential fence register mapping.
*/
u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size,
unsigned int tiling, unsigned int stride)
{
GEM_BUG_ON(!size);
/*
* Minimum alignment is 4k (GTT page size), but might be greater
* if a fence register is needed for the object.
*/
if (tiling == I915_TILING_NONE)
return I915_GTT_MIN_ALIGNMENT;
if (INTEL_GEN(i915) >= 4)
return I965_FENCE_PAGE;
/*
* Previous chips need to be aligned to the size of the smallest
* fence register that can contain the object.
*/
return i915_gem_fence_size(i915, size, tiling, stride);
}
/* Check pitch constriants for all chips & tiling formats */
static bool
i915_tiling_ok(struct drm_i915_gem_object *obj,
unsigned int tiling, unsigned int stride)
{
struct drm_i915_private *i915 = to_i915(obj->base.dev);
unsigned int tile_width;
/* Linear is always fine */
if (tiling == I915_TILING_NONE)
return true;
if (tiling > I915_TILING_LAST)
return false;
/* check maximum stride & object size */
/* i965+ stores the end address of the gtt mapping in the fence
* reg, so dont bother to check the size */
if (INTEL_GEN(i915) >= 7) {
if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)
return false;
} else if (INTEL_GEN(i915) >= 4) {
if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
return false;
} else {
if (stride > 8192)
return false;
if (!is_power_of_2(stride))
return false;
}
if (IS_GEN(i915, 2) ||
(tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915)))
tile_width = 128;
else
tile_width = 512;
if (!stride || !IS_ALIGNED(stride, tile_width))
return false;
return true;
}
static bool i915_vma_fence_prepare(struct i915_vma *vma,
int tiling_mode, unsigned int stride)
{
struct drm_i915_private *i915 = vma->vm->i915;
u32 size, alignment;
if (!i915_vma_is_map_and_fenceable(vma))
return true;
size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride);
if (vma->node.size < size)
return false;
alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride);
if (!IS_ALIGNED(vma->node.start, alignment))
return false;
return true;
}
/* Make the current GTT allocation valid for the change in tiling. */
static int
i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
int tiling_mode, unsigned int stride)
{
struct i915_vma *vma;
int ret;
if (tiling_mode == I915_TILING_NONE)
return 0;
for_each_ggtt_vma(vma, obj) {
if (i915_vma_fence_prepare(vma, tiling_mode, stride))
continue;
ret = i915_vma_unbind(vma);
if (ret)
return ret;
}
return 0;
}
int
i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
unsigned int tiling, unsigned int stride)
{
struct drm_i915_private *i915 = to_i915(obj->base.dev);
struct i915_vma *vma;
int err;
/* Make sure we don't cross-contaminate obj->tiling_and_stride */
BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK);
GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));
GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));
lockdep_assert_held(&i915->drm.struct_mutex);
if ((tiling | stride) == obj->tiling_and_stride)
return 0;
if (i915_gem_object_is_framebuffer(obj))
return -EBUSY;
/* We need to rebind the object if its current allocation
* no longer meets the alignment restrictions for its new
* tiling mode. Otherwise we can just leave it alone, but
* need to ensure that any fence register is updated before
* the next fenced (either through the GTT or by the BLT unit
* on older GPUs) access.
*
* After updating the tiling parameters, we then flag whether
* we need to update an associated fence register. Note this
* has to also include the unfenced register the GPU uses
* whilst executing a fenced command for an untiled object.
*/
err = i915_gem_object_fence_prepare(obj, tiling, stride);
if (err)
return err;
i915_gem_object_lock(obj);
if (i915_gem_object_is_framebuffer(obj)) {
i915_gem_object_unlock(obj);
return -EBUSY;
}
/* If the memory has unknown (i.e. varying) swizzling, we pin the
* pages to prevent them being swapped out and causing corruption
* due to the change in swizzling.
*/
mutex_lock(&obj->mm.lock);
if (i915_gem_object_has_pages(obj) &&
obj->mm.madv == I915_MADV_WILLNEED &&
i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
if (tiling == I915_TILING_NONE) {
GEM_BUG_ON(!obj->mm.quirked);
__i915_gem_object_unpin_pages(obj);
obj->mm.quirked = false;
}
if (!i915_gem_object_is_tiled(obj)) {
GEM_BUG_ON(obj->mm.quirked);
__i915_gem_object_pin_pages(obj);
obj->mm.quirked = true;
}
}
mutex_unlock(&obj->mm.lock);
for_each_ggtt_vma(vma, obj) {
vma->fence_size =
i915_gem_fence_size(i915, vma->size, tiling, stride);
vma->fence_alignment =
i915_gem_fence_alignment(i915,
vma->size, tiling, stride);
if (vma->fence)
vma->fence->dirty = true;
}
obj->tiling_and_stride = tiling | stride;
i915_gem_object_unlock(obj);
/* Force the fence to be reacquired for GTT access */
i915_gem_object_release_mmap(obj);
/* Try to preallocate memory required to save swizzling on put-pages */
if (i915_gem_object_needs_bit17_swizzle(obj)) {
if (!obj->bit_17) {
obj->bit_17 = bitmap_zalloc(obj->base.size >> PAGE_SHIFT,
GFP_KERNEL);
}
} else {
bitmap_free(obj->bit_17);
obj->bit_17 = NULL;
}
return 0;
}
/**
* i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode
* @dev: DRM device
* @data: data pointer for the ioctl
* @file: DRM file for the ioctl call
*
* Sets the tiling mode of an object, returning the required swizzling of
* bit 6 of addresses in the object.
*
* Called by the user via ioctl.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int
i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
struct drm_file *file)
{
struct drm_i915_gem_set_tiling *args = data;
struct drm_i915_gem_object *obj;
int err;
obj = i915_gem_object_lookup(file, args->handle);
if (!obj)
return -ENOENT;
/*
* The tiling mode of proxy objects is handled by its generator, and
* not allowed to be changed by userspace.
*/
if (i915_gem_object_is_proxy(obj)) {
err = -ENXIO;
goto err;
}
if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {
err = -EINVAL;
goto err;
}
if (args->tiling_mode == I915_TILING_NONE) {
args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
args->stride = 0;
} else {
if (args->tiling_mode == I915_TILING_X)
args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_x;
else
args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_y;
/* Hide bit 17 swizzling from the user. This prevents old Mesa
* from aborting the application on sw fallbacks to bit 17,
* and we use the pread/pwrite bit17 paths to swizzle for it.
* If there was a user that was relying on the swizzle
* information for drm_intel_bo_map()ed reads/writes this would
* break it, but we don't have any of those.
*/
if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
/* If we can't handle the swizzling, make it untiled. */
if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
args->tiling_mode = I915_TILING_NONE;
args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
args->stride = 0;
}
}
err = mutex_lock_interruptible(&dev->struct_mutex);
if (err)
goto err;
err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);
mutex_unlock(&dev->struct_mutex);
/* We have to maintain this existing ABI... */
args->stride = i915_gem_object_get_stride(obj);
args->tiling_mode = i915_gem_object_get_tiling(obj);
err:
i915_gem_object_put(obj);
return err;
}
/**
* i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode
* @dev: DRM device
* @data: data pointer for the ioctl
* @file: DRM file for the ioctl call
*
* Returns the current tiling mode and required bit 6 swizzling for the object.
*
* Called by the user via ioctl.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int
i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
struct drm_file *file)
{
struct drm_i915_gem_get_tiling *args = data;
struct drm_i915_private *dev_priv = to_i915(dev);
struct drm_i915_gem_object *obj;
int err = -ENOENT;
rcu_read_lock();
obj = i915_gem_object_lookup_rcu(file, args->handle);
if (obj) {
args->tiling_mode =
READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
err = 0;
}
rcu_read_unlock();
if (unlikely(err))
return err;
switch (args->tiling_mode) {
case I915_TILING_X:
args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
break;
case I915_TILING_Y:
args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
break;
default:
case I915_TILING_NONE:
args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
break;
}
/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;
else
args->phys_swizzle_mode = args->swizzle_mode;
if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
return 0;
}

832
drivers/gpu/drm/i915/gem/i915_gem_userptr.c Normal file
파일 보기

@@ -0,0 +1,832 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2012-2014 Intel Corporation
*/
#include <linux/mmu_context.h>
#include <linux/mmu_notifier.h>
#include <linux/mempolicy.h>
#include <linux/swap.h>
#include <linux/sched/mm.h>
#include <drm/i915_drm.h>
#include "i915_gem_ioctls.h"
#include "i915_gem_object.h"
#include "i915_trace.h"
#include "intel_drv.h"
struct i915_mm_struct {
struct mm_struct *mm;
struct drm_i915_private *i915;
struct i915_mmu_notifier *mn;
struct hlist_node node;
struct kref kref;
struct work_struct work;
};
#if defined(CONFIG_MMU_NOTIFIER)
#include <linux/interval_tree.h>
struct i915_mmu_notifier {
spinlock_t lock;
struct hlist_node node;
struct mmu_notifier mn;
struct rb_root_cached objects;
struct i915_mm_struct *mm;
};
struct i915_mmu_object {
struct i915_mmu_notifier *mn;
struct drm_i915_gem_object *obj;
struct interval_tree_node it;
};
static void add_object(struct i915_mmu_object *mo)
{
GEM_BUG_ON(!RB_EMPTY_NODE(&mo->it.rb));
interval_tree_insert(&mo->it, &mo->mn->objects);
}
static void del_object(struct i915_mmu_object *mo)
{
if (RB_EMPTY_NODE(&mo->it.rb))
return;
interval_tree_remove(&mo->it, &mo->mn->objects);
RB_CLEAR_NODE(&mo->it.rb);
}
static void
__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value)
{
struct i915_mmu_object *mo = obj->userptr.mmu_object;
/*
* During mm_invalidate_range we need to cancel any userptr that
* overlaps the range being invalidated. Doing so requires the
* struct_mutex, and that risks recursion. In order to cause
* recursion, the user must alias the userptr address space with
* a GTT mmapping (possible with a MAP_FIXED) - then when we have
* to invalidate that mmaping, mm_invalidate_range is called with
* the userptr address *and* the struct_mutex held. To prevent that
* we set a flag under the i915_mmu_notifier spinlock to indicate
* whether this object is valid.
*/
if (!mo)
return;
spin_lock(&mo->mn->lock);
if (value)
add_object(mo);
else
del_object(mo);
spin_unlock(&mo->mn->lock);
}
static int
userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
const struct mmu_notifier_range *range)
{
struct i915_mmu_notifier *mn =
container_of(_mn, struct i915_mmu_notifier, mn);
struct interval_tree_node *it;
struct mutex *unlock = NULL;
unsigned long end;
int ret = 0;
if (RB_EMPTY_ROOT(&mn->objects.rb_root))
return 0;
/* interval ranges are inclusive, but invalidate range is exclusive */
end = range->end - 1;
spin_lock(&mn->lock);
it = interval_tree_iter_first(&mn->objects, range->start, end);
while (it) {
struct drm_i915_gem_object *obj;
if (!mmu_notifier_range_blockable(range)) {
ret = -EAGAIN;
break;
}
/*
* The mmu_object is released late when destroying the
* GEM object so it is entirely possible to gain a
* reference on an object in the process of being freed
* since our serialisation is via the spinlock and not
* the struct_mutex - and consequently use it after it
* is freed and then double free it. To prevent that
* use-after-free we only acquire a reference on the
* object if it is not in the process of being destroyed.
*/
obj = container_of(it, struct i915_mmu_object, it)->obj;
if (!kref_get_unless_zero(&obj->base.refcount)) {
it = interval_tree_iter_next(it, range->start, end);
continue;
}
spin_unlock(&mn->lock);
if (!unlock) {
unlock = &mn->mm->i915->drm.struct_mutex;
switch (mutex_trylock_recursive(unlock)) {
default:
case MUTEX_TRYLOCK_FAILED:
if (mutex_lock_killable_nested(unlock, I915_MM_SHRINKER)) {
i915_gem_object_put(obj);
return -EINTR;
}
/* fall through */
case MUTEX_TRYLOCK_SUCCESS:
break;
case MUTEX_TRYLOCK_RECURSIVE:
unlock = ERR_PTR(-EEXIST);
break;
}
}
ret = i915_gem_object_unbind(obj);
if (ret == 0)
ret = __i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
i915_gem_object_put(obj);
if (ret)
goto unlock;
spin_lock(&mn->lock);
/*
* As we do not (yet) protect the mmu from concurrent insertion
* over this range, there is no guarantee that this search will
* terminate given a pathologic workload.
*/
it = interval_tree_iter_first(&mn->objects, range->start, end);
}
spin_unlock(&mn->lock);
unlock:
if (!IS_ERR_OR_NULL(unlock))
mutex_unlock(unlock);
return ret;
}
static const struct mmu_notifier_ops i915_gem_userptr_notifier = {
.invalidate_range_start = userptr_mn_invalidate_range_start,
};
static struct i915_mmu_notifier *
i915_mmu_notifier_create(struct i915_mm_struct *mm)
{
struct i915_mmu_notifier *mn;
mn = kmalloc(sizeof(*mn), GFP_KERNEL);
if (mn == NULL)
return ERR_PTR(-ENOMEM);
spin_lock_init(&mn->lock);
mn->mn.ops = &i915_gem_userptr_notifier;
mn->objects = RB_ROOT_CACHED;
mn->mm = mm;
return mn;
}
static void
i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
{
struct i915_mmu_object *mo;
mo = fetch_and_zero(&obj->userptr.mmu_object);
if (!mo)
return;
spin_lock(&mo->mn->lock);
del_object(mo);
spin_unlock(&mo->mn->lock);
kfree(mo);
}
static struct i915_mmu_notifier *
i915_mmu_notifier_find(struct i915_mm_struct *mm)
{
struct i915_mmu_notifier *mn;
int err = 0;
mn = mm->mn;
if (mn)
return mn;
mn = i915_mmu_notifier_create(mm);
if (IS_ERR(mn))
err = PTR_ERR(mn);
down_write(&mm->mm->mmap_sem);
mutex_lock(&mm->i915->mm_lock);
if (mm->mn == NULL && !err) {
/* Protected by mmap_sem (write-lock) */
err = __mmu_notifier_register(&mn->mn, mm->mm);
if (!err) {
/* Protected by mm_lock */
mm->mn = fetch_and_zero(&mn);
}
} else if (mm->mn) {
/*
* Someone else raced and successfully installed the mmu
* notifier, we can cancel our own errors.
*/
err = 0;
}
mutex_unlock(&mm->i915->mm_lock);
up_write(&mm->mm->mmap_sem);
if (mn && !IS_ERR(mn))
kfree(mn);
return err ? ERR_PTR(err) : mm->mn;
}
static int
i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
unsigned flags)
{
struct i915_mmu_notifier *mn;
struct i915_mmu_object *mo;
if (flags & I915_USERPTR_UNSYNCHRONIZED)
return capable(CAP_SYS_ADMIN) ? 0 : -EPERM;
if (WARN_ON(obj->userptr.mm == NULL))
return -EINVAL;
mn = i915_mmu_notifier_find(obj->userptr.mm);
if (IS_ERR(mn))
return PTR_ERR(mn);
mo = kzalloc(sizeof(*mo), GFP_KERNEL);
if (!mo)
return -ENOMEM;
mo->mn = mn;
mo->obj = obj;
mo->it.start = obj->userptr.ptr;
mo->it.last = obj->userptr.ptr + obj->base.size - 1;
RB_CLEAR_NODE(&mo->it.rb);
obj->userptr.mmu_object = mo;
return 0;
}
static void
i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
struct mm_struct *mm)
{
if (mn == NULL)
return;
mmu_notifier_unregister(&mn->mn, mm);
kfree(mn);
}
#else
static void
__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value)
{
}
static void
i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
{
}
static int
i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
unsigned flags)
{
if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0)
return -ENODEV;
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
return 0;
}
static void
i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
struct mm_struct *mm)
{
}
#endif
static struct i915_mm_struct *
__i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real)
{
struct i915_mm_struct *mm;
/* Protected by dev_priv->mm_lock */
hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real)
if (mm->mm == real)
return mm;
return NULL;
}
static int
i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj)
{
struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
struct i915_mm_struct *mm;
int ret = 0;
/* During release of the GEM object we hold the struct_mutex. This
* precludes us from calling mmput() at that time as that may be
* the last reference and so call exit_mmap(). exit_mmap() will
* attempt to reap the vma, and if we were holding a GTT mmap
* would then call drm_gem_vm_close() and attempt to reacquire
* the struct mutex. So in order to avoid that recursion, we have
* to defer releasing the mm reference until after we drop the
* struct_mutex, i.e. we need to schedule a worker to do the clean
* up.
*/
mutex_lock(&dev_priv->mm_lock);
mm = __i915_mm_struct_find(dev_priv, current->mm);
if (mm == NULL) {
mm = kmalloc(sizeof(*mm), GFP_KERNEL);
if (mm == NULL) {
ret = -ENOMEM;
goto out;
}
kref_init(&mm->kref);
mm->i915 = to_i915(obj->base.dev);
mm->mm = current->mm;
mmgrab(current->mm);
mm->mn = NULL;
/* Protected by dev_priv->mm_lock */
hash_add(dev_priv->mm_structs,
&mm->node, (unsigned long)mm->mm);
} else
kref_get(&mm->kref);
obj->userptr.mm = mm;
out:
mutex_unlock(&dev_priv->mm_lock);
return ret;
}
static void
__i915_mm_struct_free__worker(struct work_struct *work)
{
struct i915_mm_struct *mm = container_of(work, typeof(*mm), work);
i915_mmu_notifier_free(mm->mn, mm->mm);
mmdrop(mm->mm);
kfree(mm);
}
static void
__i915_mm_struct_free(struct kref *kref)
{
struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref);
/* Protected by dev_priv->mm_lock */
hash_del(&mm->node);
mutex_unlock(&mm->i915->mm_lock);
INIT_WORK(&mm->work, __i915_mm_struct_free__worker);
queue_work(mm->i915->mm.userptr_wq, &mm->work);
}
static void
i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj)
{
if (obj->userptr.mm == NULL)
return;
kref_put_mutex(&obj->userptr.mm->kref,
__i915_mm_struct_free,
&to_i915(obj->base.dev)->mm_lock);
obj->userptr.mm = NULL;
}
struct get_pages_work {
struct work_struct work;
struct drm_i915_gem_object *obj;
struct task_struct *task;
};
static struct sg_table *
__i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj,
struct page **pvec, int num_pages)
{
unsigned int max_segment = i915_sg_segment_size();
struct sg_table *st;
unsigned int sg_page_sizes;
int ret;
st = kmalloc(sizeof(*st), GFP_KERNEL);
if (!st)
return ERR_PTR(-ENOMEM);
alloc_table:
ret = __sg_alloc_table_from_pages(st, pvec, num_pages,
0, num_pages << PAGE_SHIFT,
max_segment,
GFP_KERNEL);
if (ret) {
kfree(st);
return ERR_PTR(ret);
}
ret = i915_gem_gtt_prepare_pages(obj, st);
if (ret) {
sg_free_table(st);
if (max_segment > PAGE_SIZE) {
max_segment = PAGE_SIZE;
goto alloc_table;
}
kfree(st);
return ERR_PTR(ret);
}
sg_page_sizes = i915_sg_page_sizes(st->sgl);
__i915_gem_object_set_pages(obj, st, sg_page_sizes);
return st;
}
static void
__i915_gem_userptr_get_pages_worker(struct work_struct *_work)
{
struct get_pages_work *work = container_of(_work, typeof(*work), work);
struct drm_i915_gem_object *obj = work->obj;
const int npages = obj->base.size >> PAGE_SHIFT;
struct page **pvec;
int pinned, ret;
ret = -ENOMEM;
pinned = 0;
pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
if (pvec != NULL) {
struct mm_struct *mm = obj->userptr.mm->mm;
unsigned int flags = 0;
if (!i915_gem_object_is_readonly(obj))
flags |= FOLL_WRITE;
ret = -EFAULT;
if (mmget_not_zero(mm)) {
down_read(&mm->mmap_sem);
while (pinned < npages) {
ret = get_user_pages_remote
(work->task, mm,
obj->userptr.ptr + pinned * PAGE_SIZE,
npages - pinned,
flags,
pvec + pinned, NULL, NULL);
if (ret < 0)
break;
pinned += ret;
}
up_read(&mm->mmap_sem);
mmput(mm);
}
}
mutex_lock(&obj->mm.lock);
if (obj->userptr.work == &work->work) {
struct sg_table *pages = ERR_PTR(ret);
if (pinned == npages) {
pages = __i915_gem_userptr_alloc_pages(obj, pvec,
npages);
if (!IS_ERR(pages)) {
pinned = 0;
pages = NULL;
}
}
obj->userptr.work = ERR_CAST(pages);
if (IS_ERR(pages))
__i915_gem_userptr_set_active(obj, false);
}
mutex_unlock(&obj->mm.lock);
release_pages(pvec, pinned);
kvfree(pvec);
i915_gem_object_put(obj);
put_task_struct(work->task);
kfree(work);
}
static struct sg_table *
__i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj)
{
struct get_pages_work *work;
/* Spawn a worker so that we can acquire the
* user pages without holding our mutex. Access
* to the user pages requires mmap_sem, and we have
* a strict lock ordering of mmap_sem, struct_mutex -
* we already hold struct_mutex here and so cannot
* call gup without encountering a lock inversion.
*
* Userspace will keep on repeating the operation
* (thanks to EAGAIN) until either we hit the fast
* path or the worker completes. If the worker is
* cancelled or superseded, the task is still run
* but the results ignored. (This leads to
* complications that we may have a stray object
* refcount that we need to be wary of when
* checking for existing objects during creation.)
* If the worker encounters an error, it reports
* that error back to this function through
* obj->userptr.work = ERR_PTR.
*/
work = kmalloc(sizeof(*work), GFP_KERNEL);
if (work == NULL)
return ERR_PTR(-ENOMEM);
obj->userptr.work = &work->work;
work->obj = i915_gem_object_get(obj);
work->task = current;
get_task_struct(work->task);
INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker);
queue_work(to_i915(obj->base.dev)->mm.userptr_wq, &work->work);
return ERR_PTR(-EAGAIN);
}
static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
{
const int num_pages = obj->base.size >> PAGE_SHIFT;
struct mm_struct *mm = obj->userptr.mm->mm;
struct page **pvec;
struct sg_table *pages;
bool active;
int pinned;
/* If userspace should engineer that these pages are replaced in
* the vma between us binding this page into the GTT and completion
* of rendering... Their loss. If they change the mapping of their
* pages they need to create a new bo to point to the new vma.
*
* However, that still leaves open the possibility of the vma
* being copied upon fork. Which falls under the same userspace
* synchronisation issue as a regular bo, except that this time
* the process may not be expecting that a particular piece of
* memory is tied to the GPU.
*
* Fortunately, we can hook into the mmu_notifier in order to
* discard the page references prior to anything nasty happening
* to the vma (discard or cloning) which should prevent the more
* egregious cases from causing harm.
*/
if (obj->userptr.work) {
/* active flag should still be held for the pending work */
if (IS_ERR(obj->userptr.work))
return PTR_ERR(obj->userptr.work);
else
return -EAGAIN;
}
pvec = NULL;
pinned = 0;
if (mm == current->mm) {
pvec = kvmalloc_array(num_pages, sizeof(struct page *),
GFP_KERNEL |
__GFP_NORETRY |
__GFP_NOWARN);
if (pvec) /* defer to worker if malloc fails */
pinned = __get_user_pages_fast(obj->userptr.ptr,
num_pages,
!i915_gem_object_is_readonly(obj),
pvec);
}
active = false;
if (pinned < 0) {
pages = ERR_PTR(pinned);
pinned = 0;
} else if (pinned < num_pages) {
pages = __i915_gem_userptr_get_pages_schedule(obj);
active = pages == ERR_PTR(-EAGAIN);
} else {
pages = __i915_gem_userptr_alloc_pages(obj, pvec, num_pages);
active = !IS_ERR(pages);
}
if (active)
__i915_gem_userptr_set_active(obj, true);
if (IS_ERR(pages))
release_pages(pvec, pinned);
kvfree(pvec);
return PTR_ERR_OR_ZERO(pages);
}
static void
i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj,
struct sg_table *pages)
{
struct sgt_iter sgt_iter;
struct page *page;
/* Cancel any inflight work and force them to restart their gup */
obj->userptr.work = NULL;
__i915_gem_userptr_set_active(obj, false);
if (!pages)
return;
__i915_gem_object_release_shmem(obj, pages, true);
i915_gem_gtt_finish_pages(obj, pages);
for_each_sgt_page(page, sgt_iter, pages) {
if (obj->mm.dirty)
set_page_dirty(page);
mark_page_accessed(page);
put_page(page);
}
obj->mm.dirty = false;
sg_free_table(pages);
kfree(pages);
}
static void
i915_gem_userptr_release(struct drm_i915_gem_object *obj)
{
i915_gem_userptr_release__mmu_notifier(obj);
i915_gem_userptr_release__mm_struct(obj);
}
static int
i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
{
if (obj->userptr.mmu_object)
return 0;
return i915_gem_userptr_init__mmu_notifier(obj, 0);
}
static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
I915_GEM_OBJECT_IS_SHRINKABLE |
I915_GEM_OBJECT_ASYNC_CANCEL,
.get_pages = i915_gem_userptr_get_pages,
.put_pages = i915_gem_userptr_put_pages,
.dmabuf_export = i915_gem_userptr_dmabuf_export,
.release = i915_gem_userptr_release,
};
/*
* Creates a new mm object that wraps some normal memory from the process
* context - user memory.
*
* We impose several restrictions upon the memory being mapped
* into the GPU.
* 1. It must be page aligned (both start/end addresses, i.e ptr and size).
* 2. It must be normal system memory, not a pointer into another map of IO
* space (e.g. it must not be a GTT mmapping of another object).
* 3. We only allow a bo as large as we could in theory map into the GTT,
* that is we limit the size to the total size of the GTT.
* 4. The bo is marked as being snoopable. The backing pages are left
* accessible directly by the CPU, but reads and writes by the GPU may
* incur the cost of a snoop (unless you have an LLC architecture).
*
* Synchronisation between multiple users and the GPU is left to userspace
* through the normal set-domain-ioctl. The kernel will enforce that the
* GPU relinquishes the VMA before it is returned back to the system
* i.e. upon free(), munmap() or process termination. However, the userspace
* malloc() library may not immediately relinquish the VMA after free() and
* instead reuse it whilst the GPU is still reading and writing to the VMA.
* Caveat emptor.
*
* Also note, that the object created here is not currently a "first class"
* object, in that several ioctls are banned. These are the CPU access
* ioctls: mmap(), pwrite and pread. In practice, you are expected to use
* direct access via your pointer rather than use those ioctls. Another
* restriction is that we do not allow userptr surfaces to be pinned to the
* hardware and so we reject any attempt to create a framebuffer out of a
* userptr.
*
* If you think this is a good interface to use to pass GPU memory between
* drivers, please use dma-buf instead. In fact, wherever possible use
* dma-buf instead.
*/
int
i915_gem_userptr_ioctl(struct drm_device *dev,
void *data,
struct drm_file *file)
{
struct drm_i915_private *dev_priv = to_i915(dev);
struct drm_i915_gem_userptr *args = data;
struct drm_i915_gem_object *obj;
int ret;
u32 handle;
if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
/* We cannot support coherent userptr objects on hw without
* LLC and broken snooping.
*/
return -ENODEV;
}
if (args->flags & ~(I915_USERPTR_READ_ONLY |
I915_USERPTR_UNSYNCHRONIZED))
return -EINVAL;
if (!args->user_size)
return -EINVAL;
if (offset_in_page(args->user_ptr | args->user_size))
return -EINVAL;
if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size))
return -EFAULT;
if (args->flags & I915_USERPTR_READ_ONLY) {
struct i915_hw_ppgtt *ppgtt;
/*
* On almost all of the older hw, we cannot tell the GPU that
* a page is readonly.
*/
ppgtt = dev_priv->kernel_context->ppgtt;
if (!ppgtt || !ppgtt->vm.has_read_only)
return -ENODEV;
}
obj = i915_gem_object_alloc();
if (obj == NULL)
return -ENOMEM;
drm_gem_private_object_init(dev, &obj->base, args->user_size);
i915_gem_object_init(obj, &i915_gem_userptr_ops);
obj->read_domains = I915_GEM_DOMAIN_CPU;
obj->write_domain = I915_GEM_DOMAIN_CPU;
i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
obj->userptr.ptr = args->user_ptr;
if (args->flags & I915_USERPTR_READ_ONLY)
i915_gem_object_set_readonly(obj);
/* And keep a pointer to the current->mm for resolving the user pages
* at binding. This means that we need to hook into the mmu_notifier
* in order to detect if the mmu is destroyed.
*/
ret = i915_gem_userptr_init__mm_struct(obj);
if (ret == 0)
ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags);
if (ret == 0)
ret = drm_gem_handle_create(file, &obj->base, &handle);
/* drop reference from allocate - handle holds it now */
i915_gem_object_put(obj);
if (ret)
return ret;
args->handle = handle;
return 0;
}
int i915_gem_init_userptr(struct drm_i915_private *dev_priv)
{
mutex_init(&dev_priv->mm_lock);
hash_init(dev_priv->mm_structs);
dev_priv->mm.userptr_wq =
alloc_workqueue("i915-userptr-acquire",
WQ_HIGHPRI | WQ_UNBOUND,
0);
if (!dev_priv->mm.userptr_wq)
return -ENOMEM;
return 0;
}
void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv)
{
destroy_workqueue(dev_priv->mm.userptr_wq);
}

57
drivers/gpu/drm/i915/gem/i915_gemfs.c Normal file
파일 보기

@@ -0,0 +1,57 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2017 Intel Corporation
*/
#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/pagemap.h>
#include "i915_drv.h"
#include "i915_gemfs.h"
int i915_gemfs_init(struct drm_i915_private *i915)
{
struct file_system_type *type;
struct vfsmount *gemfs;
type = get_fs_type("tmpfs");
if (!type)
return -ENODEV;
gemfs = kern_mount(type);
if (IS_ERR(gemfs))
return PTR_ERR(gemfs);
/*
* Enable huge-pages for objects that are at least HPAGE_PMD_SIZE, most
* likely 2M. Note that within_size may overallocate huge-pages, if say
* we allocate an object of size 2M + 4K, we may get 2M + 2M, but under
* memory pressure shmem should split any huge-pages which can be
* shrunk.
*/
if (has_transparent_hugepage()) {
struct super_block *sb = gemfs->mnt_sb;
/* FIXME: Disabled until we get W/A for read BW issue. */
char options[] = "huge=never";
int flags = 0;
int err;
err = sb->s_op->remount_fs(sb, &flags, options);
if (err) {
kern_unmount(gemfs);
return err;
}
}
i915->mm.gemfs = gemfs;
return 0;
}
void i915_gemfs_fini(struct drm_i915_private *i915)
{
kern_unmount(i915->mm.gemfs);
}

16
drivers/gpu/drm/i915/gem/i915_gemfs.h Normal file
파일 보기

@@ -0,0 +1,16 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2017 Intel Corporation
*/
#ifndef __I915_GEMFS_H__
#define __I915_GEMFS_H__
struct drm_i915_private;
int i915_gemfs_init(struct drm_i915_private *i915);
void i915_gemfs_fini(struct drm_i915_private *i915);
#endif

121
drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c Normal file
파일 보기

@@ -0,0 +1,121 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#include "huge_gem_object.h"
static void huge_free_pages(struct drm_i915_gem_object *obj,
struct sg_table *pages)
{
unsigned long nreal = obj->scratch / PAGE_SIZE;
struct scatterlist *sg;
for (sg = pages->sgl; sg && nreal--; sg = __sg_next(sg))
__free_page(sg_page(sg));
sg_free_table(pages);
kfree(pages);
}
static int huge_get_pages(struct drm_i915_gem_object *obj)
{
#define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
const unsigned long nreal = obj->scratch / PAGE_SIZE;
const unsigned long npages = obj->base.size / PAGE_SIZE;
struct scatterlist *sg, *src, *end;
struct sg_table *pages;
unsigned long n;
pages = kmalloc(sizeof(*pages), GFP);
if (!pages)
return -ENOMEM;
if (sg_alloc_table(pages, npages, GFP)) {
kfree(pages);
return -ENOMEM;
}
sg = pages->sgl;
for (n = 0; n < nreal; n++) {
struct page *page;
page = alloc_page(GFP | __GFP_HIGHMEM);
if (!page) {
sg_mark_end(sg);
goto err;
}
sg_set_page(sg, page, PAGE_SIZE, 0);
sg = __sg_next(sg);
}
if (nreal < npages) {
for (end = sg, src = pages->sgl; sg; sg = __sg_next(sg)) {
sg_set_page(sg, sg_page(src), PAGE_SIZE, 0);
src = __sg_next(src);
if (src == end)
src = pages->sgl;
}
}
if (i915_gem_gtt_prepare_pages(obj, pages))
goto err;
__i915_gem_object_set_pages(obj, pages, PAGE_SIZE);
return 0;
err:
huge_free_pages(obj, pages);
return -ENOMEM;
#undef GFP
}
static void huge_put_pages(struct drm_i915_gem_object *obj,
struct sg_table *pages)
{
i915_gem_gtt_finish_pages(obj, pages);
huge_free_pages(obj, pages);
obj->mm.dirty = false;
}
static const struct drm_i915_gem_object_ops huge_ops = {
.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
I915_GEM_OBJECT_IS_SHRINKABLE,
.get_pages = huge_get_pages,
.put_pages = huge_put_pages,
};
struct drm_i915_gem_object *
huge_gem_object(struct drm_i915_private *i915,
phys_addr_t phys_size,
dma_addr_t dma_size)
{
struct drm_i915_gem_object *obj;
unsigned int cache_level;
GEM_BUG_ON(!phys_size || phys_size > dma_size);
GEM_BUG_ON(!IS_ALIGNED(phys_size, PAGE_SIZE));
GEM_BUG_ON(!IS_ALIGNED(dma_size, I915_GTT_PAGE_SIZE));
if (overflows_type(dma_size, obj->base.size))
return ERR_PTR(-E2BIG);
obj = i915_gem_object_alloc();
if (!obj)
return ERR_PTR(-ENOMEM);
drm_gem_private_object_init(&i915->drm, &obj->base, dma_size);
i915_gem_object_init(obj, &huge_ops);
obj->read_domains = I915_GEM_DOMAIN_CPU;
obj->write_domain = I915_GEM_DOMAIN_CPU;
cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
i915_gem_object_set_cache_coherency(obj, cache_level);
obj->scratch = phys_size;
return obj;
}

27
drivers/gpu/drm/i915/gem/selftests/huge_gem_object.h Normal file
파일 보기

@@ -0,0 +1,27 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#ifndef __HUGE_GEM_OBJECT_H
#define __HUGE_GEM_OBJECT_H
struct drm_i915_gem_object *
huge_gem_object(struct drm_i915_private *i915,
phys_addr_t phys_size,
dma_addr_t dma_size);
static inline phys_addr_t
huge_gem_object_phys_size(struct drm_i915_gem_object *obj)
{
return obj->scratch;
}
static inline dma_addr_t
huge_gem_object_dma_size(struct drm_i915_gem_object *obj)
{
return obj->base.size;
}
#endif /* !__HUGE_GEM_OBJECT_H */

1780
drivers/gpu/drm/i915/gem/selftests/huge_pages.c Normal file
파일 보기

파일 크기가 너무 크기때문에 변경 상태를 표시하지 않습니다. Load Diff

379
drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c Normal file
파일 보기

@@ -0,0 +1,379 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2017 Intel Corporation
*/
#include <linux/prime_numbers.h>
#include "i915_selftest.h"
#include "selftests/i915_random.h"
static int cpu_set(struct drm_i915_gem_object *obj,
unsigned long offset,
u32 v)
{
unsigned int needs_clflush;
struct page *page;
void *map;
u32 *cpu;
int err;
err = i915_gem_object_prepare_write(obj, &needs_clflush);
if (err)
return err;
page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
map = kmap_atomic(page);
cpu = map + offset_in_page(offset);
if (needs_clflush & CLFLUSH_BEFORE)
drm_clflush_virt_range(cpu, sizeof(*cpu));
*cpu = v;
if (needs_clflush & CLFLUSH_AFTER)
drm_clflush_virt_range(cpu, sizeof(*cpu));
kunmap_atomic(map);
i915_gem_object_finish_access(obj);
return 0;
}
static int cpu_get(struct drm_i915_gem_object *obj,
unsigned long offset,
u32 *v)
{
unsigned int needs_clflush;
struct page *page;
void *map;
u32 *cpu;
int err;
err = i915_gem_object_prepare_read(obj, &needs_clflush);
if (err)
return err;
page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
map = kmap_atomic(page);
cpu = map + offset_in_page(offset);
if (needs_clflush & CLFLUSH_BEFORE)
drm_clflush_virt_range(cpu, sizeof(*cpu));
*v = *cpu;
kunmap_atomic(map);
i915_gem_object_finish_access(obj);
return 0;
}
static int gtt_set(struct drm_i915_gem_object *obj,
unsigned long offset,
u32 v)
{
struct i915_vma *vma;
u32 __iomem *map;
int err;
err = i915_gem_object_set_to_gtt_domain(obj, true);
if (err)
return err;
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
if (IS_ERR(vma))
return PTR_ERR(vma);
map = i915_vma_pin_iomap(vma);
i915_vma_unpin(vma);
if (IS_ERR(map))
return PTR_ERR(map);
iowrite32(v, &map[offset / sizeof(*map)]);
i915_vma_unpin_iomap(vma);
return 0;
}
static int gtt_get(struct drm_i915_gem_object *obj,
unsigned long offset,
u32 *v)
{
struct i915_vma *vma;
u32 __iomem *map;
int err;
err = i915_gem_object_set_to_gtt_domain(obj, false);
if (err)
return err;
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
if (IS_ERR(vma))
return PTR_ERR(vma);
map = i915_vma_pin_iomap(vma);
i915_vma_unpin(vma);
if (IS_ERR(map))
return PTR_ERR(map);
*v = ioread32(&map[offset / sizeof(*map)]);
i915_vma_unpin_iomap(vma);
return 0;
}
static int wc_set(struct drm_i915_gem_object *obj,
unsigned long offset,
u32 v)
{
u32 *map;
int err;
err = i915_gem_object_set_to_wc_domain(obj, true);
if (err)
return err;
map = i915_gem_object_pin_map(obj, I915_MAP_WC);
if (IS_ERR(map))
return PTR_ERR(map);
map[offset / sizeof(*map)] = v;
i915_gem_object_unpin_map(obj);
return 0;
}
static int wc_get(struct drm_i915_gem_object *obj,
unsigned long offset,
u32 *v)
{
u32 *map;
int err;
err = i915_gem_object_set_to_wc_domain(obj, false);
if (err)
return err;
map = i915_gem_object_pin_map(obj, I915_MAP_WC);
if (IS_ERR(map))
return PTR_ERR(map);
*v = map[offset / sizeof(*map)];
i915_gem_object_unpin_map(obj);
return 0;
}
static int gpu_set(struct drm_i915_gem_object *obj,
unsigned long offset,
u32 v)
{
struct drm_i915_private *i915 = to_i915(obj->base.dev);
struct i915_request *rq;
struct i915_vma *vma;
u32 *cs;
int err;
err = i915_gem_object_set_to_gtt_domain(obj, true);
if (err)
return err;
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
if (IS_ERR(vma))
return PTR_ERR(vma);
rq = i915_request_create(i915->engine[RCS0]->kernel_context);
if (IS_ERR(rq)) {
i915_vma_unpin(vma);
return PTR_ERR(rq);
}
cs = intel_ring_begin(rq, 4);
if (IS_ERR(cs)) {
i915_request_add(rq);
i915_vma_unpin(vma);
return PTR_ERR(cs);
}
if (INTEL_GEN(i915) >= 8) {
*cs++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22;
*cs++ = lower_32_bits(i915_ggtt_offset(vma) + offset);
*cs++ = upper_32_bits(i915_ggtt_offset(vma) + offset);
*cs++ = v;
} else if (INTEL_GEN(i915) >= 4) {
*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
*cs++ = 0;
*cs++ = i915_ggtt_offset(vma) + offset;
*cs++ = v;
} else {
*cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
*cs++ = i915_ggtt_offset(vma) + offset;
*cs++ = v;
*cs++ = MI_NOOP;
}
intel_ring_advance(rq, cs);
err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
i915_vma_unpin(vma);
i915_request_add(rq);
return err;
}
static bool always_valid(struct drm_i915_private *i915)
{
return true;
}
static bool needs_fence_registers(struct drm_i915_private *i915)
{
return !i915_terminally_wedged(i915);
}
static bool needs_mi_store_dword(struct drm_i915_private *i915)
{
if (i915_terminally_wedged(i915))
return false;
return intel_engine_can_store_dword(i915->engine[RCS0]);
}
static const struct igt_coherency_mode {
const char *name;
int (*set)(struct drm_i915_gem_object *, unsigned long offset, u32 v);
int (*get)(struct drm_i915_gem_object *, unsigned long offset, u32 *v);
bool (*valid)(struct drm_i915_private *i915);
} igt_coherency_mode[] = {
{ "cpu", cpu_set, cpu_get, always_valid },
{ "gtt", gtt_set, gtt_get, needs_fence_registers },
{ "wc", wc_set, wc_get, always_valid },
{ "gpu", gpu_set, NULL, needs_mi_store_dword },
{ },
};
static int igt_gem_coherency(void *arg)
{
const unsigned int ncachelines = PAGE_SIZE/64;
I915_RND_STATE(prng);
struct drm_i915_private *i915 = arg;
const struct igt_coherency_mode *read, *write, *over;
struct drm_i915_gem_object *obj;
intel_wakeref_t wakeref;
unsigned long count, n;
u32 *offsets, *values;
int err = 0;
/* We repeatedly write, overwrite and read from a sequence of
* cachelines in order to try and detect incoherency (unflushed writes
* from either the CPU or GPU). Each setter/getter uses our cache
* domain API which should prevent incoherency.
*/
offsets = kmalloc_array(ncachelines, 2*sizeof(u32), GFP_KERNEL);
if (!offsets)
return -ENOMEM;
for (count = 0; count < ncachelines; count++)
offsets[count] = count * 64 + 4 * (count % 16);
values = offsets + ncachelines;
mutex_lock(&i915->drm.struct_mutex);
wakeref = intel_runtime_pm_get(i915);
for (over = igt_coherency_mode; over->name; over++) {
if (!over->set)
continue;
if (!over->valid(i915))
continue;
for (write = igt_coherency_mode; write->name; write++) {
if (!write->set)
continue;
if (!write->valid(i915))
continue;
for (read = igt_coherency_mode; read->name; read++) {
if (!read->get)
continue;
if (!read->valid(i915))
continue;
for_each_prime_number_from(count, 1, ncachelines) {
obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
if (IS_ERR(obj)) {
err = PTR_ERR(obj);
goto unlock;
}
i915_random_reorder(offsets, ncachelines, &prng);
for (n = 0; n < count; n++)
values[n] = prandom_u32_state(&prng);
for (n = 0; n < count; n++) {
err = over->set(obj, offsets[n], ~values[n]);
if (err) {
pr_err("Failed to set stale value[%ld/%ld] in object using %s, err=%d\n",
n, count, over->name, err);
goto put_object;
}
}
for (n = 0; n < count; n++) {
err = write->set(obj, offsets[n], values[n]);
if (err) {
pr_err("Failed to set value[%ld/%ld] in object using %s, err=%d\n",
n, count, write->name, err);
goto put_object;
}
}
for (n = 0; n < count; n++) {
u32 found;
err = read->get(obj, offsets[n], &found);
if (err) {
pr_err("Failed to get value[%ld/%ld] in object using %s, err=%d\n",
n, count, read->name, err);
goto put_object;
}
if (found != values[n]) {
pr_err("Value[%ld/%ld] mismatch, (overwrite with %s) wrote [%s] %x read [%s] %x (inverse %x), at offset %x\n",
n, count, over->name,
write->name, values[n],
read->name, found,
~values[n], offsets[n]);
err = -EINVAL;
goto put_object;
}
}
__i915_gem_object_release_unless_active(obj);
}
}
}
}
unlock:
intel_runtime_pm_put(i915, wakeref);
mutex_unlock(&i915->drm.struct_mutex);
kfree(offsets);
return err;
put_object:
__i915_gem_object_release_unless_active(obj);
goto unlock;
}
int i915_gem_coherency_live_selftests(struct drm_i915_private *i915)
{
static const struct i915_subtest tests[] = {
SUBTEST(igt_gem_coherency),
};
return i915_subtests(tests, i915);
}

1736
drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c Normal file
파일 보기

파일 크기가 너무 크기때문에 변경 상태를 표시하지 않습니다. Load Diff

386
drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c Normal file
파일 보기

@@ -0,0 +1,386 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#include "i915_selftest.h"
#include "mock_dmabuf.h"
#include "selftests/mock_gem_device.h"
static int igt_dmabuf_export(void *arg)
{
struct drm_i915_private *i915 = arg;
struct drm_i915_gem_object *obj;
struct dma_buf *dmabuf;
obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
if (IS_ERR(obj))
return PTR_ERR(obj);
dmabuf = i915_gem_prime_export(&i915->drm, &obj->base, 0);
i915_gem_object_put(obj);
if (IS_ERR(dmabuf)) {
pr_err("i915_gem_prime_export failed with err=%d\n",
(int)PTR_ERR(dmabuf));
return PTR_ERR(dmabuf);
}
dma_buf_put(dmabuf);
return 0;
}
static int igt_dmabuf_import_self(void *arg)
{
struct drm_i915_private *i915 = arg;
struct drm_i915_gem_object *obj;
struct drm_gem_object *import;
struct dma_buf *dmabuf;
int err;
obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
if (IS_ERR(obj))
return PTR_ERR(obj);
dmabuf = i915_gem_prime_export(&i915->drm, &obj->base, 0);
if (IS_ERR(dmabuf)) {
pr_err("i915_gem_prime_export failed with err=%d\n",
(int)PTR_ERR(dmabuf));
err = PTR_ERR(dmabuf);
goto out;
}
import = i915_gem_prime_import(&i915->drm, dmabuf);
if (IS_ERR(import)) {
pr_err("i915_gem_prime_import failed with err=%d\n",
(int)PTR_ERR(import));
err = PTR_ERR(import);
goto out_dmabuf;
}
if (import != &obj->base) {
pr_err("i915_gem_prime_import created a new object!\n");
err = -EINVAL;
goto out_import;
}
err = 0;
out_import:
i915_gem_object_put(to_intel_bo(import));
out_dmabuf:
dma_buf_put(dmabuf);
out:
i915_gem_object_put(obj);
return err;
}
static int igt_dmabuf_import(void *arg)
{
struct drm_i915_private *i915 = arg;
struct drm_i915_gem_object *obj;
struct dma_buf *dmabuf;
void *obj_map, *dma_map;
u32 pattern[] = { 0, 0xaa, 0xcc, 0x55, 0xff };
int err, i;
dmabuf = mock_dmabuf(1);
if (IS_ERR(dmabuf))
return PTR_ERR(dmabuf);
obj = to_intel_bo(i915_gem_prime_import(&i915->drm, dmabuf));
if (IS_ERR(obj)) {
pr_err("i915_gem_prime_import failed with err=%d\n",
(int)PTR_ERR(obj));
err = PTR_ERR(obj);
goto out_dmabuf;
}
if (obj->base.dev != &i915->drm) {
pr_err("i915_gem_prime_import created a non-i915 object!\n");
err = -EINVAL;
goto out_obj;
}
if (obj->base.size != PAGE_SIZE) {
pr_err("i915_gem_prime_import is wrong size found %lld, expected %ld\n",
(long long)obj->base.size, PAGE_SIZE);
err = -EINVAL;
goto out_obj;
}
dma_map = dma_buf_vmap(dmabuf);
if (!dma_map) {
pr_err("dma_buf_vmap failed\n");
err = -ENOMEM;
goto out_obj;
}
if (0) { /* Can not yet map dmabuf */
obj_map = i915_gem_object_pin_map(obj, I915_MAP_WB);
if (IS_ERR(obj_map)) {
err = PTR_ERR(obj_map);
pr_err("i915_gem_object_pin_map failed with err=%d\n", err);
goto out_dma_map;
}
for (i = 0; i < ARRAY_SIZE(pattern); i++) {
memset(dma_map, pattern[i], PAGE_SIZE);
if (memchr_inv(obj_map, pattern[i], PAGE_SIZE)) {
err = -EINVAL;
pr_err("imported vmap not all set to %x!\n", pattern[i]);
i915_gem_object_unpin_map(obj);
goto out_dma_map;
}
}
for (i = 0; i < ARRAY_SIZE(pattern); i++) {
memset(obj_map, pattern[i], PAGE_SIZE);
if (memchr_inv(dma_map, pattern[i], PAGE_SIZE)) {
err = -EINVAL;
pr_err("exported vmap not all set to %x!\n", pattern[i]);
i915_gem_object_unpin_map(obj);
goto out_dma_map;
}
}
i915_gem_object_unpin_map(obj);
}
err = 0;
out_dma_map:
dma_buf_vunmap(dmabuf, dma_map);
out_obj:
i915_gem_object_put(obj);
out_dmabuf:
dma_buf_put(dmabuf);
return err;
}
static int igt_dmabuf_import_ownership(void *arg)
{
struct drm_i915_private *i915 = arg;
struct drm_i915_gem_object *obj;
struct dma_buf *dmabuf;
void *ptr;
int err;
dmabuf = mock_dmabuf(1);
if (IS_ERR(dmabuf))
return PTR_ERR(dmabuf);
ptr = dma_buf_vmap(dmabuf);
if (!ptr) {
pr_err("dma_buf_vmap failed\n");
err = -ENOMEM;
goto err_dmabuf;
}
memset(ptr, 0xc5, PAGE_SIZE);
dma_buf_vunmap(dmabuf, ptr);
obj = to_intel_bo(i915_gem_prime_import(&i915->drm, dmabuf));
if (IS_ERR(obj)) {
pr_err("i915_gem_prime_import failed with err=%d\n",
(int)PTR_ERR(obj));
err = PTR_ERR(obj);
goto err_dmabuf;
}
dma_buf_put(dmabuf);
err = i915_gem_object_pin_pages(obj);
if (err) {
pr_err("i915_gem_object_pin_pages failed with err=%d\n", err);
goto out_obj;
}
err = 0;
i915_gem_object_unpin_pages(obj);
out_obj:
i915_gem_object_put(obj);
return err;
err_dmabuf:
dma_buf_put(dmabuf);
return err;
}
static int igt_dmabuf_export_vmap(void *arg)
{
struct drm_i915_private *i915 = arg;
struct drm_i915_gem_object *obj;
struct dma_buf *dmabuf;
void *ptr;
int err;
obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
if (IS_ERR(obj))
return PTR_ERR(obj);
dmabuf = i915_gem_prime_export(&i915->drm, &obj->base, 0);
if (IS_ERR(dmabuf)) {
pr_err("i915_gem_prime_export failed with err=%d\n",
(int)PTR_ERR(dmabuf));
err = PTR_ERR(dmabuf);
goto err_obj;
}
i915_gem_object_put(obj);
ptr = dma_buf_vmap(dmabuf);
if (!ptr) {
pr_err("dma_buf_vmap failed\n");
err = -ENOMEM;
goto out;
}
if (memchr_inv(ptr, 0, dmabuf->size)) {
pr_err("Exported object not initialiased to zero!\n");
err = -EINVAL;
goto out;
}
memset(ptr, 0xc5, dmabuf->size);
err = 0;
dma_buf_vunmap(dmabuf, ptr);
out:
dma_buf_put(dmabuf);
return err;
err_obj:
i915_gem_object_put(obj);
return err;
}
static int igt_dmabuf_export_kmap(void *arg)
{
struct drm_i915_private *i915 = arg;
struct drm_i915_gem_object *obj;
struct dma_buf *dmabuf;
void *ptr;
int err;
obj = i915_gem_object_create_shmem(i915, 2 * PAGE_SIZE);
if (IS_ERR(obj))
return PTR_ERR(obj);
dmabuf = i915_gem_prime_export(&i915->drm, &obj->base, 0);
i915_gem_object_put(obj);
if (IS_ERR(dmabuf)) {
err = PTR_ERR(dmabuf);
pr_err("i915_gem_prime_export failed with err=%d\n", err);
return err;
}
ptr = dma_buf_kmap(dmabuf, 0);
if (!ptr) {
pr_err("dma_buf_kmap failed\n");
err = -ENOMEM;
goto err;
}
if (memchr_inv(ptr, 0, PAGE_SIZE)) {
dma_buf_kunmap(dmabuf, 0, ptr);
pr_err("Exported page[0] not initialiased to zero!\n");
err = -EINVAL;
goto err;
}
memset(ptr, 0xc5, PAGE_SIZE);
dma_buf_kunmap(dmabuf, 0, ptr);
ptr = i915_gem_object_pin_map(obj, I915_MAP_WB);
if (IS_ERR(ptr)) {
err = PTR_ERR(ptr);
pr_err("i915_gem_object_pin_map failed with err=%d\n", err);
goto err;
}
memset(ptr + PAGE_SIZE, 0xaa, PAGE_SIZE);
i915_gem_object_flush_map(obj);
i915_gem_object_unpin_map(obj);
ptr = dma_buf_kmap(dmabuf, 1);
if (!ptr) {
pr_err("dma_buf_kmap failed\n");
err = -ENOMEM;
goto err;
}
if (memchr_inv(ptr, 0xaa, PAGE_SIZE)) {
dma_buf_kunmap(dmabuf, 1, ptr);
pr_err("Exported page[1] not set to 0xaa!\n");
err = -EINVAL;
goto err;
}
memset(ptr, 0xc5, PAGE_SIZE);
dma_buf_kunmap(dmabuf, 1, ptr);
ptr = dma_buf_kmap(dmabuf, 0);
if (!ptr) {
pr_err("dma_buf_kmap failed\n");
err = -ENOMEM;
goto err;
}
if (memchr_inv(ptr, 0xc5, PAGE_SIZE)) {
dma_buf_kunmap(dmabuf, 0, ptr);
pr_err("Exported page[0] did not retain 0xc5!\n");
err = -EINVAL;
goto err;
}
dma_buf_kunmap(dmabuf, 0, ptr);
ptr = dma_buf_kmap(dmabuf, 2);
if (ptr) {
pr_err("Erroneously kmapped beyond the end of the object!\n");
dma_buf_kunmap(dmabuf, 2, ptr);
err = -EINVAL;
goto err;
}
ptr = dma_buf_kmap(dmabuf, -1);
if (ptr) {
pr_err("Erroneously kmapped before the start of the object!\n");
dma_buf_kunmap(dmabuf, -1, ptr);
err = -EINVAL;
goto err;
}
err = 0;
err:
dma_buf_put(dmabuf);
return err;
}
int i915_gem_dmabuf_mock_selftests(void)
{
static const struct i915_subtest tests[] = {
SUBTEST(igt_dmabuf_export),
SUBTEST(igt_dmabuf_import_self),
SUBTEST(igt_dmabuf_import),
SUBTEST(igt_dmabuf_import_ownership),
SUBTEST(igt_dmabuf_export_vmap),
SUBTEST(igt_dmabuf_export_kmap),
};
struct drm_i915_private *i915;
int err;
i915 = mock_gem_device();
if (!i915)
return -ENOMEM;
err = i915_subtests(tests, i915);
drm_dev_put(&i915->drm);
return err;
}
int i915_gem_dmabuf_live_selftests(struct drm_i915_private *i915)
{
static const struct i915_subtest tests[] = {
SUBTEST(igt_dmabuf_export),
};
return i915_subtests(tests, i915);
}

2
drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
파일 보기

@@ -7,8 +7,8 @@
#include <linux/prime_numbers.h>
#include "gt/intel_gt_pm.h"
#include "huge_gem_object.h"
#include "i915_selftest.h"
#include "selftests/huge_gem_object.h"
#include "selftests/igt_flush_test.h"
struct tile {

99
drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c Normal file
파일 보기

@@ -0,0 +1,99 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#include "i915_selftest.h"
#include "huge_gem_object.h"
#include "selftests/igt_flush_test.h"
#include "selftests/mock_gem_device.h"
static int igt_gem_object(void *arg)
{
struct drm_i915_private *i915 = arg;
struct drm_i915_gem_object *obj;
int err = -ENOMEM;
/* Basic test to ensure we can create an object */
obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
if (IS_ERR(obj)) {
err = PTR_ERR(obj);
pr_err("i915_gem_object_create failed, err=%d\n", err);
goto out;
}
err = 0;
i915_gem_object_put(obj);
out:
return err;
}
static int igt_gem_huge(void *arg)
{
const unsigned int nreal = 509; /* just to be awkward */
struct drm_i915_private *i915 = arg;
struct drm_i915_gem_object *obj;
unsigned int n;
int err;
/* Basic sanitycheck of our huge fake object allocation */
obj = huge_gem_object(i915,
nreal * PAGE_SIZE,
i915->ggtt.vm.total + PAGE_SIZE);
if (IS_ERR(obj))
return PTR_ERR(obj);
err = i915_gem_object_pin_pages(obj);
if (err) {
pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
nreal, obj->base.size / PAGE_SIZE, err);
goto out;
}
for (n = 0; n < obj->base.size / PAGE_SIZE; n++) {
if (i915_gem_object_get_page(obj, n) !=
i915_gem_object_get_page(obj, n % nreal)) {
pr_err("Page lookup mismatch at index %u [%u]\n",
n, n % nreal);
err = -EINVAL;
goto out_unpin;
}
}
out_unpin:
i915_gem_object_unpin_pages(obj);
out:
i915_gem_object_put(obj);
return err;
}
int i915_gem_object_mock_selftests(void)
{
static const struct i915_subtest tests[] = {
SUBTEST(igt_gem_object),
};
struct drm_i915_private *i915;
int err;
i915 = mock_gem_device();
if (!i915)
return -ENOMEM;
err = i915_subtests(tests, i915);
drm_dev_put(&i915->drm);
return err;
}
int i915_gem_object_live_selftests(struct drm_i915_private *i915)
{
static const struct i915_subtest tests[] = {
SUBTEST(igt_gem_huge),
};
return i915_subtests(tests, i915);
}

34
drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c Normal file
파일 보기

@@ -0,0 +1,34 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2018 Intel Corporation
*/
#include "igt_gem_utils.h"
#include "gem/i915_gem_context.h"
#include "gem/i915_gem_pm.h"
#include "gt/intel_context.h"
#include "i915_request.h"
struct i915_request *
igt_request_alloc(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
{
struct intel_context *ce;
struct i915_request *rq;
/*
* Pinning the contexts may generate requests in order to acquire
* GGTT space, so do this first before we reserve a seqno for
* ourselves.
*/
ce = i915_gem_context_get_engine(ctx, engine->id);
if (IS_ERR(ce))
return ERR_CAST(ce);
rq = intel_context_create_request(ce);
intel_context_put(ce);
return rq;
}

17
drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.h Normal file
파일 보기

@@ -0,0 +1,17 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2018 Intel Corporation
*/
#ifndef __IGT_GEM_UTILS_H__
#define __IGT_GEM_UTILS_H__
struct i915_request;
struct i915_gem_context;
struct intel_engine_cs;
struct i915_request *
igt_request_alloc(struct i915_gem_context *ctx, struct intel_engine_cs *engine);
#endif /* __IGT_GEM_UTILS_H__ */

111
drivers/gpu/drm/i915/gem/selftests/mock_context.c Normal file
파일 보기

@@ -0,0 +1,111 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#include "mock_context.h"
#include "selftests/mock_gtt.h"
struct i915_gem_context *
mock_context(struct drm_i915_private *i915,
const char *name)
{
struct i915_gem_context *ctx;
struct i915_gem_engines *e;
int ret;
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx)
return NULL;
kref_init(&ctx->ref);
INIT_LIST_HEAD(&ctx->link);
ctx->i915 = i915;
mutex_init(&ctx->engines_mutex);
e = default_engines(ctx);
if (IS_ERR(e))
goto err_free;
RCU_INIT_POINTER(ctx->engines, e);
INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL);
INIT_LIST_HEAD(&ctx->handles_list);
INIT_LIST_HEAD(&ctx->hw_id_link);
mutex_init(&ctx->mutex);
ret = i915_gem_context_pin_hw_id(ctx);
if (ret < 0)
goto err_engines;
if (name) {
struct i915_hw_ppgtt *ppgtt;
ctx->name = kstrdup(name, GFP_KERNEL);
if (!ctx->name)
goto err_put;
ppgtt = mock_ppgtt(i915, name);
if (!ppgtt)
goto err_put;
__set_ppgtt(ctx, ppgtt);
}
return ctx;
err_engines:
free_engines(rcu_access_pointer(ctx->engines));
err_free:
kfree(ctx);
return NULL;
err_put:
i915_gem_context_set_closed(ctx);
i915_gem_context_put(ctx);
return NULL;
}
void mock_context_close(struct i915_gem_context *ctx)
{
context_close(ctx);
}
void mock_init_contexts(struct drm_i915_private *i915)
{
init_contexts(i915);
}
struct i915_gem_context *
live_context(struct drm_i915_private *i915, struct drm_file *file)
{
struct i915_gem_context *ctx;
int err;
lockdep_assert_held(&i915->drm.struct_mutex);
ctx = i915_gem_create_context(i915, 0);
if (IS_ERR(ctx))
return ctx;
err = gem_context_register(ctx, file->driver_priv);
if (err < 0)
goto err_ctx;
return ctx;
err_ctx:
context_close(ctx);
return ERR_PTR(err);
}
struct i915_gem_context *
kernel_context(struct drm_i915_private *i915)
{
return i915_gem_context_create_kernel(i915, I915_PRIORITY_NORMAL);
}
void kernel_context_close(struct i915_gem_context *ctx)
{
context_close(ctx);
}

24
drivers/gpu/drm/i915/gem/selftests/mock_context.h Normal file
파일 보기

@@ -0,0 +1,24 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#ifndef __MOCK_CONTEXT_H
#define __MOCK_CONTEXT_H
void mock_init_contexts(struct drm_i915_private *i915);
struct i915_gem_context *
mock_context(struct drm_i915_private *i915,
const char *name);
void mock_context_close(struct i915_gem_context *ctx);
struct i915_gem_context *
live_context(struct drm_i915_private *i915, struct drm_file *file);
struct i915_gem_context *kernel_context(struct drm_i915_private *i915);
void kernel_context_close(struct i915_gem_context *ctx);
#endif /* !__MOCK_CONTEXT_H */

144
drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c Normal file
파일 보기

@@ -0,0 +1,144 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#include "mock_dmabuf.h"
static struct sg_table *mock_map_dma_buf(struct dma_buf_attachment *attachment,
enum dma_data_direction dir)
{
struct mock_dmabuf *mock = to_mock(attachment->dmabuf);
struct sg_table *st;
struct scatterlist *sg;
int i, err;
st = kmalloc(sizeof(*st), GFP_KERNEL);
if (!st)
return ERR_PTR(-ENOMEM);
err = sg_alloc_table(st, mock->npages, GFP_KERNEL);
if (err)
goto err_free;
sg = st->sgl;
for (i = 0; i < mock->npages; i++) {
sg_set_page(sg, mock->pages[i], PAGE_SIZE, 0);
sg = sg_next(sg);
}
if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) {
err = -ENOMEM;
goto err_st;
}
return st;
err_st:
sg_free_table(st);
err_free:
kfree(st);
return ERR_PTR(err);
}
static void mock_unmap_dma_buf(struct dma_buf_attachment *attachment,
struct sg_table *st,
enum dma_data_direction dir)
{
dma_unmap_sg(attachment->dev, st->sgl, st->nents, dir);
sg_free_table(st);
kfree(st);
}
static void mock_dmabuf_release(struct dma_buf *dma_buf)
{
struct mock_dmabuf *mock = to_mock(dma_buf);
int i;
for (i = 0; i < mock->npages; i++)
put_page(mock->pages[i]);
kfree(mock);
}
static void *mock_dmabuf_vmap(struct dma_buf *dma_buf)
{
struct mock_dmabuf *mock = to_mock(dma_buf);
return vm_map_ram(mock->pages, mock->npages, 0, PAGE_KERNEL);
}
static void mock_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
{
struct mock_dmabuf *mock = to_mock(dma_buf);
vm_unmap_ram(vaddr, mock->npages);
}
static void *mock_dmabuf_kmap(struct dma_buf *dma_buf, unsigned long page_num)
{
struct mock_dmabuf *mock = to_mock(dma_buf);
return kmap(mock->pages[page_num]);
}
static void mock_dmabuf_kunmap(struct dma_buf *dma_buf, unsigned long page_num, void *addr)
{
struct mock_dmabuf *mock = to_mock(dma_buf);
return kunmap(mock->pages[page_num]);
}
static int mock_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma)
{
return -ENODEV;
}
static const struct dma_buf_ops mock_dmabuf_ops = {
.map_dma_buf = mock_map_dma_buf,
.unmap_dma_buf = mock_unmap_dma_buf,
.release = mock_dmabuf_release,
.map = mock_dmabuf_kmap,
.unmap = mock_dmabuf_kunmap,
.mmap = mock_dmabuf_mmap,
.vmap = mock_dmabuf_vmap,
.vunmap = mock_dmabuf_vunmap,
};
static struct dma_buf *mock_dmabuf(int npages)
{
struct mock_dmabuf *mock;
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
struct dma_buf *dmabuf;
int i;
mock = kmalloc(sizeof(*mock) + npages * sizeof(struct page *),
GFP_KERNEL);
if (!mock)
return ERR_PTR(-ENOMEM);
mock->npages = npages;
for (i = 0; i < npages; i++) {
mock->pages[i] = alloc_page(GFP_KERNEL);
if (!mock->pages[i])
goto err;
}
exp_info.ops = &mock_dmabuf_ops;
exp_info.size = npages * PAGE_SIZE;
exp_info.flags = O_CLOEXEC;
exp_info.priv = mock;
dmabuf = dma_buf_export(&exp_info);
if (IS_ERR(dmabuf))
goto err;
return dmabuf;
err:
while (i--)
put_page(mock->pages[i]);
kfree(mock);
return ERR_PTR(-ENOMEM);
}

22
drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.h Normal file
파일 보기

@@ -0,0 +1,22 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#ifndef __MOCK_DMABUF_H__
#define __MOCK_DMABUF_H__
#include <linux/dma-buf.h>
struct mock_dmabuf {
int npages;
struct page *pages[];
};
static struct mock_dmabuf *to_mock(struct dma_buf *buf)
{
return buf->priv;
}
#endif /* !__MOCK_DMABUF_H__ */

14
drivers/gpu/drm/i915/gem/selftests/mock_gem_object.h Normal file
파일 보기

@@ -0,0 +1,14 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2016 Intel Corporation
*/
#ifndef __MOCK_GEM_OBJECT_H__
#define __MOCK_GEM_OBJECT_H__
struct mock_object {
struct drm_i915_gem_object base;
};
#endif /* !__MOCK_GEM_OBJECT_H__ */