blob: 5fa3419461c5ddb46fa2b70b797b8c4ad3567e59 [file] [log] [blame]
Eric Anholt673a3942008-07-30 12:06:12 -07001/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
Chris Wilson1c5d22f2009-08-25 11:15:50 +010032#include "i915_trace.h"
Jesse Barnes652c3932009-08-17 13:31:43 -070033#include "intel_drv.h"
Hugh Dickins5949eac2011-06-27 16:18:18 -070034#include <linux/shmem_fs.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090035#include <linux/slab.h>
Eric Anholt673a3942008-07-30 12:06:12 -070036#include <linux/swap.h>
Jesse Barnes79e53942008-11-07 14:24:08 -080037#include <linux/pci.h>
Daniel Vetter1286ff72012-05-10 15:25:09 +020038#include <linux/dma-buf.h>
Eric Anholt673a3942008-07-30 12:06:12 -070039
Chris Wilson05394f32010-11-08 19:18:58 +000040static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
41static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
Chris Wilson88241782011-01-07 17:09:48 +000042static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
43 unsigned alignment,
Chris Wilson86a1ee22012-08-11 15:41:04 +010044 bool map_and_fenceable,
45 bool nonblocking);
Chris Wilson05394f32010-11-08 19:18:58 +000046static int i915_gem_phys_pwrite(struct drm_device *dev,
47 struct drm_i915_gem_object *obj,
Dave Airlie71acb5e2008-12-30 20:31:46 +100048 struct drm_i915_gem_pwrite *args,
Chris Wilson05394f32010-11-08 19:18:58 +000049 struct drm_file *file);
Eric Anholt673a3942008-07-30 12:06:12 -070050
Chris Wilson61050802012-04-17 15:31:31 +010051static void i915_gem_write_fence(struct drm_device *dev, int reg,
52 struct drm_i915_gem_object *obj);
53static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
54 struct drm_i915_fence_reg *fence,
55 bool enable);
56
Chris Wilson17250b72010-10-28 12:51:39 +010057static int i915_gem_inactive_shrink(struct shrinker *shrinker,
Ying Han1495f232011-05-24 17:12:27 -070058 struct shrink_control *sc);
Chris Wilson6c085a72012-08-20 11:40:46 +020059static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
60static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
Daniel Vetter8c599672011-12-14 13:57:31 +010061static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
Chris Wilson31169712009-09-14 16:50:28 +010062
Chris Wilson61050802012-04-17 15:31:31 +010063static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
64{
65 if (obj->tiling_mode)
66 i915_gem_release_mmap(obj);
67
68 /* As we do not have an associated fence register, we will force
69 * a tiling change if we ever need to acquire one.
70 */
Chris Wilson5d82e3e2012-04-21 16:23:23 +010071 obj->fence_dirty = false;
Chris Wilson61050802012-04-17 15:31:31 +010072 obj->fence_reg = I915_FENCE_REG_NONE;
73}
74
Chris Wilson73aa8082010-09-30 11:46:12 +010075/* some bookkeeping */
76static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
77 size_t size)
78{
79 dev_priv->mm.object_count++;
80 dev_priv->mm.object_memory += size;
81}
82
83static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
84 size_t size)
85{
86 dev_priv->mm.object_count--;
87 dev_priv->mm.object_memory -= size;
88}
89
Chris Wilson21dd3732011-01-26 15:55:56 +000090static int
91i915_gem_wait_for_error(struct drm_device *dev)
Chris Wilson30dbf0c2010-09-25 10:19:17 +010092{
93 struct drm_i915_private *dev_priv = dev->dev_private;
94 struct completion *x = &dev_priv->error_completion;
95 unsigned long flags;
96 int ret;
97
98 if (!atomic_read(&dev_priv->mm.wedged))
99 return 0;
100
Daniel Vetter0a6759c2012-07-04 22:18:41 +0200101 /*
102 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
103 * userspace. If it takes that long something really bad is going on and
104 * we should simply try to bail out and fail as gracefully as possible.
105 */
106 ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
107 if (ret == 0) {
108 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
109 return -EIO;
110 } else if (ret < 0) {
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100111 return ret;
Daniel Vetter0a6759c2012-07-04 22:18:41 +0200112 }
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100113
Chris Wilson21dd3732011-01-26 15:55:56 +0000114 if (atomic_read(&dev_priv->mm.wedged)) {
115 /* GPU is hung, bump the completion count to account for
116 * the token we just consumed so that we never hit zero and
117 * end up waiting upon a subsequent completion event that
118 * will never happen.
119 */
120 spin_lock_irqsave(&x->wait.lock, flags);
121 x->done++;
122 spin_unlock_irqrestore(&x->wait.lock, flags);
123 }
124 return 0;
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100125}
126
Chris Wilson54cf91d2010-11-25 18:00:26 +0000127int i915_mutex_lock_interruptible(struct drm_device *dev)
Chris Wilson76c1dec2010-09-25 11:22:51 +0100128{
Chris Wilson76c1dec2010-09-25 11:22:51 +0100129 int ret;
130
Chris Wilson21dd3732011-01-26 15:55:56 +0000131 ret = i915_gem_wait_for_error(dev);
Chris Wilson76c1dec2010-09-25 11:22:51 +0100132 if (ret)
133 return ret;
134
135 ret = mutex_lock_interruptible(&dev->struct_mutex);
136 if (ret)
137 return ret;
138
Chris Wilson23bc5982010-09-29 16:10:57 +0100139 WARN_ON(i915_verify_lists(dev));
Chris Wilson76c1dec2010-09-25 11:22:51 +0100140 return 0;
141}
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100142
Chris Wilson7d1c4802010-08-07 21:45:03 +0100143static inline bool
Chris Wilson05394f32010-11-08 19:18:58 +0000144i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
Chris Wilson7d1c4802010-08-07 21:45:03 +0100145{
Chris Wilson6c085a72012-08-20 11:40:46 +0200146 return obj->gtt_space && !obj->active;
Chris Wilson7d1c4802010-08-07 21:45:03 +0100147}
148
Eric Anholt673a3942008-07-30 12:06:12 -0700149int
150i915_gem_init_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +0000151 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -0700152{
Eric Anholt673a3942008-07-30 12:06:12 -0700153 struct drm_i915_gem_init *args = data;
Chris Wilson20217462010-11-23 15:26:33 +0000154
Daniel Vetter7bb6fb82012-04-24 08:22:52 +0200155 if (drm_core_check_feature(dev, DRIVER_MODESET))
156 return -ENODEV;
157
Chris Wilson20217462010-11-23 15:26:33 +0000158 if (args->gtt_start >= args->gtt_end ||
159 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
160 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -0700161
Daniel Vetterf534bc02012-03-26 22:37:04 +0200162 /* GEM with user mode setting was never supported on ilk and later. */
163 if (INTEL_INFO(dev)->gen >= 5)
164 return -ENODEV;
165
Eric Anholt673a3942008-07-30 12:06:12 -0700166 mutex_lock(&dev->struct_mutex);
Daniel Vetter644ec022012-03-26 09:45:40 +0200167 i915_gem_init_global_gtt(dev, args->gtt_start,
168 args->gtt_end, args->gtt_end);
Eric Anholt673a3942008-07-30 12:06:12 -0700169 mutex_unlock(&dev->struct_mutex);
170
Chris Wilson20217462010-11-23 15:26:33 +0000171 return 0;
Eric Anholt673a3942008-07-30 12:06:12 -0700172}
173
Eric Anholt5a125c32008-10-22 21:40:13 -0700174int
175i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +0000176 struct drm_file *file)
Eric Anholt5a125c32008-10-22 21:40:13 -0700177{
Chris Wilson73aa8082010-09-30 11:46:12 +0100178 struct drm_i915_private *dev_priv = dev->dev_private;
Eric Anholt5a125c32008-10-22 21:40:13 -0700179 struct drm_i915_gem_get_aperture *args = data;
Chris Wilson6299f992010-11-24 12:23:44 +0000180 struct drm_i915_gem_object *obj;
181 size_t pinned;
Eric Anholt5a125c32008-10-22 21:40:13 -0700182
Chris Wilson6299f992010-11-24 12:23:44 +0000183 pinned = 0;
Chris Wilson73aa8082010-09-30 11:46:12 +0100184 mutex_lock(&dev->struct_mutex);
Chris Wilson6c085a72012-08-20 11:40:46 +0200185 list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
Chris Wilson1b502472012-04-24 15:47:30 +0100186 if (obj->pin_count)
187 pinned += obj->gtt_space->size;
Chris Wilson73aa8082010-09-30 11:46:12 +0100188 mutex_unlock(&dev->struct_mutex);
Eric Anholt5a125c32008-10-22 21:40:13 -0700189
Chris Wilson6299f992010-11-24 12:23:44 +0000190 args->aper_size = dev_priv->mm.gtt_total;
Akshay Joshi0206e352011-08-16 15:34:10 -0400191 args->aper_available_size = args->aper_size - pinned;
Chris Wilson6299f992010-11-24 12:23:44 +0000192
Eric Anholt5a125c32008-10-22 21:40:13 -0700193 return 0;
194}
195
Dave Airlieff72145b2011-02-07 12:16:14 +1000196static int
197i915_gem_create(struct drm_file *file,
198 struct drm_device *dev,
199 uint64_t size,
200 uint32_t *handle_p)
Eric Anholt673a3942008-07-30 12:06:12 -0700201{
Chris Wilson05394f32010-11-08 19:18:58 +0000202 struct drm_i915_gem_object *obj;
Pekka Paalanena1a2d1d2009-08-23 12:40:55 +0300203 int ret;
204 u32 handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700205
Dave Airlieff72145b2011-02-07 12:16:14 +1000206 size = roundup(size, PAGE_SIZE);
Chris Wilson8ffc0242011-09-14 14:14:28 +0200207 if (size == 0)
208 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -0700209
210 /* Allocate the new object */
Dave Airlieff72145b2011-02-07 12:16:14 +1000211 obj = i915_gem_alloc_object(dev, size);
Eric Anholt673a3942008-07-30 12:06:12 -0700212 if (obj == NULL)
213 return -ENOMEM;
214
Chris Wilson05394f32010-11-08 19:18:58 +0000215 ret = drm_gem_handle_create(file, &obj->base, &handle);
Chris Wilson1dfd9752010-09-06 14:44:14 +0100216 if (ret) {
Chris Wilson05394f32010-11-08 19:18:58 +0000217 drm_gem_object_release(&obj->base);
218 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
Chris Wilson202f2fe2010-10-14 13:20:40 +0100219 kfree(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700220 return ret;
Chris Wilson1dfd9752010-09-06 14:44:14 +0100221 }
222
Chris Wilson202f2fe2010-10-14 13:20:40 +0100223 /* drop reference from allocate - handle holds it now */
Chris Wilson05394f32010-11-08 19:18:58 +0000224 drm_gem_object_unreference(&obj->base);
Chris Wilson202f2fe2010-10-14 13:20:40 +0100225 trace_i915_gem_object_create(obj);
226
Dave Airlieff72145b2011-02-07 12:16:14 +1000227 *handle_p = handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700228 return 0;
229}
230
Dave Airlieff72145b2011-02-07 12:16:14 +1000231int
232i915_gem_dumb_create(struct drm_file *file,
233 struct drm_device *dev,
234 struct drm_mode_create_dumb *args)
235{
236 /* have to work out size/pitch and return them */
Chris Wilsoned0291f2011-03-19 08:21:45 +0000237 args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
Dave Airlieff72145b2011-02-07 12:16:14 +1000238 args->size = args->pitch * args->height;
239 return i915_gem_create(file, dev,
240 args->size, &args->handle);
241}
242
243int i915_gem_dumb_destroy(struct drm_file *file,
244 struct drm_device *dev,
245 uint32_t handle)
246{
247 return drm_gem_handle_delete(file, handle);
248}
249
250/**
251 * Creates a new mm object and returns a handle to it.
252 */
253int
254i915_gem_create_ioctl(struct drm_device *dev, void *data,
255 struct drm_file *file)
256{
257 struct drm_i915_gem_create *args = data;
Daniel Vetter63ed2cb2012-04-23 16:50:50 +0200258
Dave Airlieff72145b2011-02-07 12:16:14 +1000259 return i915_gem_create(file, dev,
260 args->size, &args->handle);
261}
262
Chris Wilson05394f32010-11-08 19:18:58 +0000263static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
Eric Anholt280b7132009-03-12 16:56:27 -0700264{
Chris Wilson05394f32010-11-08 19:18:58 +0000265 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
Eric Anholt280b7132009-03-12 16:56:27 -0700266
267 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
Chris Wilson05394f32010-11-08 19:18:58 +0000268 obj->tiling_mode != I915_TILING_NONE;
Eric Anholt280b7132009-03-12 16:56:27 -0700269}
270
Daniel Vetter8c599672011-12-14 13:57:31 +0100271static inline int
Daniel Vetter8461d222011-12-14 13:57:32 +0100272__copy_to_user_swizzled(char __user *cpu_vaddr,
273 const char *gpu_vaddr, int gpu_offset,
274 int length)
275{
276 int ret, cpu_offset = 0;
277
278 while (length > 0) {
279 int cacheline_end = ALIGN(gpu_offset + 1, 64);
280 int this_length = min(cacheline_end - gpu_offset, length);
281 int swizzled_gpu_offset = gpu_offset ^ 64;
282
283 ret = __copy_to_user(cpu_vaddr + cpu_offset,
284 gpu_vaddr + swizzled_gpu_offset,
285 this_length);
286 if (ret)
287 return ret + length;
288
289 cpu_offset += this_length;
290 gpu_offset += this_length;
291 length -= this_length;
292 }
293
294 return 0;
295}
296
297static inline int
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -0700298__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
299 const char __user *cpu_vaddr,
Daniel Vetter8c599672011-12-14 13:57:31 +0100300 int length)
301{
302 int ret, cpu_offset = 0;
303
304 while (length > 0) {
305 int cacheline_end = ALIGN(gpu_offset + 1, 64);
306 int this_length = min(cacheline_end - gpu_offset, length);
307 int swizzled_gpu_offset = gpu_offset ^ 64;
308
309 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
310 cpu_vaddr + cpu_offset,
311 this_length);
312 if (ret)
313 return ret + length;
314
315 cpu_offset += this_length;
316 gpu_offset += this_length;
317 length -= this_length;
318 }
319
320 return 0;
321}
322
Daniel Vetterd174bd62012-03-25 19:47:40 +0200323/* Per-page copy function for the shmem pread fastpath.
324 * Flushes invalid cachelines before reading the target if
325 * needs_clflush is set. */
Eric Anholteb014592009-03-10 11:44:52 -0700326static int
Daniel Vetterd174bd62012-03-25 19:47:40 +0200327shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
328 char __user *user_data,
329 bool page_do_bit17_swizzling, bool needs_clflush)
330{
331 char *vaddr;
332 int ret;
333
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200334 if (unlikely(page_do_bit17_swizzling))
Daniel Vetterd174bd62012-03-25 19:47:40 +0200335 return -EINVAL;
336
337 vaddr = kmap_atomic(page);
338 if (needs_clflush)
339 drm_clflush_virt_range(vaddr + shmem_page_offset,
340 page_length);
341 ret = __copy_to_user_inatomic(user_data,
342 vaddr + shmem_page_offset,
343 page_length);
344 kunmap_atomic(vaddr);
345
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100346 return ret ? -EFAULT : 0;
Daniel Vetterd174bd62012-03-25 19:47:40 +0200347}
348
Daniel Vetter23c18c72012-03-25 19:47:42 +0200349static void
350shmem_clflush_swizzled_range(char *addr, unsigned long length,
351 bool swizzled)
352{
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200353 if (unlikely(swizzled)) {
Daniel Vetter23c18c72012-03-25 19:47:42 +0200354 unsigned long start = (unsigned long) addr;
355 unsigned long end = (unsigned long) addr + length;
356
357 /* For swizzling simply ensure that we always flush both
358 * channels. Lame, but simple and it works. Swizzled
359 * pwrite/pread is far from a hotpath - current userspace
360 * doesn't use it at all. */
361 start = round_down(start, 128);
362 end = round_up(end, 128);
363
364 drm_clflush_virt_range((void *)start, end - start);
365 } else {
366 drm_clflush_virt_range(addr, length);
367 }
368
369}
370
Daniel Vetterd174bd62012-03-25 19:47:40 +0200371/* Only difference to the fast-path function is that this can handle bit17
372 * and uses non-atomic copy and kmap functions. */
373static int
374shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
375 char __user *user_data,
376 bool page_do_bit17_swizzling, bool needs_clflush)
377{
378 char *vaddr;
379 int ret;
380
381 vaddr = kmap(page);
382 if (needs_clflush)
Daniel Vetter23c18c72012-03-25 19:47:42 +0200383 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
384 page_length,
385 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200386
387 if (page_do_bit17_swizzling)
388 ret = __copy_to_user_swizzled(user_data,
389 vaddr, shmem_page_offset,
390 page_length);
391 else
392 ret = __copy_to_user(user_data,
393 vaddr + shmem_page_offset,
394 page_length);
395 kunmap(page);
396
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100397 return ret ? - EFAULT : 0;
Daniel Vetterd174bd62012-03-25 19:47:40 +0200398}
399
Eric Anholteb014592009-03-10 11:44:52 -0700400static int
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200401i915_gem_shmem_pread(struct drm_device *dev,
402 struct drm_i915_gem_object *obj,
403 struct drm_i915_gem_pread *args,
404 struct drm_file *file)
Eric Anholteb014592009-03-10 11:44:52 -0700405{
Daniel Vetter8461d222011-12-14 13:57:32 +0100406 char __user *user_data;
Eric Anholteb014592009-03-10 11:44:52 -0700407 ssize_t remain;
Daniel Vetter8461d222011-12-14 13:57:32 +0100408 loff_t offset;
Ben Widawskyeb2c0c82012-02-15 14:42:43 +0100409 int shmem_page_offset, page_length, ret = 0;
Daniel Vetter8461d222011-12-14 13:57:32 +0100410 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200411 int hit_slowpath = 0;
Daniel Vetter96d79b52012-03-25 19:47:36 +0200412 int prefaulted = 0;
Daniel Vetter84897312012-03-25 19:47:31 +0200413 int needs_clflush = 0;
Chris Wilson9da3da62012-06-01 15:20:22 +0100414 struct scatterlist *sg;
415 int i;
Eric Anholteb014592009-03-10 11:44:52 -0700416
Daniel Vetter8461d222011-12-14 13:57:32 +0100417 user_data = (char __user *) (uintptr_t) args->data_ptr;
Eric Anholteb014592009-03-10 11:44:52 -0700418 remain = args->size;
419
Daniel Vetter8461d222011-12-14 13:57:32 +0100420 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700421
Daniel Vetter84897312012-03-25 19:47:31 +0200422 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
423 /* If we're not in the cpu read domain, set ourself into the gtt
424 * read domain and manually flush cachelines (if required). This
425 * optimizes for the case when the gpu will dirty the data
426 * anyway again before the next pread happens. */
427 if (obj->cache_level == I915_CACHE_NONE)
428 needs_clflush = 1;
Chris Wilson6c085a72012-08-20 11:40:46 +0200429 if (obj->gtt_space) {
430 ret = i915_gem_object_set_to_gtt_domain(obj, false);
431 if (ret)
432 return ret;
433 }
Daniel Vetter84897312012-03-25 19:47:31 +0200434 }
Eric Anholteb014592009-03-10 11:44:52 -0700435
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100436 ret = i915_gem_object_get_pages(obj);
437 if (ret)
438 return ret;
439
440 i915_gem_object_pin_pages(obj);
441
Eric Anholteb014592009-03-10 11:44:52 -0700442 offset = args->offset;
Daniel Vetter8461d222011-12-14 13:57:32 +0100443
Chris Wilson9da3da62012-06-01 15:20:22 +0100444 for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
Chris Wilsone5281cc2010-10-28 13:45:36 +0100445 struct page *page;
446
Chris Wilson9da3da62012-06-01 15:20:22 +0100447 if (i < offset >> PAGE_SHIFT)
448 continue;
449
450 if (remain <= 0)
451 break;
452
Eric Anholteb014592009-03-10 11:44:52 -0700453 /* Operation in this page
454 *
Eric Anholteb014592009-03-10 11:44:52 -0700455 * shmem_page_offset = offset within page in shmem file
Eric Anholteb014592009-03-10 11:44:52 -0700456 * page_length = bytes to copy for this page
457 */
Chris Wilsonc8cbbb82011-05-12 22:17:11 +0100458 shmem_page_offset = offset_in_page(offset);
Eric Anholteb014592009-03-10 11:44:52 -0700459 page_length = remain;
460 if ((shmem_page_offset + page_length) > PAGE_SIZE)
461 page_length = PAGE_SIZE - shmem_page_offset;
Eric Anholteb014592009-03-10 11:44:52 -0700462
Chris Wilson9da3da62012-06-01 15:20:22 +0100463 page = sg_page(sg);
Daniel Vetter8461d222011-12-14 13:57:32 +0100464 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
465 (page_to_phys(page) & (1 << 17)) != 0;
466
Daniel Vetterd174bd62012-03-25 19:47:40 +0200467 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
468 user_data, page_do_bit17_swizzling,
469 needs_clflush);
470 if (ret == 0)
471 goto next_page;
Eric Anholteb014592009-03-10 11:44:52 -0700472
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200473 hit_slowpath = 1;
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200474 mutex_unlock(&dev->struct_mutex);
475
Daniel Vetter96d79b52012-03-25 19:47:36 +0200476 if (!prefaulted) {
Daniel Vetterf56f8212012-03-25 19:47:41 +0200477 ret = fault_in_multipages_writeable(user_data, remain);
Daniel Vetter96d79b52012-03-25 19:47:36 +0200478 /* Userspace is tricking us, but we've already clobbered
479 * its pages with the prefault and promised to write the
480 * data up to the first fault. Hence ignore any errors
481 * and just continue. */
482 (void)ret;
483 prefaulted = 1;
484 }
485
Daniel Vetterd174bd62012-03-25 19:47:40 +0200486 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
487 user_data, page_do_bit17_swizzling,
488 needs_clflush);
Eric Anholteb014592009-03-10 11:44:52 -0700489
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200490 mutex_lock(&dev->struct_mutex);
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100491
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200492next_page:
Chris Wilsone5281cc2010-10-28 13:45:36 +0100493 mark_page_accessed(page);
Chris Wilsone5281cc2010-10-28 13:45:36 +0100494
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100495 if (ret)
Daniel Vetter8461d222011-12-14 13:57:32 +0100496 goto out;
Daniel Vetter8461d222011-12-14 13:57:32 +0100497
Eric Anholteb014592009-03-10 11:44:52 -0700498 remain -= page_length;
Daniel Vetter8461d222011-12-14 13:57:32 +0100499 user_data += page_length;
Eric Anholteb014592009-03-10 11:44:52 -0700500 offset += page_length;
501 }
502
Chris Wilson4f27b752010-10-14 15:26:45 +0100503out:
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100504 i915_gem_object_unpin_pages(obj);
505
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200506 if (hit_slowpath) {
507 /* Fixup: Kill any reinstated backing storage pages */
508 if (obj->madv == __I915_MADV_PURGED)
509 i915_gem_object_truncate(obj);
510 }
Eric Anholteb014592009-03-10 11:44:52 -0700511
512 return ret;
513}
514
Eric Anholt673a3942008-07-30 12:06:12 -0700515/**
516 * Reads data from the object referenced by handle.
517 *
518 * On error, the contents of *data are undefined.
519 */
520int
521i915_gem_pread_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +0000522 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -0700523{
524 struct drm_i915_gem_pread *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +0000525 struct drm_i915_gem_object *obj;
Chris Wilson35b62a82010-09-26 20:23:38 +0100526 int ret = 0;
Eric Anholt673a3942008-07-30 12:06:12 -0700527
Chris Wilson51311d02010-11-17 09:10:42 +0000528 if (args->size == 0)
529 return 0;
530
531 if (!access_ok(VERIFY_WRITE,
532 (char __user *)(uintptr_t)args->data_ptr,
533 args->size))
534 return -EFAULT;
535
Chris Wilson4f27b752010-10-14 15:26:45 +0100536 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100537 if (ret)
Chris Wilson4f27b752010-10-14 15:26:45 +0100538 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700539
Chris Wilson05394f32010-11-08 19:18:58 +0000540 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +0000541 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100542 ret = -ENOENT;
543 goto unlock;
Chris Wilson4f27b752010-10-14 15:26:45 +0100544 }
Eric Anholt673a3942008-07-30 12:06:12 -0700545
Chris Wilson7dcd2492010-09-26 20:21:44 +0100546 /* Bounds check source. */
Chris Wilson05394f32010-11-08 19:18:58 +0000547 if (args->offset > obj->base.size ||
548 args->size > obj->base.size - args->offset) {
Chris Wilsonce9d4192010-09-26 20:50:05 +0100549 ret = -EINVAL;
Chris Wilson35b62a82010-09-26 20:23:38 +0100550 goto out;
Chris Wilsonce9d4192010-09-26 20:50:05 +0100551 }
552
Daniel Vetter1286ff72012-05-10 15:25:09 +0200553 /* prime objects have no backing filp to GEM pread/pwrite
554 * pages from.
555 */
556 if (!obj->base.filp) {
557 ret = -EINVAL;
558 goto out;
559 }
560
Chris Wilsondb53a302011-02-03 11:57:46 +0000561 trace_i915_gem_object_pread(obj, args->offset, args->size);
562
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200563 ret = i915_gem_shmem_pread(dev, obj, args, file);
Eric Anholt673a3942008-07-30 12:06:12 -0700564
Chris Wilson35b62a82010-09-26 20:23:38 +0100565out:
Chris Wilson05394f32010-11-08 19:18:58 +0000566 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100567unlock:
Chris Wilson4f27b752010-10-14 15:26:45 +0100568 mutex_unlock(&dev->struct_mutex);
Eric Anholteb014592009-03-10 11:44:52 -0700569 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700570}
571
Keith Packard0839ccb2008-10-30 19:38:48 -0700572/* This is the fast write path which cannot handle
573 * page faults in the source data
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700574 */
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700575
Keith Packard0839ccb2008-10-30 19:38:48 -0700576static inline int
577fast_user_write(struct io_mapping *mapping,
578 loff_t page_base, int page_offset,
579 char __user *user_data,
580 int length)
581{
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -0700582 void __iomem *vaddr_atomic;
583 void *vaddr;
Keith Packard0839ccb2008-10-30 19:38:48 -0700584 unsigned long unwritten;
585
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -0700586 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -0700587 /* We can use the cpu mem copy function because this is X86. */
588 vaddr = (void __force*)vaddr_atomic + page_offset;
589 unwritten = __copy_from_user_inatomic_nocache(vaddr,
Keith Packard0839ccb2008-10-30 19:38:48 -0700590 user_data, length);
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -0700591 io_mapping_unmap_atomic(vaddr_atomic);
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100592 return unwritten;
Keith Packard0839ccb2008-10-30 19:38:48 -0700593}
594
Eric Anholt3de09aa2009-03-09 09:42:23 -0700595/**
596 * This is the fast pwrite path, where we copy the data directly from the
597 * user into the GTT, uncached.
598 */
Eric Anholt673a3942008-07-30 12:06:12 -0700599static int
Chris Wilson05394f32010-11-08 19:18:58 +0000600i915_gem_gtt_pwrite_fast(struct drm_device *dev,
601 struct drm_i915_gem_object *obj,
Eric Anholt3de09aa2009-03-09 09:42:23 -0700602 struct drm_i915_gem_pwrite *args,
Chris Wilson05394f32010-11-08 19:18:58 +0000603 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -0700604{
Keith Packard0839ccb2008-10-30 19:38:48 -0700605 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700606 ssize_t remain;
Keith Packard0839ccb2008-10-30 19:38:48 -0700607 loff_t offset, page_base;
Eric Anholt673a3942008-07-30 12:06:12 -0700608 char __user *user_data;
Daniel Vetter935aaa62012-03-25 19:47:35 +0200609 int page_offset, page_length, ret;
610
Chris Wilson86a1ee22012-08-11 15:41:04 +0100611 ret = i915_gem_object_pin(obj, 0, true, true);
Daniel Vetter935aaa62012-03-25 19:47:35 +0200612 if (ret)
613 goto out;
614
615 ret = i915_gem_object_set_to_gtt_domain(obj, true);
616 if (ret)
617 goto out_unpin;
618
619 ret = i915_gem_object_put_fence(obj);
620 if (ret)
621 goto out_unpin;
Eric Anholt673a3942008-07-30 12:06:12 -0700622
623 user_data = (char __user *) (uintptr_t) args->data_ptr;
624 remain = args->size;
Eric Anholt673a3942008-07-30 12:06:12 -0700625
Chris Wilson05394f32010-11-08 19:18:58 +0000626 offset = obj->gtt_offset + args->offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700627
628 while (remain > 0) {
629 /* Operation in this page
630 *
Keith Packard0839ccb2008-10-30 19:38:48 -0700631 * page_base = page offset within aperture
632 * page_offset = offset within page
633 * page_length = bytes to copy for this page
Eric Anholt673a3942008-07-30 12:06:12 -0700634 */
Chris Wilsonc8cbbb82011-05-12 22:17:11 +0100635 page_base = offset & PAGE_MASK;
636 page_offset = offset_in_page(offset);
Keith Packard0839ccb2008-10-30 19:38:48 -0700637 page_length = remain;
638 if ((page_offset + remain) > PAGE_SIZE)
639 page_length = PAGE_SIZE - page_offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700640
Keith Packard0839ccb2008-10-30 19:38:48 -0700641 /* If we get a fault while copying data, then (presumably) our
Eric Anholt3de09aa2009-03-09 09:42:23 -0700642 * source page isn't available. Return the error and we'll
643 * retry in the slow path.
Keith Packard0839ccb2008-10-30 19:38:48 -0700644 */
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100645 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
Daniel Vetter935aaa62012-03-25 19:47:35 +0200646 page_offset, user_data, page_length)) {
647 ret = -EFAULT;
648 goto out_unpin;
649 }
Eric Anholt673a3942008-07-30 12:06:12 -0700650
Keith Packard0839ccb2008-10-30 19:38:48 -0700651 remain -= page_length;
652 user_data += page_length;
653 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700654 }
Eric Anholt673a3942008-07-30 12:06:12 -0700655
Daniel Vetter935aaa62012-03-25 19:47:35 +0200656out_unpin:
657 i915_gem_object_unpin(obj);
658out:
Eric Anholt3de09aa2009-03-09 09:42:23 -0700659 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700660}
661
Daniel Vetterd174bd62012-03-25 19:47:40 +0200662/* Per-page copy function for the shmem pwrite fastpath.
663 * Flushes invalid cachelines before writing to the target if
664 * needs_clflush_before is set and flushes out any written cachelines after
665 * writing if needs_clflush is set. */
Eric Anholt673a3942008-07-30 12:06:12 -0700666static int
Daniel Vetterd174bd62012-03-25 19:47:40 +0200667shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
668 char __user *user_data,
669 bool page_do_bit17_swizzling,
670 bool needs_clflush_before,
671 bool needs_clflush_after)
Eric Anholt673a3942008-07-30 12:06:12 -0700672{
Daniel Vetterd174bd62012-03-25 19:47:40 +0200673 char *vaddr;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700674 int ret;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700675
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200676 if (unlikely(page_do_bit17_swizzling))
Daniel Vetterd174bd62012-03-25 19:47:40 +0200677 return -EINVAL;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700678
Daniel Vetterd174bd62012-03-25 19:47:40 +0200679 vaddr = kmap_atomic(page);
680 if (needs_clflush_before)
681 drm_clflush_virt_range(vaddr + shmem_page_offset,
682 page_length);
683 ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
684 user_data,
685 page_length);
686 if (needs_clflush_after)
687 drm_clflush_virt_range(vaddr + shmem_page_offset,
688 page_length);
689 kunmap_atomic(vaddr);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700690
Chris Wilson755d2212012-09-04 21:02:55 +0100691 return ret ? -EFAULT : 0;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700692}
693
Daniel Vetterd174bd62012-03-25 19:47:40 +0200694/* Only difference to the fast-path function is that this can handle bit17
695 * and uses non-atomic copy and kmap functions. */
Eric Anholt3043c602008-10-02 12:24:47 -0700696static int
Daniel Vetterd174bd62012-03-25 19:47:40 +0200697shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
698 char __user *user_data,
699 bool page_do_bit17_swizzling,
700 bool needs_clflush_before,
701 bool needs_clflush_after)
Eric Anholt673a3942008-07-30 12:06:12 -0700702{
Daniel Vetterd174bd62012-03-25 19:47:40 +0200703 char *vaddr;
704 int ret;
Eric Anholt40123c12009-03-09 13:42:30 -0700705
Daniel Vetterd174bd62012-03-25 19:47:40 +0200706 vaddr = kmap(page);
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200707 if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
Daniel Vetter23c18c72012-03-25 19:47:42 +0200708 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
709 page_length,
710 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200711 if (page_do_bit17_swizzling)
712 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
Chris Wilsone5281cc2010-10-28 13:45:36 +0100713 user_data,
714 page_length);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200715 else
716 ret = __copy_from_user(vaddr + shmem_page_offset,
717 user_data,
718 page_length);
719 if (needs_clflush_after)
Daniel Vetter23c18c72012-03-25 19:47:42 +0200720 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
721 page_length,
722 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200723 kunmap(page);
Chris Wilsone5281cc2010-10-28 13:45:36 +0100724
Chris Wilson755d2212012-09-04 21:02:55 +0100725 return ret ? -EFAULT : 0;
Eric Anholt40123c12009-03-09 13:42:30 -0700726}
727
Eric Anholt40123c12009-03-09 13:42:30 -0700728static int
Daniel Vettere244a442012-03-25 19:47:28 +0200729i915_gem_shmem_pwrite(struct drm_device *dev,
730 struct drm_i915_gem_object *obj,
731 struct drm_i915_gem_pwrite *args,
732 struct drm_file *file)
Eric Anholt40123c12009-03-09 13:42:30 -0700733{
Eric Anholt40123c12009-03-09 13:42:30 -0700734 ssize_t remain;
Daniel Vetter8c599672011-12-14 13:57:31 +0100735 loff_t offset;
736 char __user *user_data;
Ben Widawskyeb2c0c82012-02-15 14:42:43 +0100737 int shmem_page_offset, page_length, ret = 0;
Daniel Vetter8c599672011-12-14 13:57:31 +0100738 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
Daniel Vettere244a442012-03-25 19:47:28 +0200739 int hit_slowpath = 0;
Daniel Vetter58642882012-03-25 19:47:37 +0200740 int needs_clflush_after = 0;
741 int needs_clflush_before = 0;
Chris Wilson9da3da62012-06-01 15:20:22 +0100742 int i;
743 struct scatterlist *sg;
Eric Anholt40123c12009-03-09 13:42:30 -0700744
Daniel Vetter8c599672011-12-14 13:57:31 +0100745 user_data = (char __user *) (uintptr_t) args->data_ptr;
Eric Anholt40123c12009-03-09 13:42:30 -0700746 remain = args->size;
747
Daniel Vetter8c599672011-12-14 13:57:31 +0100748 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700749
Daniel Vetter58642882012-03-25 19:47:37 +0200750 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
751 /* If we're not in the cpu write domain, set ourself into the gtt
752 * write domain and manually flush cachelines (if required). This
753 * optimizes for the case when the gpu will use the data
754 * right away and we therefore have to clflush anyway. */
755 if (obj->cache_level == I915_CACHE_NONE)
756 needs_clflush_after = 1;
Chris Wilson6c085a72012-08-20 11:40:46 +0200757 if (obj->gtt_space) {
758 ret = i915_gem_object_set_to_gtt_domain(obj, true);
759 if (ret)
760 return ret;
761 }
Daniel Vetter58642882012-03-25 19:47:37 +0200762 }
763 /* Same trick applies for invalidate partially written cachelines before
764 * writing. */
765 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
766 && obj->cache_level == I915_CACHE_NONE)
767 needs_clflush_before = 1;
768
Chris Wilson755d2212012-09-04 21:02:55 +0100769 ret = i915_gem_object_get_pages(obj);
770 if (ret)
771 return ret;
772
773 i915_gem_object_pin_pages(obj);
774
Eric Anholt40123c12009-03-09 13:42:30 -0700775 offset = args->offset;
Chris Wilson05394f32010-11-08 19:18:58 +0000776 obj->dirty = 1;
Eric Anholt40123c12009-03-09 13:42:30 -0700777
Chris Wilson9da3da62012-06-01 15:20:22 +0100778 for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
Chris Wilsone5281cc2010-10-28 13:45:36 +0100779 struct page *page;
Daniel Vetter58642882012-03-25 19:47:37 +0200780 int partial_cacheline_write;
Chris Wilsone5281cc2010-10-28 13:45:36 +0100781
Chris Wilson9da3da62012-06-01 15:20:22 +0100782 if (i < offset >> PAGE_SHIFT)
783 continue;
784
785 if (remain <= 0)
786 break;
787
Eric Anholt40123c12009-03-09 13:42:30 -0700788 /* Operation in this page
789 *
Eric Anholt40123c12009-03-09 13:42:30 -0700790 * shmem_page_offset = offset within page in shmem file
Eric Anholt40123c12009-03-09 13:42:30 -0700791 * page_length = bytes to copy for this page
792 */
Chris Wilsonc8cbbb82011-05-12 22:17:11 +0100793 shmem_page_offset = offset_in_page(offset);
Eric Anholt40123c12009-03-09 13:42:30 -0700794
795 page_length = remain;
796 if ((shmem_page_offset + page_length) > PAGE_SIZE)
797 page_length = PAGE_SIZE - shmem_page_offset;
Eric Anholt40123c12009-03-09 13:42:30 -0700798
Daniel Vetter58642882012-03-25 19:47:37 +0200799 /* If we don't overwrite a cacheline completely we need to be
800 * careful to have up-to-date data by first clflushing. Don't
801 * overcomplicate things and flush the entire patch. */
802 partial_cacheline_write = needs_clflush_before &&
803 ((shmem_page_offset | page_length)
804 & (boot_cpu_data.x86_clflush_size - 1));
805
Chris Wilson9da3da62012-06-01 15:20:22 +0100806 page = sg_page(sg);
Daniel Vetter8c599672011-12-14 13:57:31 +0100807 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
808 (page_to_phys(page) & (1 << 17)) != 0;
809
Daniel Vetterd174bd62012-03-25 19:47:40 +0200810 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
811 user_data, page_do_bit17_swizzling,
812 partial_cacheline_write,
813 needs_clflush_after);
814 if (ret == 0)
815 goto next_page;
Eric Anholt40123c12009-03-09 13:42:30 -0700816
Daniel Vettere244a442012-03-25 19:47:28 +0200817 hit_slowpath = 1;
Daniel Vettere244a442012-03-25 19:47:28 +0200818 mutex_unlock(&dev->struct_mutex);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200819 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
820 user_data, page_do_bit17_swizzling,
821 partial_cacheline_write,
822 needs_clflush_after);
Eric Anholt40123c12009-03-09 13:42:30 -0700823
Daniel Vettere244a442012-03-25 19:47:28 +0200824 mutex_lock(&dev->struct_mutex);
Chris Wilson755d2212012-09-04 21:02:55 +0100825
Daniel Vettere244a442012-03-25 19:47:28 +0200826next_page:
Chris Wilsone5281cc2010-10-28 13:45:36 +0100827 set_page_dirty(page);
828 mark_page_accessed(page);
Chris Wilsone5281cc2010-10-28 13:45:36 +0100829
Chris Wilson755d2212012-09-04 21:02:55 +0100830 if (ret)
Daniel Vetter8c599672011-12-14 13:57:31 +0100831 goto out;
Daniel Vetter8c599672011-12-14 13:57:31 +0100832
Eric Anholt40123c12009-03-09 13:42:30 -0700833 remain -= page_length;
Daniel Vetter8c599672011-12-14 13:57:31 +0100834 user_data += page_length;
Eric Anholt40123c12009-03-09 13:42:30 -0700835 offset += page_length;
836 }
837
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100838out:
Chris Wilson755d2212012-09-04 21:02:55 +0100839 i915_gem_object_unpin_pages(obj);
840
Daniel Vettere244a442012-03-25 19:47:28 +0200841 if (hit_slowpath) {
842 /* Fixup: Kill any reinstated backing storage pages */
843 if (obj->madv == __I915_MADV_PURGED)
844 i915_gem_object_truncate(obj);
845 /* and flush dirty cachelines in case the object isn't in the cpu write
846 * domain anymore. */
847 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
848 i915_gem_clflush_object(obj);
849 intel_gtt_chipset_flush();
850 }
Daniel Vetter8c599672011-12-14 13:57:31 +0100851 }
Eric Anholt40123c12009-03-09 13:42:30 -0700852
Daniel Vetter58642882012-03-25 19:47:37 +0200853 if (needs_clflush_after)
854 intel_gtt_chipset_flush();
855
Eric Anholt40123c12009-03-09 13:42:30 -0700856 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700857}
858
859/**
860 * Writes data to the object referenced by handle.
861 *
862 * On error, the contents of the buffer that were to be modified are undefined.
863 */
864int
865i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100866 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -0700867{
868 struct drm_i915_gem_pwrite *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +0000869 struct drm_i915_gem_object *obj;
Chris Wilson51311d02010-11-17 09:10:42 +0000870 int ret;
871
872 if (args->size == 0)
873 return 0;
874
875 if (!access_ok(VERIFY_READ,
876 (char __user *)(uintptr_t)args->data_ptr,
877 args->size))
878 return -EFAULT;
879
Daniel Vetterf56f8212012-03-25 19:47:41 +0200880 ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
881 args->size);
Chris Wilson51311d02010-11-17 09:10:42 +0000882 if (ret)
883 return -EFAULT;
Eric Anholt673a3942008-07-30 12:06:12 -0700884
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100885 ret = i915_mutex_lock_interruptible(dev);
886 if (ret)
887 return ret;
888
Chris Wilson05394f32010-11-08 19:18:58 +0000889 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +0000890 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100891 ret = -ENOENT;
892 goto unlock;
893 }
Eric Anholt673a3942008-07-30 12:06:12 -0700894
Chris Wilson7dcd2492010-09-26 20:21:44 +0100895 /* Bounds check destination. */
Chris Wilson05394f32010-11-08 19:18:58 +0000896 if (args->offset > obj->base.size ||
897 args->size > obj->base.size - args->offset) {
Chris Wilsonce9d4192010-09-26 20:50:05 +0100898 ret = -EINVAL;
Chris Wilson35b62a82010-09-26 20:23:38 +0100899 goto out;
Chris Wilsonce9d4192010-09-26 20:50:05 +0100900 }
901
Daniel Vetter1286ff72012-05-10 15:25:09 +0200902 /* prime objects have no backing filp to GEM pread/pwrite
903 * pages from.
904 */
905 if (!obj->base.filp) {
906 ret = -EINVAL;
907 goto out;
908 }
909
Chris Wilsondb53a302011-02-03 11:57:46 +0000910 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
911
Daniel Vetter935aaa62012-03-25 19:47:35 +0200912 ret = -EFAULT;
Eric Anholt673a3942008-07-30 12:06:12 -0700913 /* We can only do the GTT pwrite on untiled buffers, as otherwise
914 * it would end up going through the fenced access, and we'll get
915 * different detiling behavior between reading and writing.
916 * pread/pwrite currently are reading and writing from the CPU
917 * perspective, requiring manual detiling by the client.
918 */
Daniel Vetter5c0480f2011-12-14 13:57:30 +0100919 if (obj->phys_obj) {
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100920 ret = i915_gem_phys_pwrite(dev, obj, args, file);
Daniel Vetter5c0480f2011-12-14 13:57:30 +0100921 goto out;
922 }
923
Chris Wilson86a1ee22012-08-11 15:41:04 +0100924 if (obj->cache_level == I915_CACHE_NONE &&
Daniel Vetterc07496f2012-04-13 15:51:51 +0200925 obj->tiling_mode == I915_TILING_NONE &&
Daniel Vetter5c0480f2011-12-14 13:57:30 +0100926 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100927 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
Daniel Vetter935aaa62012-03-25 19:47:35 +0200928 /* Note that the gtt paths might fail with non-page-backed user
929 * pointers (e.g. gtt mappings when moving data between
930 * textures). Fallback to the shmem path in that case. */
Eric Anholt40123c12009-03-09 13:42:30 -0700931 }
Eric Anholt673a3942008-07-30 12:06:12 -0700932
Chris Wilson86a1ee22012-08-11 15:41:04 +0100933 if (ret == -EFAULT || ret == -ENOSPC)
Daniel Vetter935aaa62012-03-25 19:47:35 +0200934 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
Daniel Vetter5c0480f2011-12-14 13:57:30 +0100935
Chris Wilson35b62a82010-09-26 20:23:38 +0100936out:
Chris Wilson05394f32010-11-08 19:18:58 +0000937 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100938unlock:
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100939 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -0700940 return ret;
941}
942
Chris Wilsonb3612372012-08-24 09:35:08 +0100943int
944i915_gem_check_wedge(struct drm_i915_private *dev_priv,
945 bool interruptible)
946{
947 if (atomic_read(&dev_priv->mm.wedged)) {
948 struct completion *x = &dev_priv->error_completion;
949 bool recovery_complete;
950 unsigned long flags;
951
952 /* Give the error handler a chance to run. */
953 spin_lock_irqsave(&x->wait.lock, flags);
954 recovery_complete = x->done > 0;
955 spin_unlock_irqrestore(&x->wait.lock, flags);
956
957 /* Non-interruptible callers can't handle -EAGAIN, hence return
958 * -EIO unconditionally for these. */
959 if (!interruptible)
960 return -EIO;
961
962 /* Recovery complete, but still wedged means reset failure. */
963 if (recovery_complete)
964 return -EIO;
965
966 return -EAGAIN;
967 }
968
969 return 0;
970}
971
972/*
973 * Compare seqno against outstanding lazy request. Emit a request if they are
974 * equal.
975 */
976static int
977i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
978{
979 int ret;
980
981 BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
982
983 ret = 0;
984 if (seqno == ring->outstanding_lazy_request)
985 ret = i915_add_request(ring, NULL, NULL);
986
987 return ret;
988}
989
990/**
991 * __wait_seqno - wait until execution of seqno has finished
992 * @ring: the ring expected to report seqno
993 * @seqno: duh!
994 * @interruptible: do an interruptible wait (normally yes)
995 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
996 *
997 * Returns 0 if the seqno was found within the alloted time. Else returns the
998 * errno with remaining time filled in timeout argument.
999 */
1000static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
1001 bool interruptible, struct timespec *timeout)
1002{
1003 drm_i915_private_t *dev_priv = ring->dev->dev_private;
1004 struct timespec before, now, wait_time={1,0};
1005 unsigned long timeout_jiffies;
1006 long end;
1007 bool wait_forever = true;
1008 int ret;
1009
1010 if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
1011 return 0;
1012
1013 trace_i915_gem_request_wait_begin(ring, seqno);
1014
1015 if (timeout != NULL) {
1016 wait_time = *timeout;
1017 wait_forever = false;
1018 }
1019
1020 timeout_jiffies = timespec_to_jiffies(&wait_time);
1021
1022 if (WARN_ON(!ring->irq_get(ring)))
1023 return -ENODEV;
1024
1025 /* Record current time in case interrupted by signal, or wedged * */
1026 getrawmonotonic(&before);
1027
1028#define EXIT_COND \
1029 (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
1030 atomic_read(&dev_priv->mm.wedged))
1031 do {
1032 if (interruptible)
1033 end = wait_event_interruptible_timeout(ring->irq_queue,
1034 EXIT_COND,
1035 timeout_jiffies);
1036 else
1037 end = wait_event_timeout(ring->irq_queue, EXIT_COND,
1038 timeout_jiffies);
1039
1040 ret = i915_gem_check_wedge(dev_priv, interruptible);
1041 if (ret)
1042 end = ret;
1043 } while (end == 0 && wait_forever);
1044
1045 getrawmonotonic(&now);
1046
1047 ring->irq_put(ring);
1048 trace_i915_gem_request_wait_end(ring, seqno);
1049#undef EXIT_COND
1050
1051 if (timeout) {
1052 struct timespec sleep_time = timespec_sub(now, before);
1053 *timeout = timespec_sub(*timeout, sleep_time);
1054 }
1055
1056 switch (end) {
1057 case -EIO:
1058 case -EAGAIN: /* Wedged */
1059 case -ERESTARTSYS: /* Signal */
1060 return (int)end;
1061 case 0: /* Timeout */
1062 if (timeout)
1063 set_normalized_timespec(timeout, 0, 0);
1064 return -ETIME;
1065 default: /* Completed */
1066 WARN_ON(end < 0); /* We're not aware of other errors */
1067 return 0;
1068 }
1069}
1070
1071/**
1072 * Waits for a sequence number to be signaled, and cleans up the
1073 * request and object lists appropriately for that event.
1074 */
1075int
1076i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
1077{
1078 struct drm_device *dev = ring->dev;
1079 struct drm_i915_private *dev_priv = dev->dev_private;
1080 bool interruptible = dev_priv->mm.interruptible;
1081 int ret;
1082
1083 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1084 BUG_ON(seqno == 0);
1085
1086 ret = i915_gem_check_wedge(dev_priv, interruptible);
1087 if (ret)
1088 return ret;
1089
1090 ret = i915_gem_check_olr(ring, seqno);
1091 if (ret)
1092 return ret;
1093
1094 return __wait_seqno(ring, seqno, interruptible, NULL);
1095}
1096
1097/**
1098 * Ensures that all rendering to the object has completed and the object is
1099 * safe to unbind from the GTT or access from the CPU.
1100 */
1101static __must_check int
1102i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1103 bool readonly)
1104{
1105 struct intel_ring_buffer *ring = obj->ring;
1106 u32 seqno;
1107 int ret;
1108
1109 seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1110 if (seqno == 0)
1111 return 0;
1112
1113 ret = i915_wait_seqno(ring, seqno);
1114 if (ret)
1115 return ret;
1116
1117 i915_gem_retire_requests_ring(ring);
1118
1119 /* Manually manage the write flush as we may have not yet
1120 * retired the buffer.
1121 */
1122 if (obj->last_write_seqno &&
1123 i915_seqno_passed(seqno, obj->last_write_seqno)) {
1124 obj->last_write_seqno = 0;
1125 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1126 }
1127
1128 return 0;
1129}
1130
Chris Wilson3236f572012-08-24 09:35:09 +01001131/* A nonblocking variant of the above wait. This is a highly dangerous routine
1132 * as the object state may change during this call.
1133 */
1134static __must_check int
1135i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1136 bool readonly)
1137{
1138 struct drm_device *dev = obj->base.dev;
1139 struct drm_i915_private *dev_priv = dev->dev_private;
1140 struct intel_ring_buffer *ring = obj->ring;
1141 u32 seqno;
1142 int ret;
1143
1144 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1145 BUG_ON(!dev_priv->mm.interruptible);
1146
1147 seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1148 if (seqno == 0)
1149 return 0;
1150
1151 ret = i915_gem_check_wedge(dev_priv, true);
1152 if (ret)
1153 return ret;
1154
1155 ret = i915_gem_check_olr(ring, seqno);
1156 if (ret)
1157 return ret;
1158
1159 mutex_unlock(&dev->struct_mutex);
1160 ret = __wait_seqno(ring, seqno, true, NULL);
1161 mutex_lock(&dev->struct_mutex);
1162
1163 i915_gem_retire_requests_ring(ring);
1164
1165 /* Manually manage the write flush as we may have not yet
1166 * retired the buffer.
1167 */
1168 if (obj->last_write_seqno &&
1169 i915_seqno_passed(seqno, obj->last_write_seqno)) {
1170 obj->last_write_seqno = 0;
1171 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1172 }
1173
1174 return ret;
1175}
1176
Eric Anholt673a3942008-07-30 12:06:12 -07001177/**
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001178 * Called when user space prepares to use an object with the CPU, either
1179 * through the mmap ioctl's mapping or a GTT mapping.
Eric Anholt673a3942008-07-30 12:06:12 -07001180 */
1181int
1182i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001183 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001184{
1185 struct drm_i915_gem_set_domain *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001186 struct drm_i915_gem_object *obj;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001187 uint32_t read_domains = args->read_domains;
1188 uint32_t write_domain = args->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -07001189 int ret;
1190
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001191 /* Only handle setting domains to types used by the CPU. */
Chris Wilson21d509e2009-06-06 09:46:02 +01001192 if (write_domain & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001193 return -EINVAL;
1194
Chris Wilson21d509e2009-06-06 09:46:02 +01001195 if (read_domains & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001196 return -EINVAL;
1197
1198 /* Having something in the write domain implies it's in the read
1199 * domain, and only that read domain. Enforce that in the request.
1200 */
1201 if (write_domain != 0 && read_domains != write_domain)
1202 return -EINVAL;
1203
Chris Wilson76c1dec2010-09-25 11:22:51 +01001204 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001205 if (ret)
Chris Wilson76c1dec2010-09-25 11:22:51 +01001206 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07001207
Chris Wilson05394f32010-11-08 19:18:58 +00001208 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00001209 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001210 ret = -ENOENT;
1211 goto unlock;
Chris Wilson76c1dec2010-09-25 11:22:51 +01001212 }
Jesse Barnes652c3932009-08-17 13:31:43 -07001213
Chris Wilson3236f572012-08-24 09:35:09 +01001214 /* Try to flush the object off the GPU without holding the lock.
1215 * We will repeat the flush holding the lock in the normal manner
1216 * to catch cases where we are gazumped.
1217 */
1218 ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
1219 if (ret)
1220 goto unref;
1221
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001222 if (read_domains & I915_GEM_DOMAIN_GTT) {
1223 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
Eric Anholt02354392008-11-26 13:58:13 -08001224
1225 /* Silently promote "you're not bound, there was nothing to do"
1226 * to success, since the client was just asking us to
1227 * make sure everything was done.
1228 */
1229 if (ret == -EINVAL)
1230 ret = 0;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001231 } else {
Eric Anholte47c68e2008-11-14 13:35:19 -08001232 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001233 }
1234
Chris Wilson3236f572012-08-24 09:35:09 +01001235unref:
Chris Wilson05394f32010-11-08 19:18:58 +00001236 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001237unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07001238 mutex_unlock(&dev->struct_mutex);
1239 return ret;
1240}
1241
1242/**
1243 * Called when user space has done writes to this buffer
1244 */
1245int
1246i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001247 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001248{
1249 struct drm_i915_gem_sw_finish *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001250 struct drm_i915_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07001251 int ret = 0;
1252
Chris Wilson76c1dec2010-09-25 11:22:51 +01001253 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001254 if (ret)
Chris Wilson76c1dec2010-09-25 11:22:51 +01001255 return ret;
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001256
Chris Wilson05394f32010-11-08 19:18:58 +00001257 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00001258 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001259 ret = -ENOENT;
1260 goto unlock;
Eric Anholt673a3942008-07-30 12:06:12 -07001261 }
1262
Eric Anholt673a3942008-07-30 12:06:12 -07001263 /* Pinned buffers may be scanout, so flush the cache */
Chris Wilson05394f32010-11-08 19:18:58 +00001264 if (obj->pin_count)
Eric Anholte47c68e2008-11-14 13:35:19 -08001265 i915_gem_object_flush_cpu_write_domain(obj);
1266
Chris Wilson05394f32010-11-08 19:18:58 +00001267 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001268unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07001269 mutex_unlock(&dev->struct_mutex);
1270 return ret;
1271}
1272
1273/**
1274 * Maps the contents of an object, returning the address it is mapped
1275 * into.
1276 *
1277 * While the mapping holds a reference on the contents of the object, it doesn't
1278 * imply a ref on the object itself.
1279 */
1280int
1281i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001282 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001283{
1284 struct drm_i915_gem_mmap *args = data;
1285 struct drm_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07001286 unsigned long addr;
1287
Chris Wilson05394f32010-11-08 19:18:58 +00001288 obj = drm_gem_object_lookup(dev, file, args->handle);
Eric Anholt673a3942008-07-30 12:06:12 -07001289 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001290 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001291
Daniel Vetter1286ff72012-05-10 15:25:09 +02001292 /* prime objects have no backing filp to GEM mmap
1293 * pages from.
1294 */
1295 if (!obj->filp) {
1296 drm_gem_object_unreference_unlocked(obj);
1297 return -EINVAL;
1298 }
1299
Linus Torvalds6be5ceb2012-04-20 17:13:58 -07001300 addr = vm_mmap(obj->filp, 0, args->size,
Eric Anholt673a3942008-07-30 12:06:12 -07001301 PROT_READ | PROT_WRITE, MAP_SHARED,
1302 args->offset);
Luca Barbieribc9025b2010-02-09 05:49:12 +00001303 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001304 if (IS_ERR((void *)addr))
1305 return addr;
1306
1307 args->addr_ptr = (uint64_t) addr;
1308
1309 return 0;
1310}
1311
Jesse Barnesde151cf2008-11-12 10:03:55 -08001312/**
1313 * i915_gem_fault - fault a page into the GTT
1314 * vma: VMA in question
1315 * vmf: fault info
1316 *
1317 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1318 * from userspace. The fault handler takes care of binding the object to
1319 * the GTT (if needed), allocating and programming a fence register (again,
1320 * only if needed based on whether the old reg is still valid or the object
1321 * is tiled) and inserting a new PTE into the faulting process.
1322 *
1323 * Note that the faulting process may involve evicting existing objects
1324 * from the GTT and/or fence registers to make room. So performance may
1325 * suffer if the GTT working set is large or there are few fence registers
1326 * left.
1327 */
1328int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1329{
Chris Wilson05394f32010-11-08 19:18:58 +00001330 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1331 struct drm_device *dev = obj->base.dev;
Chris Wilson7d1c4802010-08-07 21:45:03 +01001332 drm_i915_private_t *dev_priv = dev->dev_private;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001333 pgoff_t page_offset;
1334 unsigned long pfn;
1335 int ret = 0;
Jesse Barnes0f973f22009-01-26 17:10:45 -08001336 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001337
1338 /* We don't use vmf->pgoff since that has the fake offset */
1339 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1340 PAGE_SHIFT;
1341
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001342 ret = i915_mutex_lock_interruptible(dev);
1343 if (ret)
1344 goto out;
Chris Wilsona00b10c2010-09-24 21:15:47 +01001345
Chris Wilsondb53a302011-02-03 11:57:46 +00001346 trace_i915_gem_object_fault(obj, page_offset, true, write);
1347
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001348 /* Now bind it into the GTT if needed */
Chris Wilson919926a2010-11-12 13:42:53 +00001349 if (!obj->map_and_fenceable) {
1350 ret = i915_gem_object_unbind(obj);
1351 if (ret)
1352 goto unlock;
Chris Wilsona00b10c2010-09-24 21:15:47 +01001353 }
Chris Wilson05394f32010-11-08 19:18:58 +00001354 if (!obj->gtt_space) {
Chris Wilson86a1ee22012-08-11 15:41:04 +01001355 ret = i915_gem_object_bind_to_gtt(obj, 0, true, false);
Chris Wilsonc7150892009-09-23 00:43:56 +01001356 if (ret)
1357 goto unlock;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001358
Eric Anholte92d03b2011-06-14 16:43:09 -07001359 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1360 if (ret)
1361 goto unlock;
1362 }
Chris Wilson4a684a42010-10-28 14:44:08 +01001363
Daniel Vetter74898d72012-02-15 23:50:22 +01001364 if (!obj->has_global_gtt_mapping)
1365 i915_gem_gtt_bind_object(obj, obj->cache_level);
1366
Chris Wilson06d98132012-04-17 15:31:24 +01001367 ret = i915_gem_object_get_fence(obj);
Chris Wilsond9e86c02010-11-10 16:40:20 +00001368 if (ret)
1369 goto unlock;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001370
Chris Wilson05394f32010-11-08 19:18:58 +00001371 if (i915_gem_object_is_inactive(obj))
1372 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
Chris Wilson7d1c4802010-08-07 21:45:03 +01001373
Chris Wilson6299f992010-11-24 12:23:44 +00001374 obj->fault_mappable = true;
1375
Daniel Vetterdd2757f2012-06-07 15:55:57 +02001376 pfn = ((dev_priv->mm.gtt_base_addr + obj->gtt_offset) >> PAGE_SHIFT) +
Jesse Barnesde151cf2008-11-12 10:03:55 -08001377 page_offset;
1378
1379 /* Finally, remap it using the new GTT offset */
1380 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
Chris Wilsonc7150892009-09-23 00:43:56 +01001381unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001382 mutex_unlock(&dev->struct_mutex);
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001383out:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001384 switch (ret) {
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001385 case -EIO:
Daniel Vettera9340cc2012-07-04 22:18:42 +02001386 /* If this -EIO is due to a gpu hang, give the reset code a
1387 * chance to clean up the mess. Otherwise return the proper
1388 * SIGBUS. */
1389 if (!atomic_read(&dev_priv->mm.wedged))
1390 return VM_FAULT_SIGBUS;
Chris Wilson045e7692010-11-07 09:18:22 +00001391 case -EAGAIN:
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001392 /* Give the error handler a chance to run and move the
1393 * objects off the GPU active list. Next time we service the
1394 * fault, we should be able to transition the page into the
1395 * GTT without touching the GPU (and so avoid further
1396 * EIO/EGAIN). If the GPU is wedged, then there is no issue
1397 * with coherency, just lost writes.
1398 */
Chris Wilson045e7692010-11-07 09:18:22 +00001399 set_need_resched();
Chris Wilsonc7150892009-09-23 00:43:56 +01001400 case 0:
1401 case -ERESTARTSYS:
Chris Wilsonbed636a2011-02-11 20:31:19 +00001402 case -EINTR:
Dmitry Rogozhkine79e0fe2012-10-03 17:15:26 +03001403 case -EBUSY:
1404 /*
1405 * EBUSY is ok: this just means that another thread
1406 * already did the job.
1407 */
Chris Wilsonc7150892009-09-23 00:43:56 +01001408 return VM_FAULT_NOPAGE;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001409 case -ENOMEM:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001410 return VM_FAULT_OOM;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001411 default:
Chris Wilsonc7150892009-09-23 00:43:56 +01001412 return VM_FAULT_SIGBUS;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001413 }
1414}
1415
1416/**
Chris Wilson901782b2009-07-10 08:18:50 +01001417 * i915_gem_release_mmap - remove physical page mappings
1418 * @obj: obj in question
1419 *
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02001420 * Preserve the reservation of the mmapping with the DRM core code, but
Chris Wilson901782b2009-07-10 08:18:50 +01001421 * relinquish ownership of the pages back to the system.
1422 *
1423 * It is vital that we remove the page mapping if we have mapped a tiled
1424 * object through the GTT and then lose the fence register due to
1425 * resource pressure. Similarly if the object has been moved out of the
1426 * aperture, than pages mapped into userspace must be revoked. Removing the
1427 * mapping will then trigger a page fault on the next user access, allowing
1428 * fixup by i915_gem_fault().
1429 */
Eric Anholtd05ca302009-07-10 13:02:26 -07001430void
Chris Wilson05394f32010-11-08 19:18:58 +00001431i915_gem_release_mmap(struct drm_i915_gem_object *obj)
Chris Wilson901782b2009-07-10 08:18:50 +01001432{
Chris Wilson6299f992010-11-24 12:23:44 +00001433 if (!obj->fault_mappable)
1434 return;
Chris Wilson901782b2009-07-10 08:18:50 +01001435
Chris Wilsonf6e47882011-03-20 21:09:12 +00001436 if (obj->base.dev->dev_mapping)
1437 unmap_mapping_range(obj->base.dev->dev_mapping,
1438 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1439 obj->base.size, 1);
Daniel Vetterfb7d5162010-10-01 22:05:20 +02001440
Chris Wilson6299f992010-11-24 12:23:44 +00001441 obj->fault_mappable = false;
Chris Wilson901782b2009-07-10 08:18:50 +01001442}
1443
Chris Wilson92b88ae2010-11-09 11:47:32 +00001444static uint32_t
Chris Wilsone28f8712011-07-18 13:11:49 -07001445i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
Chris Wilson92b88ae2010-11-09 11:47:32 +00001446{
Chris Wilsone28f8712011-07-18 13:11:49 -07001447 uint32_t gtt_size;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001448
1449 if (INTEL_INFO(dev)->gen >= 4 ||
Chris Wilsone28f8712011-07-18 13:11:49 -07001450 tiling_mode == I915_TILING_NONE)
1451 return size;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001452
1453 /* Previous chips need a power-of-two fence region when tiling */
1454 if (INTEL_INFO(dev)->gen == 3)
Chris Wilsone28f8712011-07-18 13:11:49 -07001455 gtt_size = 1024*1024;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001456 else
Chris Wilsone28f8712011-07-18 13:11:49 -07001457 gtt_size = 512*1024;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001458
Chris Wilsone28f8712011-07-18 13:11:49 -07001459 while (gtt_size < size)
1460 gtt_size <<= 1;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001461
Chris Wilsone28f8712011-07-18 13:11:49 -07001462 return gtt_size;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001463}
1464
Jesse Barnesde151cf2008-11-12 10:03:55 -08001465/**
1466 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1467 * @obj: object to check
1468 *
1469 * Return the required GTT alignment for an object, taking into account
Daniel Vetter5e783302010-11-14 22:32:36 +01001470 * potential fence register mapping.
Jesse Barnesde151cf2008-11-12 10:03:55 -08001471 */
1472static uint32_t
Chris Wilsone28f8712011-07-18 13:11:49 -07001473i915_gem_get_gtt_alignment(struct drm_device *dev,
1474 uint32_t size,
1475 int tiling_mode)
Jesse Barnesde151cf2008-11-12 10:03:55 -08001476{
Jesse Barnesde151cf2008-11-12 10:03:55 -08001477 /*
1478 * Minimum alignment is 4k (GTT page size), but might be greater
1479 * if a fence register is needed for the object.
1480 */
Chris Wilsona00b10c2010-09-24 21:15:47 +01001481 if (INTEL_INFO(dev)->gen >= 4 ||
Chris Wilsone28f8712011-07-18 13:11:49 -07001482 tiling_mode == I915_TILING_NONE)
Jesse Barnesde151cf2008-11-12 10:03:55 -08001483 return 4096;
1484
1485 /*
1486 * Previous chips need to be aligned to the size of the smallest
1487 * fence register that can contain the object.
1488 */
Chris Wilsone28f8712011-07-18 13:11:49 -07001489 return i915_gem_get_gtt_size(dev, size, tiling_mode);
Chris Wilsona00b10c2010-09-24 21:15:47 +01001490}
1491
Daniel Vetter5e783302010-11-14 22:32:36 +01001492/**
1493 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1494 * unfenced object
Chris Wilsone28f8712011-07-18 13:11:49 -07001495 * @dev: the device
1496 * @size: size of the object
1497 * @tiling_mode: tiling mode of the object
Daniel Vetter5e783302010-11-14 22:32:36 +01001498 *
1499 * Return the required GTT alignment for an object, only taking into account
1500 * unfenced tiled surface requirements.
1501 */
Chris Wilson467cffb2011-03-07 10:42:03 +00001502uint32_t
Chris Wilsone28f8712011-07-18 13:11:49 -07001503i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
1504 uint32_t size,
1505 int tiling_mode)
Daniel Vetter5e783302010-11-14 22:32:36 +01001506{
Daniel Vetter5e783302010-11-14 22:32:36 +01001507 /*
1508 * Minimum alignment is 4k (GTT page size) for sane hw.
1509 */
1510 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
Chris Wilsone28f8712011-07-18 13:11:49 -07001511 tiling_mode == I915_TILING_NONE)
Daniel Vetter5e783302010-11-14 22:32:36 +01001512 return 4096;
1513
Chris Wilsone28f8712011-07-18 13:11:49 -07001514 /* Previous hardware however needs to be aligned to a power-of-two
1515 * tile height. The simplest method for determining this is to reuse
1516 * the power-of-tile object size.
Daniel Vetter5e783302010-11-14 22:32:36 +01001517 */
Chris Wilsone28f8712011-07-18 13:11:49 -07001518 return i915_gem_get_gtt_size(dev, size, tiling_mode);
Daniel Vetter5e783302010-11-14 22:32:36 +01001519}
1520
Chris Wilsond8cb5082012-08-11 15:41:03 +01001521static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1522{
1523 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1524 int ret;
1525
1526 if (obj->base.map_list.map)
1527 return 0;
1528
1529 ret = drm_gem_create_mmap_offset(&obj->base);
1530 if (ret != -ENOSPC)
1531 return ret;
1532
1533 /* Badly fragmented mmap space? The only way we can recover
1534 * space is by destroying unwanted objects. We can't randomly release
1535 * mmap_offsets as userspace expects them to be persistent for the
1536 * lifetime of the objects. The closest we can is to release the
1537 * offsets on purgeable objects by truncating it and marking it purged,
1538 * which prevents userspace from ever using that object again.
1539 */
1540 i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
1541 ret = drm_gem_create_mmap_offset(&obj->base);
1542 if (ret != -ENOSPC)
1543 return ret;
1544
1545 i915_gem_shrink_all(dev_priv);
1546 return drm_gem_create_mmap_offset(&obj->base);
1547}
1548
1549static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1550{
1551 if (!obj->base.map_list.map)
1552 return;
1553
1554 drm_gem_free_mmap_offset(&obj->base);
1555}
1556
Jesse Barnesde151cf2008-11-12 10:03:55 -08001557int
Dave Airlieff72145b2011-02-07 12:16:14 +10001558i915_gem_mmap_gtt(struct drm_file *file,
1559 struct drm_device *dev,
1560 uint32_t handle,
1561 uint64_t *offset)
Jesse Barnesde151cf2008-11-12 10:03:55 -08001562{
Chris Wilsonda761a62010-10-27 17:37:08 +01001563 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson05394f32010-11-08 19:18:58 +00001564 struct drm_i915_gem_object *obj;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001565 int ret;
1566
Chris Wilson76c1dec2010-09-25 11:22:51 +01001567 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001568 if (ret)
Chris Wilson76c1dec2010-09-25 11:22:51 +01001569 return ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001570
Dave Airlieff72145b2011-02-07 12:16:14 +10001571 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00001572 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001573 ret = -ENOENT;
1574 goto unlock;
1575 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001576
Chris Wilson05394f32010-11-08 19:18:58 +00001577 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
Chris Wilsonda761a62010-10-27 17:37:08 +01001578 ret = -E2BIG;
Eric Anholtff56b0b2011-10-31 23:16:21 -07001579 goto out;
Chris Wilsonda761a62010-10-27 17:37:08 +01001580 }
1581
Chris Wilson05394f32010-11-08 19:18:58 +00001582 if (obj->madv != I915_MADV_WILLNEED) {
Chris Wilsonab182822009-09-22 18:46:17 +01001583 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001584 ret = -EINVAL;
1585 goto out;
Chris Wilsonab182822009-09-22 18:46:17 +01001586 }
1587
Chris Wilsond8cb5082012-08-11 15:41:03 +01001588 ret = i915_gem_object_create_mmap_offset(obj);
1589 if (ret)
1590 goto out;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001591
Dave Airlieff72145b2011-02-07 12:16:14 +10001592 *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001593
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001594out:
Chris Wilson05394f32010-11-08 19:18:58 +00001595 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001596unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001597 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001598 return ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001599}
1600
Dave Airlieff72145b2011-02-07 12:16:14 +10001601/**
1602 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1603 * @dev: DRM device
1604 * @data: GTT mapping ioctl data
1605 * @file: GEM object info
1606 *
1607 * Simply returns the fake offset to userspace so it can mmap it.
1608 * The mmap call will end up in drm_gem_mmap(), which will set things
1609 * up so we can get faults in the handler above.
1610 *
1611 * The fault handler will take care of binding the object into the GTT
1612 * (since it may have been evicted to make room for something), allocating
1613 * a fence register, and mapping the appropriate aperture address into
1614 * userspace.
1615 */
1616int
1617i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1618 struct drm_file *file)
1619{
1620 struct drm_i915_gem_mmap_gtt *args = data;
1621
Dave Airlieff72145b2011-02-07 12:16:14 +10001622 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1623}
1624
Daniel Vetter225067e2012-08-20 10:23:20 +02001625/* Immediately discard the backing storage */
1626static void
1627i915_gem_object_truncate(struct drm_i915_gem_object *obj)
Chris Wilsone5281cc2010-10-28 13:45:36 +01001628{
Chris Wilsone5281cc2010-10-28 13:45:36 +01001629 struct inode *inode;
Chris Wilsone5281cc2010-10-28 13:45:36 +01001630
Chris Wilson4d6294bf2012-08-11 15:41:05 +01001631 i915_gem_object_free_mmap_offset(obj);
Daniel Vetter1286ff72012-05-10 15:25:09 +02001632
Chris Wilson4d6294bf2012-08-11 15:41:05 +01001633 if (obj->base.filp == NULL)
1634 return;
1635
Daniel Vetter225067e2012-08-20 10:23:20 +02001636 /* Our goal here is to return as much of the memory as
1637 * is possible back to the system as we are called from OOM.
1638 * To do this we must instruct the shmfs to drop all of its
1639 * backing pages, *now*.
Chris Wilsone5281cc2010-10-28 13:45:36 +01001640 */
Chris Wilson05394f32010-11-08 19:18:58 +00001641 inode = obj->base.filp->f_path.dentry->d_inode;
Daniel Vetter225067e2012-08-20 10:23:20 +02001642 shmem_truncate_range(inode, 0, (loff_t)-1);
Hugh Dickins5949eac2011-06-27 16:18:18 -07001643
Daniel Vetter225067e2012-08-20 10:23:20 +02001644 obj->madv = __I915_MADV_PURGED;
Chris Wilsone5281cc2010-10-28 13:45:36 +01001645}
1646
Daniel Vetter225067e2012-08-20 10:23:20 +02001647static inline int
1648i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1649{
1650 return obj->madv == I915_MADV_DONTNEED;
1651}
1652
Chris Wilson37e680a2012-06-07 15:38:42 +01001653static void
Chris Wilson05394f32010-11-08 19:18:58 +00001654i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07001655{
Chris Wilson05394f32010-11-08 19:18:58 +00001656 int page_count = obj->base.size / PAGE_SIZE;
Chris Wilson9da3da62012-06-01 15:20:22 +01001657 struct scatterlist *sg;
Chris Wilson6c085a72012-08-20 11:40:46 +02001658 int ret, i;
Eric Anholt673a3942008-07-30 12:06:12 -07001659
Chris Wilson05394f32010-11-08 19:18:58 +00001660 BUG_ON(obj->madv == __I915_MADV_PURGED);
Eric Anholt856fa192009-03-19 14:10:50 -07001661
Chris Wilson6c085a72012-08-20 11:40:46 +02001662 ret = i915_gem_object_set_to_cpu_domain(obj, true);
1663 if (ret) {
1664 /* In the event of a disaster, abandon all caches and
1665 * hope for the best.
1666 */
1667 WARN_ON(ret != -EIO);
1668 i915_gem_clflush_object(obj);
1669 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1670 }
1671
Daniel Vetter6dacfd22011-09-12 21:30:02 +02001672 if (i915_gem_object_needs_bit17_swizzle(obj))
Eric Anholt280b7132009-03-12 16:56:27 -07001673 i915_gem_object_save_bit_17_swizzle(obj);
1674
Chris Wilson05394f32010-11-08 19:18:58 +00001675 if (obj->madv == I915_MADV_DONTNEED)
1676 obj->dirty = 0;
Chris Wilson3ef94da2009-09-14 16:50:29 +01001677
Chris Wilson9da3da62012-06-01 15:20:22 +01001678 for_each_sg(obj->pages->sgl, sg, page_count, i) {
1679 struct page *page = sg_page(sg);
1680
Chris Wilson05394f32010-11-08 19:18:58 +00001681 if (obj->dirty)
Chris Wilson9da3da62012-06-01 15:20:22 +01001682 set_page_dirty(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01001683
Chris Wilson05394f32010-11-08 19:18:58 +00001684 if (obj->madv == I915_MADV_WILLNEED)
Chris Wilson9da3da62012-06-01 15:20:22 +01001685 mark_page_accessed(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01001686
Chris Wilson9da3da62012-06-01 15:20:22 +01001687 page_cache_release(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01001688 }
Chris Wilson05394f32010-11-08 19:18:58 +00001689 obj->dirty = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07001690
Chris Wilson9da3da62012-06-01 15:20:22 +01001691 sg_free_table(obj->pages);
1692 kfree(obj->pages);
Chris Wilson37e680a2012-06-07 15:38:42 +01001693}
1694
1695static int
1696i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
1697{
1698 const struct drm_i915_gem_object_ops *ops = obj->ops;
1699
Chris Wilson2f745ad2012-09-04 21:02:58 +01001700 if (obj->pages == NULL)
Chris Wilson37e680a2012-06-07 15:38:42 +01001701 return 0;
1702
1703 BUG_ON(obj->gtt_space);
1704
Chris Wilsona5570172012-09-04 21:02:54 +01001705 if (obj->pages_pin_count)
1706 return -EBUSY;
1707
Chris Wilson37e680a2012-06-07 15:38:42 +01001708 ops->put_pages(obj);
Chris Wilson9da3da62012-06-01 15:20:22 +01001709 obj->pages = NULL;
Chris Wilson6c085a72012-08-20 11:40:46 +02001710
1711 list_del(&obj->gtt_list);
Chris Wilson6c085a72012-08-20 11:40:46 +02001712 if (i915_gem_object_is_purgeable(obj))
1713 i915_gem_object_truncate(obj);
1714
1715 return 0;
1716}
1717
1718static long
1719i915_gem_purge(struct drm_i915_private *dev_priv, long target)
1720{
1721 struct drm_i915_gem_object *obj, *next;
1722 long count = 0;
1723
1724 list_for_each_entry_safe(obj, next,
1725 &dev_priv->mm.unbound_list,
1726 gtt_list) {
1727 if (i915_gem_object_is_purgeable(obj) &&
Chris Wilson37e680a2012-06-07 15:38:42 +01001728 i915_gem_object_put_pages(obj) == 0) {
Chris Wilson6c085a72012-08-20 11:40:46 +02001729 count += obj->base.size >> PAGE_SHIFT;
1730 if (count >= target)
1731 return count;
1732 }
1733 }
1734
1735 list_for_each_entry_safe(obj, next,
1736 &dev_priv->mm.inactive_list,
1737 mm_list) {
1738 if (i915_gem_object_is_purgeable(obj) &&
1739 i915_gem_object_unbind(obj) == 0 &&
Chris Wilson37e680a2012-06-07 15:38:42 +01001740 i915_gem_object_put_pages(obj) == 0) {
Chris Wilson6c085a72012-08-20 11:40:46 +02001741 count += obj->base.size >> PAGE_SHIFT;
1742 if (count >= target)
1743 return count;
1744 }
1745 }
1746
1747 return count;
1748}
1749
1750static void
1751i915_gem_shrink_all(struct drm_i915_private *dev_priv)
1752{
1753 struct drm_i915_gem_object *obj, *next;
1754
1755 i915_gem_evict_everything(dev_priv->dev);
1756
1757 list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list)
Chris Wilson37e680a2012-06-07 15:38:42 +01001758 i915_gem_object_put_pages(obj);
Daniel Vetter225067e2012-08-20 10:23:20 +02001759}
1760
Chris Wilson37e680a2012-06-07 15:38:42 +01001761static int
Chris Wilson6c085a72012-08-20 11:40:46 +02001762i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07001763{
Chris Wilson6c085a72012-08-20 11:40:46 +02001764 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07001765 int page_count, i;
1766 struct address_space *mapping;
Chris Wilson9da3da62012-06-01 15:20:22 +01001767 struct sg_table *st;
1768 struct scatterlist *sg;
Eric Anholt673a3942008-07-30 12:06:12 -07001769 struct page *page;
Chris Wilson6c085a72012-08-20 11:40:46 +02001770 gfp_t gfp;
Eric Anholt673a3942008-07-30 12:06:12 -07001771
Chris Wilson6c085a72012-08-20 11:40:46 +02001772 /* Assert that the object is not currently in any GPU domain. As it
1773 * wasn't in the GTT, there shouldn't be any way it could have been in
1774 * a GPU cache
1775 */
1776 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
1777 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
1778
Chris Wilson9da3da62012-06-01 15:20:22 +01001779 st = kmalloc(sizeof(*st), GFP_KERNEL);
1780 if (st == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07001781 return -ENOMEM;
1782
Chris Wilson9da3da62012-06-01 15:20:22 +01001783 page_count = obj->base.size / PAGE_SIZE;
1784 if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
1785 sg_free_table(st);
1786 kfree(st);
1787 return -ENOMEM;
1788 }
1789
1790 /* Get the list of pages out of our struct file. They'll be pinned
1791 * at this point until we release them.
1792 *
1793 * Fail silently without starting the shrinker
1794 */
Chris Wilson6c085a72012-08-20 11:40:46 +02001795 mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
1796 gfp = mapping_gfp_mask(mapping);
Sedat Dilekd7c3b9372012-08-27 14:02:37 +02001797 gfp |= __GFP_NORETRY | __GFP_NOWARN;
Chris Wilson6c085a72012-08-20 11:40:46 +02001798 gfp &= ~(__GFP_IO | __GFP_WAIT);
Chris Wilson9da3da62012-06-01 15:20:22 +01001799 for_each_sg(st->sgl, sg, page_count, i) {
Chris Wilson6c085a72012-08-20 11:40:46 +02001800 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1801 if (IS_ERR(page)) {
1802 i915_gem_purge(dev_priv, page_count);
1803 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1804 }
1805 if (IS_ERR(page)) {
1806 /* We've tried hard to allocate the memory by reaping
1807 * our own buffer, now let the real VM do its job and
1808 * go down in flames if truly OOM.
1809 */
Sedat Dilekd7c3b9372012-08-27 14:02:37 +02001810 gfp &= ~(__GFP_NORETRY | __GFP_NOWARN);
Chris Wilson6c085a72012-08-20 11:40:46 +02001811 gfp |= __GFP_IO | __GFP_WAIT;
1812
1813 i915_gem_shrink_all(dev_priv);
1814 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1815 if (IS_ERR(page))
1816 goto err_pages;
1817
Sedat Dilekd7c3b9372012-08-27 14:02:37 +02001818 gfp |= __GFP_NORETRY | __GFP_NOWARN;
Chris Wilson6c085a72012-08-20 11:40:46 +02001819 gfp &= ~(__GFP_IO | __GFP_WAIT);
1820 }
Eric Anholt673a3942008-07-30 12:06:12 -07001821
Chris Wilson9da3da62012-06-01 15:20:22 +01001822 sg_set_page(sg, page, PAGE_SIZE, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07001823 }
1824
1825 if (i915_gem_object_needs_bit17_swizzle(obj))
1826 i915_gem_object_do_bit_17_swizzle(obj);
1827
Chris Wilson9da3da62012-06-01 15:20:22 +01001828 obj->pages = st;
Eric Anholt673a3942008-07-30 12:06:12 -07001829 return 0;
1830
1831err_pages:
Chris Wilson9da3da62012-06-01 15:20:22 +01001832 for_each_sg(st->sgl, sg, i, page_count)
1833 page_cache_release(sg_page(sg));
1834 sg_free_table(st);
1835 kfree(st);
Eric Anholt673a3942008-07-30 12:06:12 -07001836 return PTR_ERR(page);
Eric Anholt673a3942008-07-30 12:06:12 -07001837}
1838
Chris Wilson37e680a2012-06-07 15:38:42 +01001839/* Ensure that the associated pages are gathered from the backing storage
1840 * and pinned into our object. i915_gem_object_get_pages() may be called
1841 * multiple times before they are released by a single call to
1842 * i915_gem_object_put_pages() - once the pages are no longer referenced
1843 * either as a result of memory pressure (reaping pages under the shrinker)
1844 * or as the object is itself released.
1845 */
1846int
1847i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
1848{
1849 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1850 const struct drm_i915_gem_object_ops *ops = obj->ops;
1851 int ret;
1852
Chris Wilson2f745ad2012-09-04 21:02:58 +01001853 if (obj->pages)
Chris Wilson37e680a2012-06-07 15:38:42 +01001854 return 0;
1855
Chris Wilsona5570172012-09-04 21:02:54 +01001856 BUG_ON(obj->pages_pin_count);
1857
Chris Wilson37e680a2012-06-07 15:38:42 +01001858 ret = ops->get_pages(obj);
1859 if (ret)
1860 return ret;
1861
1862 list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
1863 return 0;
1864}
1865
Chris Wilson54cf91d2010-11-25 18:00:26 +00001866void
Chris Wilson05394f32010-11-08 19:18:58 +00001867i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001868 struct intel_ring_buffer *ring,
1869 u32 seqno)
Eric Anholt673a3942008-07-30 12:06:12 -07001870{
Chris Wilson05394f32010-11-08 19:18:58 +00001871 struct drm_device *dev = obj->base.dev;
Chris Wilson69dc4982010-10-19 10:36:51 +01001872 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter617dbe22010-02-11 22:16:02 +01001873
Zou Nan hai852835f2010-05-21 09:08:56 +08001874 BUG_ON(ring == NULL);
Chris Wilson05394f32010-11-08 19:18:58 +00001875 obj->ring = ring;
Eric Anholt673a3942008-07-30 12:06:12 -07001876
1877 /* Add a reference if we're newly entering the active list. */
Chris Wilson05394f32010-11-08 19:18:58 +00001878 if (!obj->active) {
1879 drm_gem_object_reference(&obj->base);
1880 obj->active = 1;
Eric Anholt673a3942008-07-30 12:06:12 -07001881 }
Daniel Vettere35a41d2010-02-11 22:13:59 +01001882
Eric Anholt673a3942008-07-30 12:06:12 -07001883 /* Move from whatever list we were on to the tail of execution. */
Chris Wilson05394f32010-11-08 19:18:58 +00001884 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1885 list_move_tail(&obj->ring_list, &ring->active_list);
Chris Wilsoncaea7472010-11-12 13:53:37 +00001886
Chris Wilson0201f1e2012-07-20 12:41:01 +01001887 obj->last_read_seqno = seqno;
Chris Wilson7dd49062012-03-21 10:48:18 +00001888
Chris Wilsoncaea7472010-11-12 13:53:37 +00001889 if (obj->fenced_gpu_access) {
Chris Wilsoncaea7472010-11-12 13:53:37 +00001890 obj->last_fenced_seqno = seqno;
Chris Wilsoncaea7472010-11-12 13:53:37 +00001891
Chris Wilson7dd49062012-03-21 10:48:18 +00001892 /* Bump MRU to take account of the delayed flush */
1893 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1894 struct drm_i915_fence_reg *reg;
1895
1896 reg = &dev_priv->fence_regs[obj->fence_reg];
1897 list_move_tail(&reg->lru_list,
1898 &dev_priv->mm.fence_list);
1899 }
Chris Wilsoncaea7472010-11-12 13:53:37 +00001900 }
1901}
1902
1903static void
Chris Wilsoncaea7472010-11-12 13:53:37 +00001904i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1905{
1906 struct drm_device *dev = obj->base.dev;
1907 struct drm_i915_private *dev_priv = dev->dev_private;
1908
Chris Wilson65ce3022012-07-20 12:41:02 +01001909 BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
Chris Wilsoncaea7472010-11-12 13:53:37 +00001910 BUG_ON(!obj->active);
Chris Wilson65ce3022012-07-20 12:41:02 +01001911
Chris Wilsonf047e392012-07-21 12:31:41 +01001912 if (obj->pin_count) /* are we a framebuffer? */
1913 intel_mark_fb_idle(obj);
1914
1915 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1916
Chris Wilson65ce3022012-07-20 12:41:02 +01001917 list_del_init(&obj->ring_list);
Chris Wilsoncaea7472010-11-12 13:53:37 +00001918 obj->ring = NULL;
1919
Chris Wilson65ce3022012-07-20 12:41:02 +01001920 obj->last_read_seqno = 0;
1921 obj->last_write_seqno = 0;
1922 obj->base.write_domain = 0;
1923
1924 obj->last_fenced_seqno = 0;
Chris Wilsoncaea7472010-11-12 13:53:37 +00001925 obj->fenced_gpu_access = false;
Chris Wilsoncaea7472010-11-12 13:53:37 +00001926
1927 obj->active = 0;
1928 drm_gem_object_unreference(&obj->base);
1929
1930 WARN_ON(i915_verify_lists(dev));
Eric Anholtce44b0e2008-11-06 16:00:31 -08001931}
Eric Anholt673a3942008-07-30 12:06:12 -07001932
Daniel Vetter53d227f2012-01-25 16:32:49 +01001933static u32
1934i915_gem_get_seqno(struct drm_device *dev)
1935{
1936 drm_i915_private_t *dev_priv = dev->dev_private;
1937 u32 seqno = dev_priv->next_seqno;
1938
1939 /* reserve 0 for non-seqno */
1940 if (++dev_priv->next_seqno == 0)
1941 dev_priv->next_seqno = 1;
1942
1943 return seqno;
1944}
1945
1946u32
1947i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
1948{
1949 if (ring->outstanding_lazy_request == 0)
1950 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
1951
1952 return ring->outstanding_lazy_request;
1953}
1954
Chris Wilson3cce4692010-10-27 16:11:02 +01001955int
Chris Wilsondb53a302011-02-03 11:57:46 +00001956i915_add_request(struct intel_ring_buffer *ring,
Chris Wilsonf787a5f2010-09-24 16:02:42 +01001957 struct drm_file *file,
Chris Wilsondb53a302011-02-03 11:57:46 +00001958 struct drm_i915_gem_request *request)
Eric Anholt673a3942008-07-30 12:06:12 -07001959{
Chris Wilsondb53a302011-02-03 11:57:46 +00001960 drm_i915_private_t *dev_priv = ring->dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07001961 uint32_t seqno;
Chris Wilsona71d8d92012-02-15 11:25:36 +00001962 u32 request_ring_position;
Eric Anholt673a3942008-07-30 12:06:12 -07001963 int was_empty;
Chris Wilson3cce4692010-10-27 16:11:02 +01001964 int ret;
1965
Daniel Vettercc889e02012-06-13 20:45:19 +02001966 /*
1967 * Emit any outstanding flushes - execbuf can fail to emit the flush
1968 * after having emitted the batchbuffer command. Hence we need to fix
1969 * things up similar to emitting the lazy request. The difference here
1970 * is that the flush _must_ happen before the next request, no matter
1971 * what.
1972 */
Chris Wilsona7b97612012-07-20 12:41:08 +01001973 ret = intel_ring_flush_all_caches(ring);
1974 if (ret)
1975 return ret;
Daniel Vettercc889e02012-06-13 20:45:19 +02001976
Chris Wilson3bb73ab2012-07-20 12:40:59 +01001977 if (request == NULL) {
1978 request = kmalloc(sizeof(*request), GFP_KERNEL);
1979 if (request == NULL)
1980 return -ENOMEM;
1981 }
1982
Daniel Vetter53d227f2012-01-25 16:32:49 +01001983 seqno = i915_gem_next_request_seqno(ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001984
Chris Wilsona71d8d92012-02-15 11:25:36 +00001985 /* Record the position of the start of the request so that
1986 * should we detect the updated seqno part-way through the
1987 * GPU processing the request, we never over-estimate the
1988 * position of the head.
1989 */
1990 request_ring_position = intel_ring_get_tail(ring);
1991
Chris Wilson3cce4692010-10-27 16:11:02 +01001992 ret = ring->add_request(ring, &seqno);
Chris Wilson3bb73ab2012-07-20 12:40:59 +01001993 if (ret) {
1994 kfree(request);
1995 return ret;
1996 }
Eric Anholt673a3942008-07-30 12:06:12 -07001997
Chris Wilsondb53a302011-02-03 11:57:46 +00001998 trace_i915_gem_request_add(ring, seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07001999
2000 request->seqno = seqno;
Zou Nan hai852835f2010-05-21 09:08:56 +08002001 request->ring = ring;
Chris Wilsona71d8d92012-02-15 11:25:36 +00002002 request->tail = request_ring_position;
Eric Anholt673a3942008-07-30 12:06:12 -07002003 request->emitted_jiffies = jiffies;
Zou Nan hai852835f2010-05-21 09:08:56 +08002004 was_empty = list_empty(&ring->request_list);
2005 list_add_tail(&request->list, &ring->request_list);
Chris Wilson3bb73ab2012-07-20 12:40:59 +01002006 request->file_priv = NULL;
Zou Nan hai852835f2010-05-21 09:08:56 +08002007
Chris Wilsondb53a302011-02-03 11:57:46 +00002008 if (file) {
2009 struct drm_i915_file_private *file_priv = file->driver_priv;
2010
Chris Wilson1c255952010-09-26 11:03:27 +01002011 spin_lock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01002012 request->file_priv = file_priv;
Eric Anholtb9624422009-06-03 07:27:35 +00002013 list_add_tail(&request->client_list,
Chris Wilsonf787a5f2010-09-24 16:02:42 +01002014 &file_priv->mm.request_list);
Chris Wilson1c255952010-09-26 11:03:27 +01002015 spin_unlock(&file_priv->mm.lock);
Eric Anholtb9624422009-06-03 07:27:35 +00002016 }
Eric Anholt673a3942008-07-30 12:06:12 -07002017
Daniel Vetter5391d0c2012-01-25 14:03:57 +01002018 ring->outstanding_lazy_request = 0;
Chris Wilsondb53a302011-02-03 11:57:46 +00002019
Ben Gamarif65d9422009-09-14 17:48:44 -04002020 if (!dev_priv->mm.suspended) {
Ben Widawsky3e0dc6b2011-06-29 10:26:42 -07002021 if (i915_enable_hangcheck) {
2022 mod_timer(&dev_priv->hangcheck_timer,
2023 jiffies +
2024 msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2025 }
Chris Wilsonf047e392012-07-21 12:31:41 +01002026 if (was_empty) {
Chris Wilsonb3b079d2010-09-13 23:44:34 +01002027 queue_delayed_work(dev_priv->wq,
2028 &dev_priv->mm.retire_work, HZ);
Chris Wilsonf047e392012-07-21 12:31:41 +01002029 intel_mark_busy(dev_priv->dev);
2030 }
Ben Gamarif65d9422009-09-14 17:48:44 -04002031 }
Daniel Vettercc889e02012-06-13 20:45:19 +02002032
Chris Wilson3cce4692010-10-27 16:11:02 +01002033 return 0;
Eric Anholt673a3942008-07-30 12:06:12 -07002034}
2035
Chris Wilsonf787a5f2010-09-24 16:02:42 +01002036static inline void
2037i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
Eric Anholt673a3942008-07-30 12:06:12 -07002038{
Chris Wilson1c255952010-09-26 11:03:27 +01002039 struct drm_i915_file_private *file_priv = request->file_priv;
Eric Anholt673a3942008-07-30 12:06:12 -07002040
Chris Wilson1c255952010-09-26 11:03:27 +01002041 if (!file_priv)
2042 return;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002043
Chris Wilson1c255952010-09-26 11:03:27 +01002044 spin_lock(&file_priv->mm.lock);
Herton Ronaldo Krzesinski09bfa512011-03-17 13:45:12 +00002045 if (request->file_priv) {
2046 list_del(&request->client_list);
2047 request->file_priv = NULL;
2048 }
Chris Wilson1c255952010-09-26 11:03:27 +01002049 spin_unlock(&file_priv->mm.lock);
Eric Anholt673a3942008-07-30 12:06:12 -07002050}
2051
Chris Wilsondfaae392010-09-22 10:31:52 +01002052static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
2053 struct intel_ring_buffer *ring)
Chris Wilson9375e442010-09-19 12:21:28 +01002054{
Chris Wilsondfaae392010-09-22 10:31:52 +01002055 while (!list_empty(&ring->request_list)) {
2056 struct drm_i915_gem_request *request;
Chris Wilson9375e442010-09-19 12:21:28 +01002057
Chris Wilsondfaae392010-09-22 10:31:52 +01002058 request = list_first_entry(&ring->request_list,
2059 struct drm_i915_gem_request,
2060 list);
2061
2062 list_del(&request->list);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01002063 i915_gem_request_remove_from_client(request);
Chris Wilsondfaae392010-09-22 10:31:52 +01002064 kfree(request);
2065 }
2066
2067 while (!list_empty(&ring->active_list)) {
Chris Wilson05394f32010-11-08 19:18:58 +00002068 struct drm_i915_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07002069
Chris Wilson05394f32010-11-08 19:18:58 +00002070 obj = list_first_entry(&ring->active_list,
2071 struct drm_i915_gem_object,
2072 ring_list);
Eric Anholt673a3942008-07-30 12:06:12 -07002073
Chris Wilson05394f32010-11-08 19:18:58 +00002074 i915_gem_object_move_to_inactive(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002075 }
Eric Anholt673a3942008-07-30 12:06:12 -07002076}
2077
Chris Wilson312817a2010-11-22 11:50:11 +00002078static void i915_gem_reset_fences(struct drm_device *dev)
2079{
2080 struct drm_i915_private *dev_priv = dev->dev_private;
2081 int i;
2082
Daniel Vetter4b9de732011-10-09 21:52:02 +02002083 for (i = 0; i < dev_priv->num_fence_regs; i++) {
Chris Wilson312817a2010-11-22 11:50:11 +00002084 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
Chris Wilson7d2cb392010-11-27 17:38:29 +00002085
Chris Wilsonada726c2012-04-17 15:31:32 +01002086 i915_gem_write_fence(dev, i, NULL);
Chris Wilson7d2cb392010-11-27 17:38:29 +00002087
Chris Wilsonada726c2012-04-17 15:31:32 +01002088 if (reg->obj)
2089 i915_gem_object_fence_lost(reg->obj);
Chris Wilson7d2cb392010-11-27 17:38:29 +00002090
Chris Wilsonada726c2012-04-17 15:31:32 +01002091 reg->pin_count = 0;
2092 reg->obj = NULL;
2093 INIT_LIST_HEAD(&reg->lru_list);
Chris Wilson312817a2010-11-22 11:50:11 +00002094 }
Chris Wilsonada726c2012-04-17 15:31:32 +01002095
2096 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
Chris Wilson312817a2010-11-22 11:50:11 +00002097}
2098
Chris Wilson069efc12010-09-30 16:53:18 +01002099void i915_gem_reset(struct drm_device *dev)
Eric Anholt673a3942008-07-30 12:06:12 -07002100{
Chris Wilsondfaae392010-09-22 10:31:52 +01002101 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson05394f32010-11-08 19:18:58 +00002102 struct drm_i915_gem_object *obj;
Chris Wilsonb4519512012-05-11 14:29:30 +01002103 struct intel_ring_buffer *ring;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002104 int i;
Eric Anholt673a3942008-07-30 12:06:12 -07002105
Chris Wilsonb4519512012-05-11 14:29:30 +01002106 for_each_ring(ring, dev_priv, i)
2107 i915_gem_reset_ring_lists(dev_priv, ring);
Chris Wilsondfaae392010-09-22 10:31:52 +01002108
Chris Wilsondfaae392010-09-22 10:31:52 +01002109 /* Move everything out of the GPU domains to ensure we do any
2110 * necessary invalidation upon reuse.
2111 */
Chris Wilson05394f32010-11-08 19:18:58 +00002112 list_for_each_entry(obj,
Chris Wilson77f01232010-09-19 12:31:36 +01002113 &dev_priv->mm.inactive_list,
Chris Wilson69dc4982010-10-19 10:36:51 +01002114 mm_list)
Chris Wilson77f01232010-09-19 12:31:36 +01002115 {
Chris Wilson05394f32010-11-08 19:18:58 +00002116 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
Chris Wilson77f01232010-09-19 12:31:36 +01002117 }
Chris Wilson069efc12010-09-30 16:53:18 +01002118
2119 /* The fence registers are invalidated so clear them out */
Chris Wilson312817a2010-11-22 11:50:11 +00002120 i915_gem_reset_fences(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07002121}
2122
2123/**
2124 * This function clears the request list as sequence numbers are passed.
2125 */
Chris Wilsona71d8d92012-02-15 11:25:36 +00002126void
Chris Wilsondb53a302011-02-03 11:57:46 +00002127i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07002128{
Eric Anholt673a3942008-07-30 12:06:12 -07002129 uint32_t seqno;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002130 int i;
Eric Anholt673a3942008-07-30 12:06:12 -07002131
Chris Wilsondb53a302011-02-03 11:57:46 +00002132 if (list_empty(&ring->request_list))
Karsten Wiese6c0594a2009-02-23 15:07:57 +01002133 return;
2134
Chris Wilsondb53a302011-02-03 11:57:46 +00002135 WARN_ON(i915_verify_lists(ring->dev));
Eric Anholt673a3942008-07-30 12:06:12 -07002136
Chris Wilsonb2eadbc2012-08-09 10:58:30 +01002137 seqno = ring->get_seqno(ring, true);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002138
Chris Wilson076e2c02011-01-21 10:07:18 +00002139 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002140 if (seqno >= ring->sync_seqno[i])
2141 ring->sync_seqno[i] = 0;
2142
Zou Nan hai852835f2010-05-21 09:08:56 +08002143 while (!list_empty(&ring->request_list)) {
Eric Anholt673a3942008-07-30 12:06:12 -07002144 struct drm_i915_gem_request *request;
Eric Anholt673a3942008-07-30 12:06:12 -07002145
Zou Nan hai852835f2010-05-21 09:08:56 +08002146 request = list_first_entry(&ring->request_list,
Eric Anholt673a3942008-07-30 12:06:12 -07002147 struct drm_i915_gem_request,
2148 list);
Eric Anholt673a3942008-07-30 12:06:12 -07002149
Chris Wilsondfaae392010-09-22 10:31:52 +01002150 if (!i915_seqno_passed(seqno, request->seqno))
Eric Anholt673a3942008-07-30 12:06:12 -07002151 break;
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002152
Chris Wilsondb53a302011-02-03 11:57:46 +00002153 trace_i915_gem_request_retire(ring, request->seqno);
Chris Wilsona71d8d92012-02-15 11:25:36 +00002154 /* We know the GPU must have read the request to have
2155 * sent us the seqno + interrupt, so use the position
2156 * of tail of the request to update the last known position
2157 * of the GPU head.
2158 */
2159 ring->last_retired_head = request->tail;
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002160
2161 list_del(&request->list);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01002162 i915_gem_request_remove_from_client(request);
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002163 kfree(request);
2164 }
2165
2166 /* Move any buffers on the active list that are no longer referenced
2167 * by the ringbuffer to the flushing/inactive lists as appropriate.
2168 */
2169 while (!list_empty(&ring->active_list)) {
Chris Wilson05394f32010-11-08 19:18:58 +00002170 struct drm_i915_gem_object *obj;
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002171
Akshay Joshi0206e352011-08-16 15:34:10 -04002172 obj = list_first_entry(&ring->active_list,
Chris Wilson05394f32010-11-08 19:18:58 +00002173 struct drm_i915_gem_object,
2174 ring_list);
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002175
Chris Wilson0201f1e2012-07-20 12:41:01 +01002176 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002177 break;
2178
Chris Wilson65ce3022012-07-20 12:41:02 +01002179 i915_gem_object_move_to_inactive(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002180 }
Chris Wilson9d34e5d2009-09-24 05:26:06 +01002181
Chris Wilsondb53a302011-02-03 11:57:46 +00002182 if (unlikely(ring->trace_irq_seqno &&
2183 i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002184 ring->irq_put(ring);
Chris Wilsondb53a302011-02-03 11:57:46 +00002185 ring->trace_irq_seqno = 0;
Chris Wilson9d34e5d2009-09-24 05:26:06 +01002186 }
Chris Wilson23bc5982010-09-29 16:10:57 +01002187
Chris Wilsondb53a302011-02-03 11:57:46 +00002188 WARN_ON(i915_verify_lists(ring->dev));
Eric Anholt673a3942008-07-30 12:06:12 -07002189}
2190
2191void
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01002192i915_gem_retire_requests(struct drm_device *dev)
2193{
2194 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonb4519512012-05-11 14:29:30 +01002195 struct intel_ring_buffer *ring;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002196 int i;
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01002197
Chris Wilsonb4519512012-05-11 14:29:30 +01002198 for_each_ring(ring, dev_priv, i)
2199 i915_gem_retire_requests_ring(ring);
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01002200}
2201
Daniel Vetter75ef9da2010-08-21 00:25:16 +02002202static void
Eric Anholt673a3942008-07-30 12:06:12 -07002203i915_gem_retire_work_handler(struct work_struct *work)
2204{
2205 drm_i915_private_t *dev_priv;
2206 struct drm_device *dev;
Chris Wilsonb4519512012-05-11 14:29:30 +01002207 struct intel_ring_buffer *ring;
Chris Wilson0a587052011-01-09 21:05:44 +00002208 bool idle;
2209 int i;
Eric Anholt673a3942008-07-30 12:06:12 -07002210
2211 dev_priv = container_of(work, drm_i915_private_t,
2212 mm.retire_work.work);
2213 dev = dev_priv->dev;
2214
Chris Wilson891b48c2010-09-29 12:26:37 +01002215 /* Come back later if the device is busy... */
2216 if (!mutex_trylock(&dev->struct_mutex)) {
2217 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2218 return;
2219 }
2220
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01002221 i915_gem_retire_requests(dev);
Zou Nan haid1b851f2010-05-21 09:08:57 +08002222
Chris Wilson0a587052011-01-09 21:05:44 +00002223 /* Send a periodic flush down the ring so we don't hold onto GEM
2224 * objects indefinitely.
2225 */
2226 idle = true;
Chris Wilsonb4519512012-05-11 14:29:30 +01002227 for_each_ring(ring, dev_priv, i) {
Chris Wilson3bb73ab2012-07-20 12:40:59 +01002228 if (ring->gpu_caches_dirty)
2229 i915_add_request(ring, NULL, NULL);
Chris Wilson0a587052011-01-09 21:05:44 +00002230
2231 idle &= list_empty(&ring->request_list);
2232 }
2233
2234 if (!dev_priv->mm.suspended && !idle)
Eric Anholt9c9fe1f2009-08-03 16:09:16 -07002235 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
Chris Wilsonf047e392012-07-21 12:31:41 +01002236 if (idle)
2237 intel_mark_idle(dev);
Chris Wilson0a587052011-01-09 21:05:44 +00002238
Eric Anholt673a3942008-07-30 12:06:12 -07002239 mutex_unlock(&dev->struct_mutex);
2240}
2241
Ben Widawsky5816d642012-04-11 11:18:19 -07002242/**
Daniel Vetter30dfebf2012-06-01 15:21:23 +02002243 * Ensures that an object will eventually get non-busy by flushing any required
2244 * write domains, emitting any outstanding lazy request and retiring and
2245 * completed requests.
2246 */
2247static int
2248i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2249{
2250 int ret;
2251
2252 if (obj->active) {
Chris Wilson0201f1e2012-07-20 12:41:01 +01002253 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
Daniel Vetter30dfebf2012-06-01 15:21:23 +02002254 if (ret)
2255 return ret;
Chris Wilson0201f1e2012-07-20 12:41:01 +01002256
Daniel Vetter30dfebf2012-06-01 15:21:23 +02002257 i915_gem_retire_requests_ring(obj->ring);
2258 }
2259
2260 return 0;
2261}
2262
2263/**
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002264 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2265 * @DRM_IOCTL_ARGS: standard ioctl arguments
2266 *
2267 * Returns 0 if successful, else an error is returned with the remaining time in
2268 * the timeout parameter.
2269 * -ETIME: object is still busy after timeout
2270 * -ERESTARTSYS: signal interrupted the wait
2271 * -ENONENT: object doesn't exist
2272 * Also possible, but rare:
2273 * -EAGAIN: GPU wedged
2274 * -ENOMEM: damn
2275 * -ENODEV: Internal IRQ fail
2276 * -E?: The add request failed
2277 *
2278 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2279 * non-zero timeout parameter the wait ioctl will wait for the given number of
2280 * nanoseconds on an object becoming unbusy. Since the wait itself does so
2281 * without holding struct_mutex the object may become re-busied before this
2282 * function completes. A similar but shorter * race condition exists in the busy
2283 * ioctl
2284 */
2285int
2286i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2287{
2288 struct drm_i915_gem_wait *args = data;
2289 struct drm_i915_gem_object *obj;
2290 struct intel_ring_buffer *ring = NULL;
Ben Widawskyeac1f142012-06-05 15:24:24 -07002291 struct timespec timeout_stack, *timeout = NULL;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002292 u32 seqno = 0;
2293 int ret = 0;
2294
Ben Widawskyeac1f142012-06-05 15:24:24 -07002295 if (args->timeout_ns >= 0) {
2296 timeout_stack = ns_to_timespec(args->timeout_ns);
2297 timeout = &timeout_stack;
2298 }
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002299
2300 ret = i915_mutex_lock_interruptible(dev);
2301 if (ret)
2302 return ret;
2303
2304 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2305 if (&obj->base == NULL) {
2306 mutex_unlock(&dev->struct_mutex);
2307 return -ENOENT;
2308 }
2309
Daniel Vetter30dfebf2012-06-01 15:21:23 +02002310 /* Need to make sure the object gets inactive eventually. */
2311 ret = i915_gem_object_flush_active(obj);
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002312 if (ret)
2313 goto out;
2314
2315 if (obj->active) {
Chris Wilson0201f1e2012-07-20 12:41:01 +01002316 seqno = obj->last_read_seqno;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002317 ring = obj->ring;
2318 }
2319
2320 if (seqno == 0)
2321 goto out;
2322
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002323 /* Do this after OLR check to make sure we make forward progress polling
2324 * on this IOCTL with a 0 timeout (like busy ioctl)
2325 */
2326 if (!args->timeout_ns) {
2327 ret = -ETIME;
2328 goto out;
2329 }
2330
2331 drm_gem_object_unreference(&obj->base);
2332 mutex_unlock(&dev->struct_mutex);
2333
Ben Widawskyeac1f142012-06-05 15:24:24 -07002334 ret = __wait_seqno(ring, seqno, true, timeout);
2335 if (timeout) {
2336 WARN_ON(!timespec_valid(timeout));
2337 args->timeout_ns = timespec_to_ns(timeout);
2338 }
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002339 return ret;
2340
2341out:
2342 drm_gem_object_unreference(&obj->base);
2343 mutex_unlock(&dev->struct_mutex);
2344 return ret;
2345}
2346
2347/**
Ben Widawsky5816d642012-04-11 11:18:19 -07002348 * i915_gem_object_sync - sync an object to a ring.
2349 *
2350 * @obj: object which may be in use on another ring.
2351 * @to: ring we wish to use the object on. May be NULL.
2352 *
2353 * This code is meant to abstract object synchronization with the GPU.
2354 * Calling with NULL implies synchronizing the object with the CPU
2355 * rather than a particular GPU ring.
2356 *
2357 * Returns 0 if successful, else propagates up the lower layer error.
2358 */
Ben Widawsky2911a352012-04-05 14:47:36 -07002359int
2360i915_gem_object_sync(struct drm_i915_gem_object *obj,
2361 struct intel_ring_buffer *to)
2362{
2363 struct intel_ring_buffer *from = obj->ring;
2364 u32 seqno;
2365 int ret, idx;
2366
2367 if (from == NULL || to == from)
2368 return 0;
2369
Ben Widawsky5816d642012-04-11 11:18:19 -07002370 if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
Chris Wilson0201f1e2012-07-20 12:41:01 +01002371 return i915_gem_object_wait_rendering(obj, false);
Ben Widawsky2911a352012-04-05 14:47:36 -07002372
2373 idx = intel_ring_sync_index(from, to);
2374
Chris Wilson0201f1e2012-07-20 12:41:01 +01002375 seqno = obj->last_read_seqno;
Ben Widawsky2911a352012-04-05 14:47:36 -07002376 if (seqno <= from->sync_seqno[idx])
2377 return 0;
2378
Ben Widawskyb4aca012012-04-25 20:50:12 -07002379 ret = i915_gem_check_olr(obj->ring, seqno);
2380 if (ret)
2381 return ret;
Ben Widawsky2911a352012-04-05 14:47:36 -07002382
Ben Widawsky1500f7e2012-04-11 11:18:21 -07002383 ret = to->sync_to(to, from, seqno);
Ben Widawskye3a5a222012-04-11 11:18:20 -07002384 if (!ret)
2385 from->sync_seqno[idx] = seqno;
Ben Widawsky2911a352012-04-05 14:47:36 -07002386
Ben Widawskye3a5a222012-04-11 11:18:20 -07002387 return ret;
Ben Widawsky2911a352012-04-05 14:47:36 -07002388}
2389
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01002390static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2391{
2392 u32 old_write_domain, old_read_domains;
2393
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01002394 /* Act a barrier for all accesses through the GTT */
2395 mb();
2396
2397 /* Force a pagefault for domain tracking on next user access */
2398 i915_gem_release_mmap(obj);
2399
Keith Packardb97c3d92011-06-24 21:02:59 -07002400 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2401 return;
2402
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01002403 old_read_domains = obj->base.read_domains;
2404 old_write_domain = obj->base.write_domain;
2405
2406 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2407 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2408
2409 trace_i915_gem_object_change_domain(obj,
2410 old_read_domains,
2411 old_write_domain);
2412}
2413
Eric Anholt673a3942008-07-30 12:06:12 -07002414/**
2415 * Unbinds an object from the GTT aperture.
2416 */
Jesse Barnes0f973f22009-01-26 17:10:45 -08002417int
Chris Wilson05394f32010-11-08 19:18:58 +00002418i915_gem_object_unbind(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07002419{
Daniel Vetter7bddb012012-02-09 17:15:47 +01002420 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07002421 int ret = 0;
2422
Chris Wilson05394f32010-11-08 19:18:58 +00002423 if (obj->gtt_space == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07002424 return 0;
2425
Chris Wilson31d8d652012-05-24 19:11:20 +01002426 if (obj->pin_count)
2427 return -EBUSY;
Eric Anholt673a3942008-07-30 12:06:12 -07002428
Chris Wilsonc4670ad2012-08-20 10:23:27 +01002429 BUG_ON(obj->pages == NULL);
2430
Chris Wilsona8198ee2011-04-13 22:04:09 +01002431 ret = i915_gem_object_finish_gpu(obj);
Chris Wilson1488fc02012-04-24 15:47:31 +01002432 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07002433 return ret;
Chris Wilson8dc17752010-07-23 23:18:51 +01002434 /* Continue on if we fail due to EIO, the GPU is hung so we
2435 * should be safe and we need to cleanup or else we might
2436 * cause memory corruption through use-after-free.
2437 */
Chris Wilsona8198ee2011-04-13 22:04:09 +01002438
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01002439 i915_gem_object_finish_gtt(obj);
Chris Wilsona8198ee2011-04-13 22:04:09 +01002440
Daniel Vetter96b47b62009-12-15 17:50:00 +01002441 /* release the fence reg _after_ flushing */
Chris Wilsond9e86c02010-11-10 16:40:20 +00002442 ret = i915_gem_object_put_fence(obj);
Chris Wilson1488fc02012-04-24 15:47:31 +01002443 if (ret)
Chris Wilsond9e86c02010-11-10 16:40:20 +00002444 return ret;
Daniel Vetter96b47b62009-12-15 17:50:00 +01002445
Chris Wilsondb53a302011-02-03 11:57:46 +00002446 trace_i915_gem_object_unbind(obj);
2447
Daniel Vetter74898d72012-02-15 23:50:22 +01002448 if (obj->has_global_gtt_mapping)
2449 i915_gem_gtt_unbind_object(obj);
Daniel Vetter7bddb012012-02-09 17:15:47 +01002450 if (obj->has_aliasing_ppgtt_mapping) {
2451 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2452 obj->has_aliasing_ppgtt_mapping = 0;
2453 }
Daniel Vetter74163902012-02-15 23:50:21 +01002454 i915_gem_gtt_finish_object(obj);
Daniel Vetter7bddb012012-02-09 17:15:47 +01002455
Chris Wilson6c085a72012-08-20 11:40:46 +02002456 list_del(&obj->mm_list);
2457 list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
Daniel Vetter75e9e912010-11-04 17:11:09 +01002458 /* Avoid an unnecessary call to unbind on rebind. */
Chris Wilson05394f32010-11-08 19:18:58 +00002459 obj->map_and_fenceable = true;
Eric Anholt673a3942008-07-30 12:06:12 -07002460
Chris Wilson05394f32010-11-08 19:18:58 +00002461 drm_mm_put_block(obj->gtt_space);
2462 obj->gtt_space = NULL;
2463 obj->gtt_offset = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07002464
Chris Wilson6c085a72012-08-20 11:40:46 +02002465 return 0;
Eric Anholt673a3942008-07-30 12:06:12 -07002466}
2467
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07002468static int i915_ring_idle(struct intel_ring_buffer *ring)
Chris Wilsona56ba562010-09-28 10:07:56 +01002469{
Chris Wilson69c2fc82012-07-20 12:41:03 +01002470 if (list_empty(&ring->active_list))
Chris Wilson64193402010-10-24 12:38:05 +01002471 return 0;
2472
Ben Widawsky199b2bc2012-05-24 15:03:11 -07002473 return i915_wait_seqno(ring, i915_gem_next_request_seqno(ring));
Chris Wilsona56ba562010-09-28 10:07:56 +01002474}
2475
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07002476int i915_gpu_idle(struct drm_device *dev)
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002477{
2478 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonb4519512012-05-11 14:29:30 +01002479 struct intel_ring_buffer *ring;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002480 int ret, i;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002481
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002482 /* Flush everything onto the inactive list. */
Chris Wilsonb4519512012-05-11 14:29:30 +01002483 for_each_ring(ring, dev_priv, i) {
Ben Widawskyb6c74882012-08-14 14:35:14 -07002484 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002485 if (ret)
2486 return ret;
Chris Wilsonb4519512012-05-11 14:29:30 +01002487
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002488 ret = i915_ring_idle(ring);
Ben Widawskyf2ef6eb2012-06-04 14:42:53 -07002489 if (ret)
2490 return ret;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002491 }
Zou Nan haid1b851f2010-05-21 09:08:57 +08002492
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01002493 return 0;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002494}
2495
Chris Wilson9ce079e2012-04-17 15:31:30 +01002496static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
2497 struct drm_i915_gem_object *obj)
Eric Anholt4e901fd2009-10-26 16:44:17 -07002498{
Eric Anholt4e901fd2009-10-26 16:44:17 -07002499 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt4e901fd2009-10-26 16:44:17 -07002500 uint64_t val;
2501
Chris Wilson9ce079e2012-04-17 15:31:30 +01002502 if (obj) {
2503 u32 size = obj->gtt_space->size;
Eric Anholt4e901fd2009-10-26 16:44:17 -07002504
Chris Wilson9ce079e2012-04-17 15:31:30 +01002505 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2506 0xfffff000) << 32;
2507 val |= obj->gtt_offset & 0xfffff000;
2508 val |= (uint64_t)((obj->stride / 128) - 1) <<
2509 SANDYBRIDGE_FENCE_PITCH_SHIFT;
Eric Anholt4e901fd2009-10-26 16:44:17 -07002510
Chris Wilson9ce079e2012-04-17 15:31:30 +01002511 if (obj->tiling_mode == I915_TILING_Y)
2512 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2513 val |= I965_FENCE_REG_VALID;
2514 } else
2515 val = 0;
Daniel Vetterc6642782010-11-12 13:46:18 +00002516
Chris Wilson9ce079e2012-04-17 15:31:30 +01002517 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
2518 POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
Eric Anholt4e901fd2009-10-26 16:44:17 -07002519}
2520
Chris Wilson9ce079e2012-04-17 15:31:30 +01002521static void i965_write_fence_reg(struct drm_device *dev, int reg,
2522 struct drm_i915_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002523{
Jesse Barnesde151cf2008-11-12 10:03:55 -08002524 drm_i915_private_t *dev_priv = dev->dev_private;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002525 uint64_t val;
2526
Chris Wilson9ce079e2012-04-17 15:31:30 +01002527 if (obj) {
2528 u32 size = obj->gtt_space->size;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002529
Chris Wilson9ce079e2012-04-17 15:31:30 +01002530 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2531 0xfffff000) << 32;
2532 val |= obj->gtt_offset & 0xfffff000;
2533 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2534 if (obj->tiling_mode == I915_TILING_Y)
2535 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2536 val |= I965_FENCE_REG_VALID;
2537 } else
2538 val = 0;
Daniel Vetterc6642782010-11-12 13:46:18 +00002539
Chris Wilson9ce079e2012-04-17 15:31:30 +01002540 I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
2541 POSTING_READ(FENCE_REG_965_0 + reg * 8);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002542}
2543
Chris Wilson9ce079e2012-04-17 15:31:30 +01002544static void i915_write_fence_reg(struct drm_device *dev, int reg,
2545 struct drm_i915_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002546{
Jesse Barnesde151cf2008-11-12 10:03:55 -08002547 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson9ce079e2012-04-17 15:31:30 +01002548 u32 val;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002549
Chris Wilson9ce079e2012-04-17 15:31:30 +01002550 if (obj) {
2551 u32 size = obj->gtt_space->size;
2552 int pitch_val;
2553 int tile_width;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002554
Chris Wilson9ce079e2012-04-17 15:31:30 +01002555 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2556 (size & -size) != size ||
2557 (obj->gtt_offset & (size - 1)),
2558 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2559 obj->gtt_offset, obj->map_and_fenceable, size);
2560
2561 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2562 tile_width = 128;
2563 else
2564 tile_width = 512;
2565
2566 /* Note: pitch better be a power of two tile widths */
2567 pitch_val = obj->stride / tile_width;
2568 pitch_val = ffs(pitch_val) - 1;
2569
2570 val = obj->gtt_offset;
2571 if (obj->tiling_mode == I915_TILING_Y)
2572 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2573 val |= I915_FENCE_SIZE_BITS(size);
2574 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2575 val |= I830_FENCE_REG_VALID;
2576 } else
2577 val = 0;
2578
2579 if (reg < 8)
2580 reg = FENCE_REG_830_0 + reg * 4;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002581 else
Chris Wilson9ce079e2012-04-17 15:31:30 +01002582 reg = FENCE_REG_945_8 + (reg - 8) * 4;
Jesse Barnes0f973f22009-01-26 17:10:45 -08002583
Chris Wilson9ce079e2012-04-17 15:31:30 +01002584 I915_WRITE(reg, val);
2585 POSTING_READ(reg);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002586}
2587
Chris Wilson9ce079e2012-04-17 15:31:30 +01002588static void i830_write_fence_reg(struct drm_device *dev, int reg,
2589 struct drm_i915_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002590{
Jesse Barnesde151cf2008-11-12 10:03:55 -08002591 drm_i915_private_t *dev_priv = dev->dev_private;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002592 uint32_t val;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002593
Chris Wilson9ce079e2012-04-17 15:31:30 +01002594 if (obj) {
2595 u32 size = obj->gtt_space->size;
2596 uint32_t pitch_val;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002597
Chris Wilson9ce079e2012-04-17 15:31:30 +01002598 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2599 (size & -size) != size ||
2600 (obj->gtt_offset & (size - 1)),
2601 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2602 obj->gtt_offset, size);
Eric Anholte76a16d2009-05-26 17:44:56 -07002603
Chris Wilson9ce079e2012-04-17 15:31:30 +01002604 pitch_val = obj->stride / 128;
2605 pitch_val = ffs(pitch_val) - 1;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002606
Chris Wilson9ce079e2012-04-17 15:31:30 +01002607 val = obj->gtt_offset;
2608 if (obj->tiling_mode == I915_TILING_Y)
2609 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2610 val |= I830_FENCE_SIZE_BITS(size);
2611 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2612 val |= I830_FENCE_REG_VALID;
2613 } else
2614 val = 0;
Daniel Vetterc6642782010-11-12 13:46:18 +00002615
Chris Wilson9ce079e2012-04-17 15:31:30 +01002616 I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2617 POSTING_READ(FENCE_REG_830_0 + reg * 4);
2618}
2619
2620static void i915_gem_write_fence(struct drm_device *dev, int reg,
2621 struct drm_i915_gem_object *obj)
2622{
2623 switch (INTEL_INFO(dev)->gen) {
2624 case 7:
2625 case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
2626 case 5:
2627 case 4: i965_write_fence_reg(dev, reg, obj); break;
2628 case 3: i915_write_fence_reg(dev, reg, obj); break;
2629 case 2: i830_write_fence_reg(dev, reg, obj); break;
2630 default: break;
2631 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08002632}
2633
Chris Wilson61050802012-04-17 15:31:31 +01002634static inline int fence_number(struct drm_i915_private *dev_priv,
2635 struct drm_i915_fence_reg *fence)
2636{
2637 return fence - dev_priv->fence_regs;
2638}
2639
2640static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2641 struct drm_i915_fence_reg *fence,
2642 bool enable)
2643{
2644 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2645 int reg = fence_number(dev_priv, fence);
2646
2647 i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2648
2649 if (enable) {
2650 obj->fence_reg = reg;
2651 fence->obj = obj;
2652 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2653 } else {
2654 obj->fence_reg = I915_FENCE_REG_NONE;
2655 fence->obj = NULL;
2656 list_del_init(&fence->lru_list);
2657 }
2658}
2659
Chris Wilsond9e86c02010-11-10 16:40:20 +00002660static int
Chris Wilsona360bb12012-04-17 15:31:25 +01002661i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
Chris Wilsond9e86c02010-11-10 16:40:20 +00002662{
Chris Wilson1c293ea2012-04-17 15:31:27 +01002663 if (obj->last_fenced_seqno) {
Chris Wilson86d5bc32012-07-20 12:41:04 +01002664 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
Chris Wilson18991842012-04-17 15:31:29 +01002665 if (ret)
2666 return ret;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002667
2668 obj->last_fenced_seqno = 0;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002669 }
2670
Chris Wilson63256ec2011-01-04 18:42:07 +00002671 /* Ensure that all CPU reads are completed before installing a fence
2672 * and all writes before removing the fence.
2673 */
2674 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2675 mb();
2676
Chris Wilson86d5bc32012-07-20 12:41:04 +01002677 obj->fenced_gpu_access = false;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002678 return 0;
2679}
2680
2681int
2682i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2683{
Chris Wilson61050802012-04-17 15:31:31 +01002684 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002685 int ret;
2686
Chris Wilsona360bb12012-04-17 15:31:25 +01002687 ret = i915_gem_object_flush_fence(obj);
Chris Wilsond9e86c02010-11-10 16:40:20 +00002688 if (ret)
2689 return ret;
2690
Chris Wilson61050802012-04-17 15:31:31 +01002691 if (obj->fence_reg == I915_FENCE_REG_NONE)
2692 return 0;
Chris Wilson1690e1e2011-12-14 13:57:08 +01002693
Chris Wilson61050802012-04-17 15:31:31 +01002694 i915_gem_object_update_fence(obj,
2695 &dev_priv->fence_regs[obj->fence_reg],
2696 false);
2697 i915_gem_object_fence_lost(obj);
Chris Wilsond9e86c02010-11-10 16:40:20 +00002698
2699 return 0;
2700}
2701
2702static struct drm_i915_fence_reg *
Chris Wilsona360bb12012-04-17 15:31:25 +01002703i915_find_fence_reg(struct drm_device *dev)
Daniel Vetterae3db242010-02-19 11:51:58 +01002704{
Daniel Vetterae3db242010-02-19 11:51:58 +01002705 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson8fe301a2012-04-17 15:31:28 +01002706 struct drm_i915_fence_reg *reg, *avail;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002707 int i;
Daniel Vetterae3db242010-02-19 11:51:58 +01002708
2709 /* First try to find a free reg */
Chris Wilsond9e86c02010-11-10 16:40:20 +00002710 avail = NULL;
Daniel Vetterae3db242010-02-19 11:51:58 +01002711 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2712 reg = &dev_priv->fence_regs[i];
2713 if (!reg->obj)
Chris Wilsond9e86c02010-11-10 16:40:20 +00002714 return reg;
Daniel Vetterae3db242010-02-19 11:51:58 +01002715
Chris Wilson1690e1e2011-12-14 13:57:08 +01002716 if (!reg->pin_count)
Chris Wilsond9e86c02010-11-10 16:40:20 +00002717 avail = reg;
Daniel Vetterae3db242010-02-19 11:51:58 +01002718 }
2719
Chris Wilsond9e86c02010-11-10 16:40:20 +00002720 if (avail == NULL)
2721 return NULL;
Daniel Vetterae3db242010-02-19 11:51:58 +01002722
2723 /* None available, try to steal one or wait for a user to finish */
Chris Wilsond9e86c02010-11-10 16:40:20 +00002724 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
Chris Wilson1690e1e2011-12-14 13:57:08 +01002725 if (reg->pin_count)
Daniel Vetterae3db242010-02-19 11:51:58 +01002726 continue;
2727
Chris Wilson8fe301a2012-04-17 15:31:28 +01002728 return reg;
Daniel Vetterae3db242010-02-19 11:51:58 +01002729 }
2730
Chris Wilson8fe301a2012-04-17 15:31:28 +01002731 return NULL;
Daniel Vetterae3db242010-02-19 11:51:58 +01002732}
2733
Jesse Barnesde151cf2008-11-12 10:03:55 -08002734/**
Chris Wilson9a5a53b2012-03-22 15:10:00 +00002735 * i915_gem_object_get_fence - set up fencing for an object
Jesse Barnesde151cf2008-11-12 10:03:55 -08002736 * @obj: object to map through a fence reg
2737 *
2738 * When mapping objects through the GTT, userspace wants to be able to write
2739 * to them without having to worry about swizzling if the object is tiled.
Jesse Barnesde151cf2008-11-12 10:03:55 -08002740 * This function walks the fence regs looking for a free one for @obj,
2741 * stealing one if it can't find any.
2742 *
2743 * It then sets up the reg based on the object's properties: address, pitch
2744 * and tiling format.
Chris Wilson9a5a53b2012-03-22 15:10:00 +00002745 *
2746 * For an untiled surface, this removes any existing fence.
Jesse Barnesde151cf2008-11-12 10:03:55 -08002747 */
Chris Wilson8c4b8c32009-06-17 22:08:52 +01002748int
Chris Wilson06d98132012-04-17 15:31:24 +01002749i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002750{
Chris Wilson05394f32010-11-08 19:18:58 +00002751 struct drm_device *dev = obj->base.dev;
Jesse Barnes79e53942008-11-07 14:24:08 -08002752 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson14415742012-04-17 15:31:33 +01002753 bool enable = obj->tiling_mode != I915_TILING_NONE;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002754 struct drm_i915_fence_reg *reg;
Daniel Vetterae3db242010-02-19 11:51:58 +01002755 int ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002756
Chris Wilson14415742012-04-17 15:31:33 +01002757 /* Have we updated the tiling parameters upon the object and so
2758 * will need to serialise the write to the associated fence register?
2759 */
Chris Wilson5d82e3e2012-04-21 16:23:23 +01002760 if (obj->fence_dirty) {
Chris Wilson14415742012-04-17 15:31:33 +01002761 ret = i915_gem_object_flush_fence(obj);
2762 if (ret)
2763 return ret;
2764 }
Chris Wilson9a5a53b2012-03-22 15:10:00 +00002765
Chris Wilsond9e86c02010-11-10 16:40:20 +00002766 /* Just update our place in the LRU if our fence is getting reused. */
Chris Wilson05394f32010-11-08 19:18:58 +00002767 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2768 reg = &dev_priv->fence_regs[obj->fence_reg];
Chris Wilson5d82e3e2012-04-21 16:23:23 +01002769 if (!obj->fence_dirty) {
Chris Wilson14415742012-04-17 15:31:33 +01002770 list_move_tail(&reg->lru_list,
2771 &dev_priv->mm.fence_list);
2772 return 0;
2773 }
2774 } else if (enable) {
2775 reg = i915_find_fence_reg(dev);
2776 if (reg == NULL)
2777 return -EDEADLK;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002778
Chris Wilson14415742012-04-17 15:31:33 +01002779 if (reg->obj) {
2780 struct drm_i915_gem_object *old = reg->obj;
2781
2782 ret = i915_gem_object_flush_fence(old);
Chris Wilson29c5a582011-03-17 15:23:22 +00002783 if (ret)
2784 return ret;
2785
Chris Wilson14415742012-04-17 15:31:33 +01002786 i915_gem_object_fence_lost(old);
Chris Wilson29c5a582011-03-17 15:23:22 +00002787 }
Chris Wilson14415742012-04-17 15:31:33 +01002788 } else
Eric Anholta09ba7f2009-08-29 12:49:51 -07002789 return 0;
Eric Anholta09ba7f2009-08-29 12:49:51 -07002790
Chris Wilson14415742012-04-17 15:31:33 +01002791 i915_gem_object_update_fence(obj, reg, enable);
Chris Wilson5d82e3e2012-04-21 16:23:23 +01002792 obj->fence_dirty = false;
Chris Wilson14415742012-04-17 15:31:33 +01002793
Chris Wilson9ce079e2012-04-17 15:31:30 +01002794 return 0;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002795}
2796
Chris Wilson42d6ab42012-07-26 11:49:32 +01002797static bool i915_gem_valid_gtt_space(struct drm_device *dev,
2798 struct drm_mm_node *gtt_space,
2799 unsigned long cache_level)
2800{
2801 struct drm_mm_node *other;
2802
2803 /* On non-LLC machines we have to be careful when putting differing
2804 * types of snoopable memory together to avoid the prefetcher
2805 * crossing memory domains and dieing.
2806 */
2807 if (HAS_LLC(dev))
2808 return true;
2809
2810 if (gtt_space == NULL)
2811 return true;
2812
2813 if (list_empty(&gtt_space->node_list))
2814 return true;
2815
2816 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
2817 if (other->allocated && !other->hole_follows && other->color != cache_level)
2818 return false;
2819
2820 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
2821 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
2822 return false;
2823
2824 return true;
2825}
2826
2827static void i915_gem_verify_gtt(struct drm_device *dev)
2828{
2829#if WATCH_GTT
2830 struct drm_i915_private *dev_priv = dev->dev_private;
2831 struct drm_i915_gem_object *obj;
2832 int err = 0;
2833
2834 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
2835 if (obj->gtt_space == NULL) {
2836 printk(KERN_ERR "object found on GTT list with no space reserved\n");
2837 err++;
2838 continue;
2839 }
2840
2841 if (obj->cache_level != obj->gtt_space->color) {
2842 printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
2843 obj->gtt_space->start,
2844 obj->gtt_space->start + obj->gtt_space->size,
2845 obj->cache_level,
2846 obj->gtt_space->color);
2847 err++;
2848 continue;
2849 }
2850
2851 if (!i915_gem_valid_gtt_space(dev,
2852 obj->gtt_space,
2853 obj->cache_level)) {
2854 printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
2855 obj->gtt_space->start,
2856 obj->gtt_space->start + obj->gtt_space->size,
2857 obj->cache_level);
2858 err++;
2859 continue;
2860 }
2861 }
2862
2863 WARN_ON(err);
2864#endif
2865}
2866
Jesse Barnesde151cf2008-11-12 10:03:55 -08002867/**
Eric Anholt673a3942008-07-30 12:06:12 -07002868 * Finds free space in the GTT aperture and binds the object there.
2869 */
2870static int
Chris Wilson05394f32010-11-08 19:18:58 +00002871i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
Daniel Vetter920afa72010-09-16 17:54:23 +02002872 unsigned alignment,
Chris Wilson86a1ee22012-08-11 15:41:04 +01002873 bool map_and_fenceable,
2874 bool nonblocking)
Eric Anholt673a3942008-07-30 12:06:12 -07002875{
Chris Wilson05394f32010-11-08 19:18:58 +00002876 struct drm_device *dev = obj->base.dev;
Eric Anholt673a3942008-07-30 12:06:12 -07002877 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07002878 struct drm_mm_node *free_space;
Daniel Vetter5e783302010-11-14 22:32:36 +01002879 u32 size, fence_size, fence_alignment, unfenced_alignment;
Daniel Vetter75e9e912010-11-04 17:11:09 +01002880 bool mappable, fenceable;
Chris Wilson07f73f62009-09-14 16:50:30 +01002881 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002882
Chris Wilson05394f32010-11-08 19:18:58 +00002883 if (obj->madv != I915_MADV_WILLNEED) {
Chris Wilson3ef94da2009-09-14 16:50:29 +01002884 DRM_ERROR("Attempting to bind a purgeable object\n");
2885 return -EINVAL;
2886 }
2887
Chris Wilsone28f8712011-07-18 13:11:49 -07002888 fence_size = i915_gem_get_gtt_size(dev,
2889 obj->base.size,
2890 obj->tiling_mode);
2891 fence_alignment = i915_gem_get_gtt_alignment(dev,
2892 obj->base.size,
2893 obj->tiling_mode);
2894 unfenced_alignment =
2895 i915_gem_get_unfenced_gtt_alignment(dev,
2896 obj->base.size,
2897 obj->tiling_mode);
Chris Wilsona00b10c2010-09-24 21:15:47 +01002898
Eric Anholt673a3942008-07-30 12:06:12 -07002899 if (alignment == 0)
Daniel Vetter5e783302010-11-14 22:32:36 +01002900 alignment = map_and_fenceable ? fence_alignment :
2901 unfenced_alignment;
Daniel Vetter75e9e912010-11-04 17:11:09 +01002902 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
Eric Anholt673a3942008-07-30 12:06:12 -07002903 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2904 return -EINVAL;
2905 }
2906
Chris Wilson05394f32010-11-08 19:18:58 +00002907 size = map_and_fenceable ? fence_size : obj->base.size;
Chris Wilsona00b10c2010-09-24 21:15:47 +01002908
Chris Wilson654fc602010-05-27 13:18:21 +01002909 /* If the object is bigger than the entire aperture, reject it early
2910 * before evicting everything in a vain attempt to find space.
2911 */
Chris Wilson05394f32010-11-08 19:18:58 +00002912 if (obj->base.size >
Daniel Vetter75e9e912010-11-04 17:11:09 +01002913 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
Chris Wilson654fc602010-05-27 13:18:21 +01002914 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2915 return -E2BIG;
2916 }
2917
Chris Wilson37e680a2012-06-07 15:38:42 +01002918 ret = i915_gem_object_get_pages(obj);
Chris Wilson6c085a72012-08-20 11:40:46 +02002919 if (ret)
2920 return ret;
2921
Eric Anholt673a3942008-07-30 12:06:12 -07002922 search_free:
Daniel Vetter75e9e912010-11-04 17:11:09 +01002923 if (map_and_fenceable)
Daniel Vetter920afa72010-09-16 17:54:23 +02002924 free_space =
Chris Wilson42d6ab42012-07-26 11:49:32 +01002925 drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
2926 size, alignment, obj->cache_level,
2927 0, dev_priv->mm.gtt_mappable_end,
2928 false);
Daniel Vetter920afa72010-09-16 17:54:23 +02002929 else
Chris Wilson42d6ab42012-07-26 11:49:32 +01002930 free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
2931 size, alignment, obj->cache_level,
2932 false);
Daniel Vetter920afa72010-09-16 17:54:23 +02002933
2934 if (free_space != NULL) {
Daniel Vetter75e9e912010-11-04 17:11:09 +01002935 if (map_and_fenceable)
Chris Wilson05394f32010-11-08 19:18:58 +00002936 obj->gtt_space =
Daniel Vetter920afa72010-09-16 17:54:23 +02002937 drm_mm_get_block_range_generic(free_space,
Chris Wilson42d6ab42012-07-26 11:49:32 +01002938 size, alignment, obj->cache_level,
Chris Wilson6b9d89b2012-07-10 11:15:23 +01002939 0, dev_priv->mm.gtt_mappable_end,
Chris Wilson42d6ab42012-07-26 11:49:32 +01002940 false);
Daniel Vetter920afa72010-09-16 17:54:23 +02002941 else
Chris Wilson05394f32010-11-08 19:18:58 +00002942 obj->gtt_space =
Chris Wilson42d6ab42012-07-26 11:49:32 +01002943 drm_mm_get_block_generic(free_space,
2944 size, alignment, obj->cache_level,
2945 false);
Daniel Vetter920afa72010-09-16 17:54:23 +02002946 }
Chris Wilson05394f32010-11-08 19:18:58 +00002947 if (obj->gtt_space == NULL) {
Daniel Vetter75e9e912010-11-04 17:11:09 +01002948 ret = i915_gem_evict_something(dev, size, alignment,
Chris Wilson42d6ab42012-07-26 11:49:32 +01002949 obj->cache_level,
Chris Wilson86a1ee22012-08-11 15:41:04 +01002950 map_and_fenceable,
2951 nonblocking);
Chris Wilson97311292009-09-21 00:22:34 +01002952 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07002953 return ret;
Chris Wilson97311292009-09-21 00:22:34 +01002954
Eric Anholt673a3942008-07-30 12:06:12 -07002955 goto search_free;
2956 }
Chris Wilson42d6ab42012-07-26 11:49:32 +01002957 if (WARN_ON(!i915_gem_valid_gtt_space(dev,
2958 obj->gtt_space,
2959 obj->cache_level))) {
2960 drm_mm_put_block(obj->gtt_space);
2961 obj->gtt_space = NULL;
2962 return -EINVAL;
2963 }
Eric Anholt673a3942008-07-30 12:06:12 -07002964
Eric Anholt673a3942008-07-30 12:06:12 -07002965
Daniel Vetter74163902012-02-15 23:50:21 +01002966 ret = i915_gem_gtt_prepare_object(obj);
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01002967 if (ret) {
Chris Wilson05394f32010-11-08 19:18:58 +00002968 drm_mm_put_block(obj->gtt_space);
2969 obj->gtt_space = NULL;
Chris Wilson6c085a72012-08-20 11:40:46 +02002970 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002971 }
Eric Anholt673a3942008-07-30 12:06:12 -07002972
Daniel Vetter0ebb9822012-02-15 23:50:24 +01002973 if (!dev_priv->mm.aliasing_ppgtt)
2974 i915_gem_gtt_bind_object(obj, obj->cache_level);
Eric Anholt673a3942008-07-30 12:06:12 -07002975
Chris Wilson6c085a72012-08-20 11:40:46 +02002976 list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
Chris Wilson05394f32010-11-08 19:18:58 +00002977 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
Chris Wilsonbf1a1092010-08-07 11:01:20 +01002978
Chris Wilson6299f992010-11-24 12:23:44 +00002979 obj->gtt_offset = obj->gtt_space->start;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002980
Daniel Vetter75e9e912010-11-04 17:11:09 +01002981 fenceable =
Chris Wilson05394f32010-11-08 19:18:58 +00002982 obj->gtt_space->size == fence_size &&
Akshay Joshi0206e352011-08-16 15:34:10 -04002983 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
Chris Wilsona00b10c2010-09-24 21:15:47 +01002984
Daniel Vetter75e9e912010-11-04 17:11:09 +01002985 mappable =
Chris Wilson05394f32010-11-08 19:18:58 +00002986 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
Chris Wilsona00b10c2010-09-24 21:15:47 +01002987
Chris Wilson05394f32010-11-08 19:18:58 +00002988 obj->map_and_fenceable = mappable && fenceable;
Daniel Vetter75e9e912010-11-04 17:11:09 +01002989
Chris Wilsondb53a302011-02-03 11:57:46 +00002990 trace_i915_gem_object_bind(obj, map_and_fenceable);
Chris Wilson42d6ab42012-07-26 11:49:32 +01002991 i915_gem_verify_gtt(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07002992 return 0;
2993}
2994
2995void
Chris Wilson05394f32010-11-08 19:18:58 +00002996i915_gem_clflush_object(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07002997{
Eric Anholt673a3942008-07-30 12:06:12 -07002998 /* If we don't have a page list set up, then we're not pinned
2999 * to GPU, and we can ignore the cache flush because it'll happen
3000 * again at bind time.
3001 */
Chris Wilson05394f32010-11-08 19:18:58 +00003002 if (obj->pages == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07003003 return;
3004
Chris Wilson9c23f7f2011-03-29 16:59:52 -07003005 /* If the GPU is snooping the contents of the CPU cache,
3006 * we do not need to manually clear the CPU cache lines. However,
3007 * the caches are only snooped when the render cache is
3008 * flushed/invalidated. As we always have to emit invalidations
3009 * and flushes when moving into and out of the RENDER domain, correct
3010 * snooping behaviour occurs naturally as the result of our domain
3011 * tracking.
3012 */
3013 if (obj->cache_level != I915_CACHE_NONE)
3014 return;
3015
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003016 trace_i915_gem_object_clflush(obj);
Eric Anholtcfa16a02009-05-26 18:46:16 -07003017
Chris Wilson9da3da62012-06-01 15:20:22 +01003018 drm_clflush_sg(obj->pages);
Eric Anholt673a3942008-07-30 12:06:12 -07003019}
3020
Eric Anholte47c68e2008-11-14 13:35:19 -08003021/** Flushes the GTT write domain for the object if it's dirty. */
3022static void
Chris Wilson05394f32010-11-08 19:18:58 +00003023i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
Eric Anholte47c68e2008-11-14 13:35:19 -08003024{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003025 uint32_t old_write_domain;
3026
Chris Wilson05394f32010-11-08 19:18:58 +00003027 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
Eric Anholte47c68e2008-11-14 13:35:19 -08003028 return;
3029
Chris Wilson63256ec2011-01-04 18:42:07 +00003030 /* No actual flushing is required for the GTT write domain. Writes
Eric Anholte47c68e2008-11-14 13:35:19 -08003031 * to it immediately go to main memory as far as we know, so there's
3032 * no chipset flush. It also doesn't land in render cache.
Chris Wilson63256ec2011-01-04 18:42:07 +00003033 *
3034 * However, we do have to enforce the order so that all writes through
3035 * the GTT land before any writes to the device, such as updates to
3036 * the GATT itself.
Eric Anholte47c68e2008-11-14 13:35:19 -08003037 */
Chris Wilson63256ec2011-01-04 18:42:07 +00003038 wmb();
3039
Chris Wilson05394f32010-11-08 19:18:58 +00003040 old_write_domain = obj->base.write_domain;
3041 obj->base.write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003042
3043 trace_i915_gem_object_change_domain(obj,
Chris Wilson05394f32010-11-08 19:18:58 +00003044 obj->base.read_domains,
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003045 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08003046}
3047
3048/** Flushes the CPU write domain for the object if it's dirty. */
3049static void
Chris Wilson05394f32010-11-08 19:18:58 +00003050i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
Eric Anholte47c68e2008-11-14 13:35:19 -08003051{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003052 uint32_t old_write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08003053
Chris Wilson05394f32010-11-08 19:18:58 +00003054 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
Eric Anholte47c68e2008-11-14 13:35:19 -08003055 return;
3056
3057 i915_gem_clflush_object(obj);
Daniel Vetter40ce6572010-11-05 18:12:18 +01003058 intel_gtt_chipset_flush();
Chris Wilson05394f32010-11-08 19:18:58 +00003059 old_write_domain = obj->base.write_domain;
3060 obj->base.write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003061
3062 trace_i915_gem_object_change_domain(obj,
Chris Wilson05394f32010-11-08 19:18:58 +00003063 obj->base.read_domains,
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003064 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08003065}
3066
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003067/**
3068 * Moves a single object to the GTT read, and possibly write domain.
3069 *
3070 * This function returns when the move is complete, including waiting on
3071 * flushes to occur.
3072 */
Jesse Barnes79e53942008-11-07 14:24:08 -08003073int
Chris Wilson20217462010-11-23 15:26:33 +00003074i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003075{
Chris Wilson8325a092012-04-24 15:52:35 +01003076 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003077 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003078 int ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003079
Eric Anholt02354392008-11-26 13:58:13 -08003080 /* Not valid to be called on unbound objects. */
Chris Wilson05394f32010-11-08 19:18:58 +00003081 if (obj->gtt_space == NULL)
Eric Anholt02354392008-11-26 13:58:13 -08003082 return -EINVAL;
3083
Chris Wilson8d7e3de2011-02-07 15:23:02 +00003084 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3085 return 0;
3086
Chris Wilson0201f1e2012-07-20 12:41:01 +01003087 ret = i915_gem_object_wait_rendering(obj, !write);
3088 if (ret)
3089 return ret;
Chris Wilson2dafb1e2010-06-07 14:03:05 +01003090
Chris Wilson72133422010-09-13 23:56:38 +01003091 i915_gem_object_flush_cpu_write_domain(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003092
Chris Wilson05394f32010-11-08 19:18:58 +00003093 old_write_domain = obj->base.write_domain;
3094 old_read_domains = obj->base.read_domains;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003095
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003096 /* It should now be out of any other write domains, and we can update
3097 * the domain values for our changes.
3098 */
Chris Wilson05394f32010-11-08 19:18:58 +00003099 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3100 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08003101 if (write) {
Chris Wilson05394f32010-11-08 19:18:58 +00003102 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3103 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3104 obj->dirty = 1;
Eric Anholte47c68e2008-11-14 13:35:19 -08003105 }
3106
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003107 trace_i915_gem_object_change_domain(obj,
3108 old_read_domains,
3109 old_write_domain);
3110
Chris Wilson8325a092012-04-24 15:52:35 +01003111 /* And bump the LRU for this access */
3112 if (i915_gem_object_is_inactive(obj))
3113 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3114
Eric Anholte47c68e2008-11-14 13:35:19 -08003115 return 0;
3116}
3117
Chris Wilsone4ffd172011-04-04 09:44:39 +01003118int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3119 enum i915_cache_level cache_level)
3120{
Daniel Vetter7bddb012012-02-09 17:15:47 +01003121 struct drm_device *dev = obj->base.dev;
3122 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsone4ffd172011-04-04 09:44:39 +01003123 int ret;
3124
3125 if (obj->cache_level == cache_level)
3126 return 0;
3127
3128 if (obj->pin_count) {
3129 DRM_DEBUG("can not change the cache level of pinned objects\n");
3130 return -EBUSY;
3131 }
3132
Chris Wilson42d6ab42012-07-26 11:49:32 +01003133 if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
3134 ret = i915_gem_object_unbind(obj);
3135 if (ret)
3136 return ret;
3137 }
3138
Chris Wilsone4ffd172011-04-04 09:44:39 +01003139 if (obj->gtt_space) {
3140 ret = i915_gem_object_finish_gpu(obj);
3141 if (ret)
3142 return ret;
3143
3144 i915_gem_object_finish_gtt(obj);
3145
3146 /* Before SandyBridge, you could not use tiling or fence
3147 * registers with snooped memory, so relinquish any fences
3148 * currently pointing to our region in the aperture.
3149 */
Chris Wilson42d6ab42012-07-26 11:49:32 +01003150 if (INTEL_INFO(dev)->gen < 6) {
Chris Wilsone4ffd172011-04-04 09:44:39 +01003151 ret = i915_gem_object_put_fence(obj);
3152 if (ret)
3153 return ret;
3154 }
3155
Daniel Vetter74898d72012-02-15 23:50:22 +01003156 if (obj->has_global_gtt_mapping)
3157 i915_gem_gtt_bind_object(obj, cache_level);
Daniel Vetter7bddb012012-02-09 17:15:47 +01003158 if (obj->has_aliasing_ppgtt_mapping)
3159 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
3160 obj, cache_level);
Chris Wilson42d6ab42012-07-26 11:49:32 +01003161
3162 obj->gtt_space->color = cache_level;
Chris Wilsone4ffd172011-04-04 09:44:39 +01003163 }
3164
3165 if (cache_level == I915_CACHE_NONE) {
3166 u32 old_read_domains, old_write_domain;
3167
3168 /* If we're coming from LLC cached, then we haven't
3169 * actually been tracking whether the data is in the
3170 * CPU cache or not, since we only allow one bit set
3171 * in obj->write_domain and have been skipping the clflushes.
3172 * Just set it to the CPU cache for now.
3173 */
3174 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3175 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
3176
3177 old_read_domains = obj->base.read_domains;
3178 old_write_domain = obj->base.write_domain;
3179
3180 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3181 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3182
3183 trace_i915_gem_object_change_domain(obj,
3184 old_read_domains,
3185 old_write_domain);
3186 }
3187
3188 obj->cache_level = cache_level;
Chris Wilson42d6ab42012-07-26 11:49:32 +01003189 i915_gem_verify_gtt(dev);
Chris Wilsone4ffd172011-04-04 09:44:39 +01003190 return 0;
3191}
3192
Ben Widawsky199adf42012-09-21 17:01:20 -07003193int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3194 struct drm_file *file)
Chris Wilsone6994ae2012-07-10 10:27:08 +01003195{
Ben Widawsky199adf42012-09-21 17:01:20 -07003196 struct drm_i915_gem_caching *args = data;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003197 struct drm_i915_gem_object *obj;
3198 int ret;
3199
3200 ret = i915_mutex_lock_interruptible(dev);
3201 if (ret)
3202 return ret;
3203
3204 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3205 if (&obj->base == NULL) {
3206 ret = -ENOENT;
3207 goto unlock;
3208 }
3209
Ben Widawsky199adf42012-09-21 17:01:20 -07003210 args->caching = obj->cache_level != I915_CACHE_NONE;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003211
3212 drm_gem_object_unreference(&obj->base);
3213unlock:
3214 mutex_unlock(&dev->struct_mutex);
3215 return ret;
3216}
3217
Ben Widawsky199adf42012-09-21 17:01:20 -07003218int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3219 struct drm_file *file)
Chris Wilsone6994ae2012-07-10 10:27:08 +01003220{
Ben Widawsky199adf42012-09-21 17:01:20 -07003221 struct drm_i915_gem_caching *args = data;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003222 struct drm_i915_gem_object *obj;
3223 enum i915_cache_level level;
3224 int ret;
3225
Ben Widawsky199adf42012-09-21 17:01:20 -07003226 switch (args->caching) {
3227 case I915_CACHING_NONE:
Chris Wilsone6994ae2012-07-10 10:27:08 +01003228 level = I915_CACHE_NONE;
3229 break;
Ben Widawsky199adf42012-09-21 17:01:20 -07003230 case I915_CACHING_CACHED:
Chris Wilsone6994ae2012-07-10 10:27:08 +01003231 level = I915_CACHE_LLC;
3232 break;
3233 default:
3234 return -EINVAL;
3235 }
3236
Ben Widawsky3bc29132012-09-26 16:15:20 -07003237 ret = i915_mutex_lock_interruptible(dev);
3238 if (ret)
3239 return ret;
3240
Chris Wilsone6994ae2012-07-10 10:27:08 +01003241 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3242 if (&obj->base == NULL) {
3243 ret = -ENOENT;
3244 goto unlock;
3245 }
3246
3247 ret = i915_gem_object_set_cache_level(obj, level);
3248
3249 drm_gem_object_unreference(&obj->base);
3250unlock:
3251 mutex_unlock(&dev->struct_mutex);
3252 return ret;
3253}
3254
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003255/*
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003256 * Prepare buffer for display plane (scanout, cursors, etc).
3257 * Can be called from an uninterruptible phase (modesetting) and allows
3258 * any flushes to be pipelined (for pageflips).
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003259 */
3260int
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003261i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3262 u32 alignment,
Chris Wilson919926a2010-11-12 13:42:53 +00003263 struct intel_ring_buffer *pipelined)
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003264{
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003265 u32 old_read_domains, old_write_domain;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003266 int ret;
3267
Chris Wilson0be73282010-12-06 14:36:27 +00003268 if (pipelined != obj->ring) {
Ben Widawsky2911a352012-04-05 14:47:36 -07003269 ret = i915_gem_object_sync(obj, pipelined);
3270 if (ret)
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003271 return ret;
3272 }
3273
Eric Anholta7ef0642011-03-29 16:59:54 -07003274 /* The display engine is not coherent with the LLC cache on gen6. As
3275 * a result, we make sure that the pinning that is about to occur is
3276 * done with uncached PTEs. This is lowest common denominator for all
3277 * chipsets.
3278 *
3279 * However for gen6+, we could do better by using the GFDT bit instead
3280 * of uncaching, which would allow us to flush all the LLC-cached data
3281 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3282 */
3283 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
3284 if (ret)
3285 return ret;
3286
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003287 /* As the user may map the buffer once pinned in the display plane
3288 * (e.g. libkms for the bootup splash), we have to ensure that we
3289 * always use map_and_fenceable for all scanout buffers.
3290 */
Chris Wilson86a1ee22012-08-11 15:41:04 +01003291 ret = i915_gem_object_pin(obj, alignment, true, false);
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003292 if (ret)
3293 return ret;
3294
Chris Wilsonb118c1e2010-05-27 13:18:14 +01003295 i915_gem_object_flush_cpu_write_domain(obj);
3296
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003297 old_write_domain = obj->base.write_domain;
Chris Wilson05394f32010-11-08 19:18:58 +00003298 old_read_domains = obj->base.read_domains;
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003299
3300 /* It should now be out of any other write domains, and we can update
3301 * the domain values for our changes.
3302 */
Chris Wilsone5f1d962012-07-20 12:41:00 +01003303 obj->base.write_domain = 0;
Chris Wilson05394f32010-11-08 19:18:58 +00003304 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003305
3306 trace_i915_gem_object_change_domain(obj,
3307 old_read_domains,
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003308 old_write_domain);
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003309
3310 return 0;
3311}
3312
Chris Wilson85345512010-11-13 09:49:11 +00003313int
Chris Wilsona8198ee2011-04-13 22:04:09 +01003314i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
Chris Wilson85345512010-11-13 09:49:11 +00003315{
Chris Wilson88241782011-01-07 17:09:48 +00003316 int ret;
3317
Chris Wilsona8198ee2011-04-13 22:04:09 +01003318 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
Chris Wilson85345512010-11-13 09:49:11 +00003319 return 0;
3320
Chris Wilson0201f1e2012-07-20 12:41:01 +01003321 ret = i915_gem_object_wait_rendering(obj, false);
Chris Wilsonc501ae72011-12-14 13:57:23 +01003322 if (ret)
3323 return ret;
3324
Chris Wilsona8198ee2011-04-13 22:04:09 +01003325 /* Ensure that we invalidate the GPU's caches and TLBs. */
3326 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
Chris Wilsonc501ae72011-12-14 13:57:23 +01003327 return 0;
Chris Wilson85345512010-11-13 09:49:11 +00003328}
3329
Eric Anholte47c68e2008-11-14 13:35:19 -08003330/**
3331 * Moves a single object to the CPU read, and possibly write domain.
3332 *
3333 * This function returns when the move is complete, including waiting on
3334 * flushes to occur.
3335 */
Chris Wilsondabdfe02012-03-26 10:10:27 +02003336int
Chris Wilson919926a2010-11-12 13:42:53 +00003337i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
Eric Anholte47c68e2008-11-14 13:35:19 -08003338{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003339 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003340 int ret;
3341
Chris Wilson8d7e3de2011-02-07 15:23:02 +00003342 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3343 return 0;
3344
Chris Wilson0201f1e2012-07-20 12:41:01 +01003345 ret = i915_gem_object_wait_rendering(obj, !write);
3346 if (ret)
3347 return ret;
Eric Anholte47c68e2008-11-14 13:35:19 -08003348
3349 i915_gem_object_flush_gtt_write_domain(obj);
3350
Chris Wilson05394f32010-11-08 19:18:58 +00003351 old_write_domain = obj->base.write_domain;
3352 old_read_domains = obj->base.read_domains;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003353
Eric Anholte47c68e2008-11-14 13:35:19 -08003354 /* Flush the CPU cache if it's still invalid. */
Chris Wilson05394f32010-11-08 19:18:58 +00003355 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
Eric Anholte47c68e2008-11-14 13:35:19 -08003356 i915_gem_clflush_object(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08003357
Chris Wilson05394f32010-11-08 19:18:58 +00003358 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
Eric Anholte47c68e2008-11-14 13:35:19 -08003359 }
3360
3361 /* It should now be out of any other write domains, and we can update
3362 * the domain values for our changes.
3363 */
Chris Wilson05394f32010-11-08 19:18:58 +00003364 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
Eric Anholte47c68e2008-11-14 13:35:19 -08003365
3366 /* If we're writing through the CPU, then the GPU read domains will
3367 * need to be invalidated at next use.
3368 */
3369 if (write) {
Chris Wilson05394f32010-11-08 19:18:58 +00003370 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3371 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
Eric Anholte47c68e2008-11-14 13:35:19 -08003372 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003373
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003374 trace_i915_gem_object_change_domain(obj,
3375 old_read_domains,
3376 old_write_domain);
3377
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003378 return 0;
3379}
3380
Eric Anholt673a3942008-07-30 12:06:12 -07003381/* Throttle our rendering by waiting until the ring has completed our requests
3382 * emitted over 20 msec ago.
3383 *
Eric Anholtb9624422009-06-03 07:27:35 +00003384 * Note that if we were to use the current jiffies each time around the loop,
3385 * we wouldn't escape the function with any frames outstanding if the time to
3386 * render a frame was over 20ms.
3387 *
Eric Anholt673a3942008-07-30 12:06:12 -07003388 * This should get us reasonable parallelism between CPU and GPU but also
3389 * relatively low latency when blocking on a particular request to finish.
3390 */
3391static int
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003392i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07003393{
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003394 struct drm_i915_private *dev_priv = dev->dev_private;
3395 struct drm_i915_file_private *file_priv = file->driver_priv;
Eric Anholtb9624422009-06-03 07:27:35 +00003396 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003397 struct drm_i915_gem_request *request;
3398 struct intel_ring_buffer *ring = NULL;
3399 u32 seqno = 0;
3400 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003401
Chris Wilsone110e8d2011-01-26 15:39:14 +00003402 if (atomic_read(&dev_priv->mm.wedged))
3403 return -EIO;
3404
Chris Wilson1c255952010-09-26 11:03:27 +01003405 spin_lock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003406 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
Eric Anholtb9624422009-06-03 07:27:35 +00003407 if (time_after_eq(request->emitted_jiffies, recent_enough))
3408 break;
3409
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003410 ring = request->ring;
3411 seqno = request->seqno;
Eric Anholtb9624422009-06-03 07:27:35 +00003412 }
Chris Wilson1c255952010-09-26 11:03:27 +01003413 spin_unlock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003414
3415 if (seqno == 0)
3416 return 0;
3417
Ben Widawsky5c81fe852012-05-24 15:03:08 -07003418 ret = __wait_seqno(ring, seqno, true, NULL);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003419 if (ret == 0)
3420 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
Eric Anholtb9624422009-06-03 07:27:35 +00003421
Eric Anholt673a3942008-07-30 12:06:12 -07003422 return ret;
3423}
3424
Eric Anholt673a3942008-07-30 12:06:12 -07003425int
Chris Wilson05394f32010-11-08 19:18:58 +00003426i915_gem_object_pin(struct drm_i915_gem_object *obj,
3427 uint32_t alignment,
Chris Wilson86a1ee22012-08-11 15:41:04 +01003428 bool map_and_fenceable,
3429 bool nonblocking)
Eric Anholt673a3942008-07-30 12:06:12 -07003430{
Eric Anholt673a3942008-07-30 12:06:12 -07003431 int ret;
3432
Chris Wilson7e81a422012-09-15 09:41:57 +01003433 if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3434 return -EBUSY;
Chris Wilsonac0c6b52010-05-27 13:18:18 +01003435
Chris Wilson05394f32010-11-08 19:18:58 +00003436 if (obj->gtt_space != NULL) {
3437 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3438 (map_and_fenceable && !obj->map_and_fenceable)) {
3439 WARN(obj->pin_count,
Chris Wilsonae7d49d2010-08-04 12:37:41 +01003440 "bo is already pinned with incorrect alignment:"
Daniel Vetter75e9e912010-11-04 17:11:09 +01003441 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3442 " obj->map_and_fenceable=%d\n",
Chris Wilson05394f32010-11-08 19:18:58 +00003443 obj->gtt_offset, alignment,
Daniel Vetter75e9e912010-11-04 17:11:09 +01003444 map_and_fenceable,
Chris Wilson05394f32010-11-08 19:18:58 +00003445 obj->map_and_fenceable);
Chris Wilsonac0c6b52010-05-27 13:18:18 +01003446 ret = i915_gem_object_unbind(obj);
3447 if (ret)
3448 return ret;
3449 }
3450 }
3451
Chris Wilson05394f32010-11-08 19:18:58 +00003452 if (obj->gtt_space == NULL) {
Chris Wilsona00b10c2010-09-24 21:15:47 +01003453 ret = i915_gem_object_bind_to_gtt(obj, alignment,
Chris Wilson86a1ee22012-08-11 15:41:04 +01003454 map_and_fenceable,
3455 nonblocking);
Chris Wilson97311292009-09-21 00:22:34 +01003456 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07003457 return ret;
Chris Wilson22c344e2009-02-11 14:26:45 +00003458 }
Jesse Barnes76446ca2009-12-17 22:05:42 -05003459
Daniel Vetter74898d72012-02-15 23:50:22 +01003460 if (!obj->has_global_gtt_mapping && map_and_fenceable)
3461 i915_gem_gtt_bind_object(obj, obj->cache_level);
3462
Chris Wilson1b502472012-04-24 15:47:30 +01003463 obj->pin_count++;
Chris Wilson6299f992010-11-24 12:23:44 +00003464 obj->pin_mappable |= map_and_fenceable;
Eric Anholt673a3942008-07-30 12:06:12 -07003465
3466 return 0;
3467}
3468
3469void
Chris Wilson05394f32010-11-08 19:18:58 +00003470i915_gem_object_unpin(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07003471{
Chris Wilson05394f32010-11-08 19:18:58 +00003472 BUG_ON(obj->pin_count == 0);
3473 BUG_ON(obj->gtt_space == NULL);
Eric Anholt673a3942008-07-30 12:06:12 -07003474
Chris Wilson1b502472012-04-24 15:47:30 +01003475 if (--obj->pin_count == 0)
Chris Wilson6299f992010-11-24 12:23:44 +00003476 obj->pin_mappable = false;
Eric Anholt673a3942008-07-30 12:06:12 -07003477}
3478
3479int
3480i915_gem_pin_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00003481 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07003482{
3483 struct drm_i915_gem_pin *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00003484 struct drm_i915_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07003485 int ret;
3486
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003487 ret = i915_mutex_lock_interruptible(dev);
3488 if (ret)
3489 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003490
Chris Wilson05394f32010-11-08 19:18:58 +00003491 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00003492 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003493 ret = -ENOENT;
3494 goto unlock;
Eric Anholt673a3942008-07-30 12:06:12 -07003495 }
Eric Anholt673a3942008-07-30 12:06:12 -07003496
Chris Wilson05394f32010-11-08 19:18:58 +00003497 if (obj->madv != I915_MADV_WILLNEED) {
Chris Wilsonbb6baf72009-09-22 14:24:13 +01003498 DRM_ERROR("Attempting to pin a purgeable buffer\n");
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003499 ret = -EINVAL;
3500 goto out;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003501 }
3502
Chris Wilson05394f32010-11-08 19:18:58 +00003503 if (obj->pin_filp != NULL && obj->pin_filp != file) {
Jesse Barnes79e53942008-11-07 14:24:08 -08003504 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3505 args->handle);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003506 ret = -EINVAL;
3507 goto out;
Jesse Barnes79e53942008-11-07 14:24:08 -08003508 }
3509
Chris Wilson05394f32010-11-08 19:18:58 +00003510 obj->user_pin_count++;
3511 obj->pin_filp = file;
3512 if (obj->user_pin_count == 1) {
Chris Wilson86a1ee22012-08-11 15:41:04 +01003513 ret = i915_gem_object_pin(obj, args->alignment, true, false);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003514 if (ret)
3515 goto out;
Eric Anholt673a3942008-07-30 12:06:12 -07003516 }
3517
3518 /* XXX - flush the CPU caches for pinned objects
3519 * as the X server doesn't manage domains yet
3520 */
Eric Anholte47c68e2008-11-14 13:35:19 -08003521 i915_gem_object_flush_cpu_write_domain(obj);
Chris Wilson05394f32010-11-08 19:18:58 +00003522 args->offset = obj->gtt_offset;
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003523out:
Chris Wilson05394f32010-11-08 19:18:58 +00003524 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003525unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07003526 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003527 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003528}
3529
3530int
3531i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00003532 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07003533{
3534 struct drm_i915_gem_pin *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00003535 struct drm_i915_gem_object *obj;
Chris Wilson76c1dec2010-09-25 11:22:51 +01003536 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003537
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003538 ret = i915_mutex_lock_interruptible(dev);
3539 if (ret)
3540 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003541
Chris Wilson05394f32010-11-08 19:18:58 +00003542 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00003543 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003544 ret = -ENOENT;
3545 goto unlock;
Eric Anholt673a3942008-07-30 12:06:12 -07003546 }
Chris Wilson76c1dec2010-09-25 11:22:51 +01003547
Chris Wilson05394f32010-11-08 19:18:58 +00003548 if (obj->pin_filp != file) {
Jesse Barnes79e53942008-11-07 14:24:08 -08003549 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3550 args->handle);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003551 ret = -EINVAL;
3552 goto out;
Jesse Barnes79e53942008-11-07 14:24:08 -08003553 }
Chris Wilson05394f32010-11-08 19:18:58 +00003554 obj->user_pin_count--;
3555 if (obj->user_pin_count == 0) {
3556 obj->pin_filp = NULL;
Jesse Barnes79e53942008-11-07 14:24:08 -08003557 i915_gem_object_unpin(obj);
3558 }
Eric Anholt673a3942008-07-30 12:06:12 -07003559
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003560out:
Chris Wilson05394f32010-11-08 19:18:58 +00003561 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003562unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07003563 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003564 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003565}
3566
3567int
3568i915_gem_busy_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00003569 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07003570{
3571 struct drm_i915_gem_busy *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00003572 struct drm_i915_gem_object *obj;
Chris Wilson30dbf0c2010-09-25 10:19:17 +01003573 int ret;
3574
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003575 ret = i915_mutex_lock_interruptible(dev);
3576 if (ret)
3577 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003578
Chris Wilson05394f32010-11-08 19:18:58 +00003579 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00003580 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003581 ret = -ENOENT;
3582 goto unlock;
Eric Anholt673a3942008-07-30 12:06:12 -07003583 }
Zou Nan haid1b851f2010-05-21 09:08:57 +08003584
Chris Wilson0be555b2010-08-04 15:36:30 +01003585 /* Count all active objects as busy, even if they are currently not used
3586 * by the gpu. Users of this interface expect objects to eventually
3587 * become non-busy without any further actions, therefore emit any
3588 * necessary flushes here.
Eric Anholtc4de0a52008-12-14 19:05:04 -08003589 */
Daniel Vetter30dfebf2012-06-01 15:21:23 +02003590 ret = i915_gem_object_flush_active(obj);
3591
Chris Wilson05394f32010-11-08 19:18:58 +00003592 args->busy = obj->active;
Chris Wilsone9808ed2012-07-04 12:25:08 +01003593 if (obj->ring) {
3594 BUILD_BUG_ON(I915_NUM_RINGS > 16);
3595 args->busy |= intel_ring_flag(obj->ring) << 16;
3596 }
Eric Anholt673a3942008-07-30 12:06:12 -07003597
Chris Wilson05394f32010-11-08 19:18:58 +00003598 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003599unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07003600 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003601 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003602}
3603
3604int
3605i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3606 struct drm_file *file_priv)
3607{
Akshay Joshi0206e352011-08-16 15:34:10 -04003608 return i915_gem_ring_throttle(dev, file_priv);
Eric Anholt673a3942008-07-30 12:06:12 -07003609}
3610
Chris Wilson3ef94da2009-09-14 16:50:29 +01003611int
3612i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3613 struct drm_file *file_priv)
3614{
3615 struct drm_i915_gem_madvise *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00003616 struct drm_i915_gem_object *obj;
Chris Wilson76c1dec2010-09-25 11:22:51 +01003617 int ret;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003618
3619 switch (args->madv) {
3620 case I915_MADV_DONTNEED:
3621 case I915_MADV_WILLNEED:
3622 break;
3623 default:
3624 return -EINVAL;
3625 }
3626
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003627 ret = i915_mutex_lock_interruptible(dev);
3628 if (ret)
3629 return ret;
3630
Chris Wilson05394f32010-11-08 19:18:58 +00003631 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00003632 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003633 ret = -ENOENT;
3634 goto unlock;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003635 }
Chris Wilson3ef94da2009-09-14 16:50:29 +01003636
Chris Wilson05394f32010-11-08 19:18:58 +00003637 if (obj->pin_count) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003638 ret = -EINVAL;
3639 goto out;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003640 }
3641
Chris Wilson05394f32010-11-08 19:18:58 +00003642 if (obj->madv != __I915_MADV_PURGED)
3643 obj->madv = args->madv;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003644
Chris Wilson6c085a72012-08-20 11:40:46 +02003645 /* if the object is no longer attached, discard its backing storage */
3646 if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
Chris Wilson2d7ef392009-09-20 23:13:10 +01003647 i915_gem_object_truncate(obj);
3648
Chris Wilson05394f32010-11-08 19:18:58 +00003649 args->retained = obj->madv != __I915_MADV_PURGED;
Chris Wilsonbb6baf72009-09-22 14:24:13 +01003650
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003651out:
Chris Wilson05394f32010-11-08 19:18:58 +00003652 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003653unlock:
Chris Wilson3ef94da2009-09-14 16:50:29 +01003654 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003655 return ret;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003656}
3657
Chris Wilson37e680a2012-06-07 15:38:42 +01003658void i915_gem_object_init(struct drm_i915_gem_object *obj,
3659 const struct drm_i915_gem_object_ops *ops)
Chris Wilson0327d6b2012-08-11 15:41:06 +01003660{
Chris Wilson0327d6b2012-08-11 15:41:06 +01003661 INIT_LIST_HEAD(&obj->mm_list);
3662 INIT_LIST_HEAD(&obj->gtt_list);
3663 INIT_LIST_HEAD(&obj->ring_list);
3664 INIT_LIST_HEAD(&obj->exec_list);
3665
Chris Wilson37e680a2012-06-07 15:38:42 +01003666 obj->ops = ops;
3667
Chris Wilson0327d6b2012-08-11 15:41:06 +01003668 obj->fence_reg = I915_FENCE_REG_NONE;
3669 obj->madv = I915_MADV_WILLNEED;
3670 /* Avoid an unnecessary call to unbind on the first bind. */
3671 obj->map_and_fenceable = true;
3672
3673 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
3674}
3675
Chris Wilson37e680a2012-06-07 15:38:42 +01003676static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
3677 .get_pages = i915_gem_object_get_pages_gtt,
3678 .put_pages = i915_gem_object_put_pages_gtt,
3679};
3680
Chris Wilson05394f32010-11-08 19:18:58 +00003681struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3682 size_t size)
Daniel Vetterac52bc52010-04-09 19:05:06 +00003683{
Daniel Vetterc397b902010-04-09 19:05:07 +00003684 struct drm_i915_gem_object *obj;
Hugh Dickins5949eac2011-06-27 16:18:18 -07003685 struct address_space *mapping;
Chris Wilsonbed1ea92012-05-24 20:48:12 +01003686 u32 mask;
Daniel Vetterc397b902010-04-09 19:05:07 +00003687
3688 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3689 if (obj == NULL)
3690 return NULL;
3691
3692 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3693 kfree(obj);
3694 return NULL;
3695 }
3696
Chris Wilsonbed1ea92012-05-24 20:48:12 +01003697 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
3698 if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
3699 /* 965gm cannot relocate objects above 4GiB. */
3700 mask &= ~__GFP_HIGHMEM;
3701 mask |= __GFP_DMA32;
3702 }
3703
Hugh Dickins5949eac2011-06-27 16:18:18 -07003704 mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
Chris Wilsonbed1ea92012-05-24 20:48:12 +01003705 mapping_set_gfp_mask(mapping, mask);
Hugh Dickins5949eac2011-06-27 16:18:18 -07003706
Chris Wilson37e680a2012-06-07 15:38:42 +01003707 i915_gem_object_init(obj, &i915_gem_object_ops);
Chris Wilson73aa8082010-09-30 11:46:12 +01003708
Daniel Vetterc397b902010-04-09 19:05:07 +00003709 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3710 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3711
Eugeni Dodonov3d29b842012-01-17 14:43:53 -02003712 if (HAS_LLC(dev)) {
3713 /* On some devices, we can have the GPU use the LLC (the CPU
Eric Anholta1871112011-03-29 16:59:55 -07003714 * cache) for about a 10% performance improvement
3715 * compared to uncached. Graphics requests other than
3716 * display scanout are coherent with the CPU in
3717 * accessing this cache. This means in this mode we
3718 * don't need to clflush on the CPU side, and on the
3719 * GPU side we only need to flush internal caches to
3720 * get data visible to the CPU.
3721 *
3722 * However, we maintain the display planes as UC, and so
3723 * need to rebind when first used as such.
3724 */
3725 obj->cache_level = I915_CACHE_LLC;
3726 } else
3727 obj->cache_level = I915_CACHE_NONE;
3728
Chris Wilson05394f32010-11-08 19:18:58 +00003729 return obj;
Daniel Vetterac52bc52010-04-09 19:05:06 +00003730}
3731
Eric Anholt673a3942008-07-30 12:06:12 -07003732int i915_gem_init_object(struct drm_gem_object *obj)
3733{
Daniel Vetterc397b902010-04-09 19:05:07 +00003734 BUG();
Jesse Barnesde151cf2008-11-12 10:03:55 -08003735
Eric Anholt673a3942008-07-30 12:06:12 -07003736 return 0;
3737}
3738
Chris Wilson1488fc02012-04-24 15:47:31 +01003739void i915_gem_free_object(struct drm_gem_object *gem_obj)
Chris Wilsonbe726152010-07-23 23:18:50 +01003740{
Chris Wilson1488fc02012-04-24 15:47:31 +01003741 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
Chris Wilson05394f32010-11-08 19:18:58 +00003742 struct drm_device *dev = obj->base.dev;
Chris Wilsonbe726152010-07-23 23:18:50 +01003743 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonbe726152010-07-23 23:18:50 +01003744
Chris Wilson26e12f82011-03-20 11:20:19 +00003745 trace_i915_gem_object_destroy(obj);
3746
Chris Wilson1488fc02012-04-24 15:47:31 +01003747 if (obj->phys_obj)
3748 i915_gem_detach_phys_object(dev, obj);
3749
3750 obj->pin_count = 0;
3751 if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
3752 bool was_interruptible;
3753
3754 was_interruptible = dev_priv->mm.interruptible;
3755 dev_priv->mm.interruptible = false;
3756
3757 WARN_ON(i915_gem_object_unbind(obj));
3758
3759 dev_priv->mm.interruptible = was_interruptible;
3760 }
3761
Chris Wilsona5570172012-09-04 21:02:54 +01003762 obj->pages_pin_count = 0;
Chris Wilson37e680a2012-06-07 15:38:42 +01003763 i915_gem_object_put_pages(obj);
Chris Wilsond8cb5082012-08-11 15:41:03 +01003764 i915_gem_object_free_mmap_offset(obj);
Chris Wilsonbe726152010-07-23 23:18:50 +01003765
Chris Wilson9da3da62012-06-01 15:20:22 +01003766 BUG_ON(obj->pages);
3767
Chris Wilson2f745ad2012-09-04 21:02:58 +01003768 if (obj->base.import_attach)
3769 drm_prime_gem_destroy(&obj->base, NULL);
3770
Chris Wilson05394f32010-11-08 19:18:58 +00003771 drm_gem_object_release(&obj->base);
3772 i915_gem_info_remove_obj(dev_priv, obj->base.size);
Chris Wilsonbe726152010-07-23 23:18:50 +01003773
Chris Wilson05394f32010-11-08 19:18:58 +00003774 kfree(obj->bit_17);
3775 kfree(obj);
Chris Wilsonbe726152010-07-23 23:18:50 +01003776}
3777
Jesse Barnes5669fca2009-02-17 15:13:31 -08003778int
Eric Anholt673a3942008-07-30 12:06:12 -07003779i915_gem_idle(struct drm_device *dev)
3780{
3781 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson29105cc2010-01-07 10:39:13 +00003782 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003783
Keith Packard6dbe2772008-10-14 21:41:13 -07003784 mutex_lock(&dev->struct_mutex);
3785
Chris Wilson87acb0a2010-10-19 10:13:00 +01003786 if (dev_priv->mm.suspended) {
Keith Packard6dbe2772008-10-14 21:41:13 -07003787 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07003788 return 0;
Keith Packard6dbe2772008-10-14 21:41:13 -07003789 }
Eric Anholt673a3942008-07-30 12:06:12 -07003790
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07003791 ret = i915_gpu_idle(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07003792 if (ret) {
3793 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07003794 return ret;
Keith Packard6dbe2772008-10-14 21:41:13 -07003795 }
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07003796 i915_gem_retire_requests(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07003797
Chris Wilson29105cc2010-01-07 10:39:13 +00003798 /* Under UMS, be paranoid and evict. */
Chris Wilsona39d7ef2012-04-24 18:22:52 +01003799 if (!drm_core_check_feature(dev, DRIVER_MODESET))
Chris Wilson6c085a72012-08-20 11:40:46 +02003800 i915_gem_evict_everything(dev);
Chris Wilson29105cc2010-01-07 10:39:13 +00003801
Chris Wilson312817a2010-11-22 11:50:11 +00003802 i915_gem_reset_fences(dev);
3803
Chris Wilson29105cc2010-01-07 10:39:13 +00003804 /* Hack! Don't let anybody do execbuf while we don't control the chip.
3805 * We need to replace this with a semaphore, or something.
3806 * And not confound mm.suspended!
3807 */
3808 dev_priv->mm.suspended = 1;
Daniel Vetterbc0c7f12010-08-20 18:18:48 +02003809 del_timer_sync(&dev_priv->hangcheck_timer);
Chris Wilson29105cc2010-01-07 10:39:13 +00003810
3811 i915_kernel_lost_context(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07003812 i915_gem_cleanup_ringbuffer(dev);
Chris Wilson29105cc2010-01-07 10:39:13 +00003813
Keith Packard6dbe2772008-10-14 21:41:13 -07003814 mutex_unlock(&dev->struct_mutex);
3815
Chris Wilson29105cc2010-01-07 10:39:13 +00003816 /* Cancel the retire work handler, which should be idle now. */
3817 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3818
Eric Anholt673a3942008-07-30 12:06:12 -07003819 return 0;
3820}
3821
Ben Widawskyb9524a12012-05-25 16:56:24 -07003822void i915_gem_l3_remap(struct drm_device *dev)
3823{
3824 drm_i915_private_t *dev_priv = dev->dev_private;
3825 u32 misccpctl;
3826 int i;
3827
3828 if (!IS_IVYBRIDGE(dev))
3829 return;
3830
3831 if (!dev_priv->mm.l3_remap_info)
3832 return;
3833
3834 misccpctl = I915_READ(GEN7_MISCCPCTL);
3835 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
3836 POSTING_READ(GEN7_MISCCPCTL);
3837
3838 for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
3839 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3840 if (remap && remap != dev_priv->mm.l3_remap_info[i/4])
3841 DRM_DEBUG("0x%x was already programmed to %x\n",
3842 GEN7_L3LOG_BASE + i, remap);
3843 if (remap && !dev_priv->mm.l3_remap_info[i/4])
3844 DRM_DEBUG_DRIVER("Clearing remapped register\n");
3845 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->mm.l3_remap_info[i/4]);
3846 }
3847
3848 /* Make sure all the writes land before disabling dop clock gating */
3849 POSTING_READ(GEN7_L3LOG_BASE);
3850
3851 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
3852}
3853
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003854void i915_gem_init_swizzling(struct drm_device *dev)
3855{
3856 drm_i915_private_t *dev_priv = dev->dev_private;
3857
Daniel Vetter11782b02012-01-31 16:47:55 +01003858 if (INTEL_INFO(dev)->gen < 5 ||
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003859 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
3860 return;
3861
3862 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
3863 DISP_TILE_SURFACE_SWIZZLING);
3864
Daniel Vetter11782b02012-01-31 16:47:55 +01003865 if (IS_GEN5(dev))
3866 return;
3867
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003868 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
3869 if (IS_GEN6(dev))
Daniel Vetter6b26c862012-04-24 14:04:12 +02003870 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003871 else
Daniel Vetter6b26c862012-04-24 14:04:12 +02003872 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003873}
Daniel Vettere21af882012-02-09 20:53:27 +01003874
3875void i915_gem_init_ppgtt(struct drm_device *dev)
3876{
3877 drm_i915_private_t *dev_priv = dev->dev_private;
3878 uint32_t pd_offset;
3879 struct intel_ring_buffer *ring;
Daniel Vetter55a254a2012-03-22 00:14:43 +01003880 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
3881 uint32_t __iomem *pd_addr;
3882 uint32_t pd_entry;
Daniel Vettere21af882012-02-09 20:53:27 +01003883 int i;
3884
3885 if (!dev_priv->mm.aliasing_ppgtt)
3886 return;
3887
Daniel Vetter55a254a2012-03-22 00:14:43 +01003888
3889 pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
3890 for (i = 0; i < ppgtt->num_pd_entries; i++) {
3891 dma_addr_t pt_addr;
3892
3893 if (dev_priv->mm.gtt->needs_dmar)
3894 pt_addr = ppgtt->pt_dma_addr[i];
3895 else
3896 pt_addr = page_to_phys(ppgtt->pt_pages[i]);
3897
3898 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
3899 pd_entry |= GEN6_PDE_VALID;
3900
3901 writel(pd_entry, pd_addr + i);
3902 }
3903 readl(pd_addr);
3904
3905 pd_offset = ppgtt->pd_offset;
Daniel Vettere21af882012-02-09 20:53:27 +01003906 pd_offset /= 64; /* in cachelines, */
3907 pd_offset <<= 16;
3908
3909 if (INTEL_INFO(dev)->gen == 6) {
Daniel Vetter48ecfa12012-04-11 20:42:40 +02003910 uint32_t ecochk, gab_ctl, ecobits;
3911
3912 ecobits = I915_READ(GAC_ECO_BITS);
3913 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
Daniel Vetterbe901a52012-04-11 20:42:39 +02003914
3915 gab_ctl = I915_READ(GAB_CTL);
3916 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
3917
3918 ecochk = I915_READ(GAM_ECOCHK);
Daniel Vettere21af882012-02-09 20:53:27 +01003919 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
3920 ECOCHK_PPGTT_CACHE64B);
Daniel Vetter6b26c862012-04-24 14:04:12 +02003921 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
Daniel Vettere21af882012-02-09 20:53:27 +01003922 } else if (INTEL_INFO(dev)->gen >= 7) {
3923 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
3924 /* GFX_MODE is per-ring on gen7+ */
3925 }
3926
Chris Wilsonb4519512012-05-11 14:29:30 +01003927 for_each_ring(ring, dev_priv, i) {
Daniel Vettere21af882012-02-09 20:53:27 +01003928 if (INTEL_INFO(dev)->gen >= 7)
3929 I915_WRITE(RING_MODE_GEN7(ring),
Daniel Vetter6b26c862012-04-24 14:04:12 +02003930 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
Daniel Vettere21af882012-02-09 20:53:27 +01003931
3932 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
3933 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
3934 }
3935}
3936
Chris Wilson67b1b572012-07-05 23:49:40 +01003937static bool
3938intel_enable_blt(struct drm_device *dev)
3939{
3940 if (!HAS_BLT(dev))
3941 return false;
3942
3943 /* The blitter was dysfunctional on early prototypes */
3944 if (IS_GEN6(dev) && dev->pdev->revision < 8) {
3945 DRM_INFO("BLT not supported on this pre-production hardware;"
3946 " graphics performance will be degraded.\n");
3947 return false;
3948 }
3949
3950 return true;
3951}
3952
Eric Anholt673a3942008-07-30 12:06:12 -07003953int
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003954i915_gem_init_hw(struct drm_device *dev)
Zou Nan hai8187a2b2010-05-21 09:08:55 +08003955{
3956 drm_i915_private_t *dev_priv = dev->dev_private;
3957 int ret;
Chris Wilson68f95ba2010-05-27 13:18:22 +01003958
Daniel Vetter8ecd1a62012-06-07 15:56:03 +02003959 if (!intel_enable_gtt())
3960 return -EIO;
3961
Ben Widawskyb9524a12012-05-25 16:56:24 -07003962 i915_gem_l3_remap(dev);
3963
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003964 i915_gem_init_swizzling(dev);
3965
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08003966 ret = intel_init_render_ring_buffer(dev);
Chris Wilson68f95ba2010-05-27 13:18:22 +01003967 if (ret)
Chris Wilsonb6913e42010-11-12 10:46:37 +00003968 return ret;
Chris Wilson68f95ba2010-05-27 13:18:22 +01003969
3970 if (HAS_BSD(dev)) {
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08003971 ret = intel_init_bsd_ring_buffer(dev);
Chris Wilson68f95ba2010-05-27 13:18:22 +01003972 if (ret)
3973 goto cleanup_render_ring;
Zou Nan haid1b851f2010-05-21 09:08:57 +08003974 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01003975
Chris Wilson67b1b572012-07-05 23:49:40 +01003976 if (intel_enable_blt(dev)) {
Chris Wilson549f7362010-10-19 11:19:32 +01003977 ret = intel_init_blt_ring_buffer(dev);
3978 if (ret)
3979 goto cleanup_bsd_ring;
3980 }
3981
Chris Wilson6f392d52010-08-07 11:01:22 +01003982 dev_priv->next_seqno = 1;
3983
Ben Widawsky254f9652012-06-04 14:42:42 -07003984 /*
3985 * XXX: There was some w/a described somewhere suggesting loading
3986 * contexts before PPGTT.
3987 */
3988 i915_gem_context_init(dev);
Daniel Vettere21af882012-02-09 20:53:27 +01003989 i915_gem_init_ppgtt(dev);
3990
Chris Wilson68f95ba2010-05-27 13:18:22 +01003991 return 0;
3992
Chris Wilson549f7362010-10-19 11:19:32 +01003993cleanup_bsd_ring:
Chris Wilson1ec14ad2010-12-04 11:30:53 +00003994 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
Chris Wilson68f95ba2010-05-27 13:18:22 +01003995cleanup_render_ring:
Chris Wilson1ec14ad2010-12-04 11:30:53 +00003996 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08003997 return ret;
3998}
3999
Chris Wilson1070a422012-04-24 15:47:41 +01004000static bool
4001intel_enable_ppgtt(struct drm_device *dev)
4002{
4003 if (i915_enable_ppgtt >= 0)
4004 return i915_enable_ppgtt;
4005
4006#ifdef CONFIG_INTEL_IOMMU
4007 /* Disable ppgtt on SNB if VT-d is on. */
4008 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
4009 return false;
4010#endif
4011
4012 return true;
4013}
4014
4015int i915_gem_init(struct drm_device *dev)
4016{
4017 struct drm_i915_private *dev_priv = dev->dev_private;
4018 unsigned long gtt_size, mappable_size;
4019 int ret;
4020
4021 gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
4022 mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
4023
4024 mutex_lock(&dev->struct_mutex);
4025 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
4026 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
4027 * aperture accordingly when using aliasing ppgtt. */
4028 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
4029
4030 i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size);
4031
4032 ret = i915_gem_init_aliasing_ppgtt(dev);
4033 if (ret) {
4034 mutex_unlock(&dev->struct_mutex);
4035 return ret;
4036 }
4037 } else {
4038 /* Let GEM Manage all of the aperture.
4039 *
4040 * However, leave one page at the end still bound to the scratch
4041 * page. There are a number of places where the hardware
4042 * apparently prefetches past the end of the object, and we've
4043 * seen multiple hangs with the GPU head pointer stuck in a
4044 * batchbuffer bound at the last page of the aperture. One page
4045 * should be enough to keep any prefetching inside of the
4046 * aperture.
4047 */
4048 i915_gem_init_global_gtt(dev, 0, mappable_size,
4049 gtt_size);
4050 }
4051
4052 ret = i915_gem_init_hw(dev);
4053 mutex_unlock(&dev->struct_mutex);
4054 if (ret) {
4055 i915_gem_cleanup_aliasing_ppgtt(dev);
4056 return ret;
4057 }
4058
Daniel Vetter53ca26c2012-04-26 23:28:03 +02004059 /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
4060 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4061 dev_priv->dri1.allow_batchbuffer = 1;
Chris Wilson1070a422012-04-24 15:47:41 +01004062 return 0;
4063}
4064
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004065void
4066i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4067{
4068 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonb4519512012-05-11 14:29:30 +01004069 struct intel_ring_buffer *ring;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00004070 int i;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004071
Chris Wilsonb4519512012-05-11 14:29:30 +01004072 for_each_ring(ring, dev_priv, i)
4073 intel_cleanup_ring_buffer(ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004074}
4075
4076int
Eric Anholt673a3942008-07-30 12:06:12 -07004077i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4078 struct drm_file *file_priv)
4079{
4080 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonb4519512012-05-11 14:29:30 +01004081 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004082
Jesse Barnes79e53942008-11-07 14:24:08 -08004083 if (drm_core_check_feature(dev, DRIVER_MODESET))
4084 return 0;
4085
Ben Gamariba1234d2009-09-14 17:48:47 -04004086 if (atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07004087 DRM_ERROR("Reenabling wedged hardware, good luck\n");
Ben Gamariba1234d2009-09-14 17:48:47 -04004088 atomic_set(&dev_priv->mm.wedged, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07004089 }
4090
Eric Anholt673a3942008-07-30 12:06:12 -07004091 mutex_lock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004092 dev_priv->mm.suspended = 0;
4093
Daniel Vetterf691e2f2012-02-02 09:58:12 +01004094 ret = i915_gem_init_hw(dev);
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004095 if (ret != 0) {
4096 mutex_unlock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004097 return ret;
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004098 }
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004099
Chris Wilson69dc4982010-10-19 10:36:51 +01004100 BUG_ON(!list_empty(&dev_priv->mm.active_list));
Eric Anholt673a3942008-07-30 12:06:12 -07004101 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
Eric Anholt673a3942008-07-30 12:06:12 -07004102 mutex_unlock(&dev->struct_mutex);
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004103
Chris Wilson5f353082010-06-07 14:03:03 +01004104 ret = drm_irq_install(dev);
4105 if (ret)
4106 goto cleanup_ringbuffer;
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004107
Eric Anholt673a3942008-07-30 12:06:12 -07004108 return 0;
Chris Wilson5f353082010-06-07 14:03:03 +01004109
4110cleanup_ringbuffer:
4111 mutex_lock(&dev->struct_mutex);
4112 i915_gem_cleanup_ringbuffer(dev);
4113 dev_priv->mm.suspended = 1;
4114 mutex_unlock(&dev->struct_mutex);
4115
4116 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004117}
4118
4119int
4120i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4121 struct drm_file *file_priv)
4122{
Jesse Barnes79e53942008-11-07 14:24:08 -08004123 if (drm_core_check_feature(dev, DRIVER_MODESET))
4124 return 0;
4125
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004126 drm_irq_uninstall(dev);
Linus Torvaldse6890f62009-09-08 17:09:24 -07004127 return i915_gem_idle(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004128}
4129
4130void
4131i915_gem_lastclose(struct drm_device *dev)
4132{
4133 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004134
Eric Anholte806b492009-01-22 09:56:58 -08004135 if (drm_core_check_feature(dev, DRIVER_MODESET))
4136 return;
4137
Keith Packard6dbe2772008-10-14 21:41:13 -07004138 ret = i915_gem_idle(dev);
4139 if (ret)
4140 DRM_ERROR("failed to idle hardware: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07004141}
4142
Chris Wilson64193402010-10-24 12:38:05 +01004143static void
4144init_ring_lists(struct intel_ring_buffer *ring)
4145{
4146 INIT_LIST_HEAD(&ring->active_list);
4147 INIT_LIST_HEAD(&ring->request_list);
Chris Wilson64193402010-10-24 12:38:05 +01004148}
4149
Eric Anholt673a3942008-07-30 12:06:12 -07004150void
4151i915_gem_load(struct drm_device *dev)
4152{
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004153 int i;
Eric Anholt673a3942008-07-30 12:06:12 -07004154 drm_i915_private_t *dev_priv = dev->dev_private;
4155
Chris Wilson69dc4982010-10-19 10:36:51 +01004156 INIT_LIST_HEAD(&dev_priv->mm.active_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004157 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
Chris Wilson6c085a72012-08-20 11:40:46 +02004158 INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4159 INIT_LIST_HEAD(&dev_priv->mm.bound_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07004160 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00004161 for (i = 0; i < I915_NUM_RINGS; i++)
4162 init_ring_lists(&dev_priv->ring[i]);
Daniel Vetter4b9de732011-10-09 21:52:02 +02004163 for (i = 0; i < I915_MAX_NUM_FENCES; i++)
Daniel Vetter007cc8a2010-04-28 11:02:31 +02004164 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004165 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4166 i915_gem_retire_work_handler);
Chris Wilson30dbf0c2010-09-25 10:19:17 +01004167 init_completion(&dev_priv->error_completion);
Chris Wilson31169712009-09-14 16:50:28 +01004168
Dave Airlie94400122010-07-20 13:15:31 +10004169 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4170 if (IS_GEN3(dev)) {
Daniel Vetter50743292012-04-26 22:02:54 +02004171 I915_WRITE(MI_ARB_STATE,
4172 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
Dave Airlie94400122010-07-20 13:15:31 +10004173 }
4174
Chris Wilson72bfa192010-12-19 11:42:05 +00004175 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4176
Jesse Barnesde151cf2008-11-12 10:03:55 -08004177 /* Old X drivers will take 0-2 for front, back, depth buffers */
Eric Anholtb397c832010-01-26 09:43:10 -08004178 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4179 dev_priv->fence_reg_start = 3;
Jesse Barnesde151cf2008-11-12 10:03:55 -08004180
Chris Wilsona6c45cf2010-09-17 00:32:17 +01004181 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
Jesse Barnesde151cf2008-11-12 10:03:55 -08004182 dev_priv->num_fence_regs = 16;
4183 else
4184 dev_priv->num_fence_regs = 8;
4185
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004186 /* Initialize fence registers to zero */
Chris Wilsonada726c2012-04-17 15:31:32 +01004187 i915_gem_reset_fences(dev);
Eric Anholt10ed13e2011-05-06 13:53:49 -07004188
Eric Anholt673a3942008-07-30 12:06:12 -07004189 i915_gem_detect_bit_6_swizzle(dev);
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05004190 init_waitqueue_head(&dev_priv->pending_flip_queue);
Chris Wilson17250b72010-10-28 12:51:39 +01004191
Chris Wilsonce453d82011-02-21 14:43:56 +00004192 dev_priv->mm.interruptible = true;
4193
Chris Wilson17250b72010-10-28 12:51:39 +01004194 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4195 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4196 register_shrinker(&dev_priv->mm.inactive_shrinker);
Eric Anholt673a3942008-07-30 12:06:12 -07004197}
Dave Airlie71acb5e2008-12-30 20:31:46 +10004198
4199/*
4200 * Create a physically contiguous memory object for this object
4201 * e.g. for cursor + overlay regs
4202 */
Chris Wilson995b6762010-08-20 13:23:26 +01004203static int i915_gem_init_phys_object(struct drm_device *dev,
4204 int id, int size, int align)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004205{
4206 drm_i915_private_t *dev_priv = dev->dev_private;
4207 struct drm_i915_gem_phys_object *phys_obj;
4208 int ret;
4209
4210 if (dev_priv->mm.phys_objs[id - 1] || !size)
4211 return 0;
4212
Eric Anholt9a298b22009-03-24 12:23:04 -07004213 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004214 if (!phys_obj)
4215 return -ENOMEM;
4216
4217 phys_obj->id = id;
4218
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004219 phys_obj->handle = drm_pci_alloc(dev, size, align);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004220 if (!phys_obj->handle) {
4221 ret = -ENOMEM;
4222 goto kfree_obj;
4223 }
4224#ifdef CONFIG_X86
4225 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4226#endif
4227
4228 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4229
4230 return 0;
4231kfree_obj:
Eric Anholt9a298b22009-03-24 12:23:04 -07004232 kfree(phys_obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004233 return ret;
4234}
4235
Chris Wilson995b6762010-08-20 13:23:26 +01004236static void i915_gem_free_phys_object(struct drm_device *dev, int id)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004237{
4238 drm_i915_private_t *dev_priv = dev->dev_private;
4239 struct drm_i915_gem_phys_object *phys_obj;
4240
4241 if (!dev_priv->mm.phys_objs[id - 1])
4242 return;
4243
4244 phys_obj = dev_priv->mm.phys_objs[id - 1];
4245 if (phys_obj->cur_obj) {
4246 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4247 }
4248
4249#ifdef CONFIG_X86
4250 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4251#endif
4252 drm_pci_free(dev, phys_obj->handle);
4253 kfree(phys_obj);
4254 dev_priv->mm.phys_objs[id - 1] = NULL;
4255}
4256
4257void i915_gem_free_all_phys_object(struct drm_device *dev)
4258{
4259 int i;
4260
Dave Airlie260883c2009-01-22 17:58:49 +10004261 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004262 i915_gem_free_phys_object(dev, i);
4263}
4264
4265void i915_gem_detach_phys_object(struct drm_device *dev,
Chris Wilson05394f32010-11-08 19:18:58 +00004266 struct drm_i915_gem_object *obj)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004267{
Chris Wilson05394f32010-11-08 19:18:58 +00004268 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
Chris Wilsone5281cc2010-10-28 13:45:36 +01004269 char *vaddr;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004270 int i;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004271 int page_count;
4272
Chris Wilson05394f32010-11-08 19:18:58 +00004273 if (!obj->phys_obj)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004274 return;
Chris Wilson05394f32010-11-08 19:18:58 +00004275 vaddr = obj->phys_obj->handle->vaddr;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004276
Chris Wilson05394f32010-11-08 19:18:58 +00004277 page_count = obj->base.size / PAGE_SIZE;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004278 for (i = 0; i < page_count; i++) {
Hugh Dickins5949eac2011-06-27 16:18:18 -07004279 struct page *page = shmem_read_mapping_page(mapping, i);
Chris Wilsone5281cc2010-10-28 13:45:36 +01004280 if (!IS_ERR(page)) {
4281 char *dst = kmap_atomic(page);
4282 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4283 kunmap_atomic(dst);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004284
Chris Wilsone5281cc2010-10-28 13:45:36 +01004285 drm_clflush_pages(&page, 1);
4286
4287 set_page_dirty(page);
4288 mark_page_accessed(page);
4289 page_cache_release(page);
4290 }
Dave Airlie71acb5e2008-12-30 20:31:46 +10004291 }
Daniel Vetter40ce6572010-11-05 18:12:18 +01004292 intel_gtt_chipset_flush();
Chris Wilsond78b47b2009-06-17 21:52:49 +01004293
Chris Wilson05394f32010-11-08 19:18:58 +00004294 obj->phys_obj->cur_obj = NULL;
4295 obj->phys_obj = NULL;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004296}
4297
4298int
4299i915_gem_attach_phys_object(struct drm_device *dev,
Chris Wilson05394f32010-11-08 19:18:58 +00004300 struct drm_i915_gem_object *obj,
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004301 int id,
4302 int align)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004303{
Chris Wilson05394f32010-11-08 19:18:58 +00004304 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004305 drm_i915_private_t *dev_priv = dev->dev_private;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004306 int ret = 0;
4307 int page_count;
4308 int i;
4309
4310 if (id > I915_MAX_PHYS_OBJECT)
4311 return -EINVAL;
4312
Chris Wilson05394f32010-11-08 19:18:58 +00004313 if (obj->phys_obj) {
4314 if (obj->phys_obj->id == id)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004315 return 0;
4316 i915_gem_detach_phys_object(dev, obj);
4317 }
4318
Dave Airlie71acb5e2008-12-30 20:31:46 +10004319 /* create a new object */
4320 if (!dev_priv->mm.phys_objs[id - 1]) {
4321 ret = i915_gem_init_phys_object(dev, id,
Chris Wilson05394f32010-11-08 19:18:58 +00004322 obj->base.size, align);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004323 if (ret) {
Chris Wilson05394f32010-11-08 19:18:58 +00004324 DRM_ERROR("failed to init phys object %d size: %zu\n",
4325 id, obj->base.size);
Chris Wilsone5281cc2010-10-28 13:45:36 +01004326 return ret;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004327 }
4328 }
4329
4330 /* bind to the object */
Chris Wilson05394f32010-11-08 19:18:58 +00004331 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4332 obj->phys_obj->cur_obj = obj;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004333
Chris Wilson05394f32010-11-08 19:18:58 +00004334 page_count = obj->base.size / PAGE_SIZE;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004335
4336 for (i = 0; i < page_count; i++) {
Chris Wilsone5281cc2010-10-28 13:45:36 +01004337 struct page *page;
4338 char *dst, *src;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004339
Hugh Dickins5949eac2011-06-27 16:18:18 -07004340 page = shmem_read_mapping_page(mapping, i);
Chris Wilsone5281cc2010-10-28 13:45:36 +01004341 if (IS_ERR(page))
4342 return PTR_ERR(page);
4343
Chris Wilsonff75b9b2010-10-30 22:52:31 +01004344 src = kmap_atomic(page);
Chris Wilson05394f32010-11-08 19:18:58 +00004345 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004346 memcpy(dst, src, PAGE_SIZE);
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -07004347 kunmap_atomic(src);
Chris Wilsone5281cc2010-10-28 13:45:36 +01004348
4349 mark_page_accessed(page);
4350 page_cache_release(page);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004351 }
4352
4353 return 0;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004354}
4355
4356static int
Chris Wilson05394f32010-11-08 19:18:58 +00004357i915_gem_phys_pwrite(struct drm_device *dev,
4358 struct drm_i915_gem_object *obj,
Dave Airlie71acb5e2008-12-30 20:31:46 +10004359 struct drm_i915_gem_pwrite *args,
4360 struct drm_file *file_priv)
4361{
Chris Wilson05394f32010-11-08 19:18:58 +00004362 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
Chris Wilsonb47b30c2010-11-08 01:12:29 +00004363 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004364
Chris Wilsonb47b30c2010-11-08 01:12:29 +00004365 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4366 unsigned long unwritten;
4367
4368 /* The physical object once assigned is fixed for the lifetime
4369 * of the obj, so we can safely drop the lock and continue
4370 * to access vaddr.
4371 */
4372 mutex_unlock(&dev->struct_mutex);
4373 unwritten = copy_from_user(vaddr, user_data, args->size);
4374 mutex_lock(&dev->struct_mutex);
4375 if (unwritten)
4376 return -EFAULT;
4377 }
Dave Airlie71acb5e2008-12-30 20:31:46 +10004378
Daniel Vetter40ce6572010-11-05 18:12:18 +01004379 intel_gtt_chipset_flush();
Dave Airlie71acb5e2008-12-30 20:31:46 +10004380 return 0;
4381}
Eric Anholtb9624422009-06-03 07:27:35 +00004382
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004383void i915_gem_release(struct drm_device *dev, struct drm_file *file)
Eric Anholtb9624422009-06-03 07:27:35 +00004384{
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004385 struct drm_i915_file_private *file_priv = file->driver_priv;
Eric Anholtb9624422009-06-03 07:27:35 +00004386
4387 /* Clean up our request list when the client is going away, so that
4388 * later retire_requests won't dereference our soon-to-be-gone
4389 * file_priv.
4390 */
Chris Wilson1c255952010-09-26 11:03:27 +01004391 spin_lock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004392 while (!list_empty(&file_priv->mm.request_list)) {
4393 struct drm_i915_gem_request *request;
4394
4395 request = list_first_entry(&file_priv->mm.request_list,
4396 struct drm_i915_gem_request,
4397 client_list);
4398 list_del(&request->client_list);
4399 request->file_priv = NULL;
4400 }
Chris Wilson1c255952010-09-26 11:03:27 +01004401 spin_unlock(&file_priv->mm.lock);
Eric Anholtb9624422009-06-03 07:27:35 +00004402}
Chris Wilson31169712009-09-14 16:50:28 +01004403
Chris Wilson31169712009-09-14 16:50:28 +01004404static int
Ying Han1495f232011-05-24 17:12:27 -07004405i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
Chris Wilson31169712009-09-14 16:50:28 +01004406{
Chris Wilson17250b72010-10-28 12:51:39 +01004407 struct drm_i915_private *dev_priv =
4408 container_of(shrinker,
4409 struct drm_i915_private,
4410 mm.inactive_shrinker);
4411 struct drm_device *dev = dev_priv->dev;
Chris Wilson6c085a72012-08-20 11:40:46 +02004412 struct drm_i915_gem_object *obj;
Ying Han1495f232011-05-24 17:12:27 -07004413 int nr_to_scan = sc->nr_to_scan;
Chris Wilson17250b72010-10-28 12:51:39 +01004414 int cnt;
4415
4416 if (!mutex_trylock(&dev->struct_mutex))
Chris Wilsonbbe2e112010-10-28 22:35:07 +01004417 return 0;
Chris Wilson31169712009-09-14 16:50:28 +01004418
Chris Wilson6c085a72012-08-20 11:40:46 +02004419 if (nr_to_scan) {
4420 nr_to_scan -= i915_gem_purge(dev_priv, nr_to_scan);
4421 if (nr_to_scan > 0)
4422 i915_gem_shrink_all(dev_priv);
Chris Wilson31169712009-09-14 16:50:28 +01004423 }
4424
Chris Wilson17250b72010-10-28 12:51:39 +01004425 cnt = 0;
Chris Wilson6c085a72012-08-20 11:40:46 +02004426 list_for_each_entry(obj, &dev_priv->mm.unbound_list, gtt_list)
Chris Wilsona5570172012-09-04 21:02:54 +01004427 if (obj->pages_pin_count == 0)
4428 cnt += obj->base.size >> PAGE_SHIFT;
Chris Wilson6c085a72012-08-20 11:40:46 +02004429 list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
Chris Wilsona5570172012-09-04 21:02:54 +01004430 if (obj->pin_count == 0 && obj->pages_pin_count == 0)
Chris Wilson6c085a72012-08-20 11:40:46 +02004431 cnt += obj->base.size >> PAGE_SHIFT;
Chris Wilson31169712009-09-14 16:50:28 +01004432
Chris Wilson17250b72010-10-28 12:51:39 +01004433 mutex_unlock(&dev->struct_mutex);
Chris Wilson6c085a72012-08-20 11:40:46 +02004434 return cnt;
Chris Wilson31169712009-09-14 16:50:28 +01004435}