blob: 1f5c67c579cf2ac4e0e9df9f100df83d08ce1a39 [file] [log] [blame]
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */
30
Joe Perches25d04792012-03-16 21:43:50 -070031#define pr_fmt(fmt) "[TTM] " fmt
32
Thomas Hellstromba4e7d92009-06-10 15:20:19 +020033#include "ttm/ttm_module.h"
34#include "ttm/ttm_bo_driver.h"
35#include "ttm/ttm_placement.h"
36#include <linux/jiffies.h>
37#include <linux/slab.h>
38#include <linux/sched.h>
39#include <linux/mm.h>
40#include <linux/file.h>
41#include <linux/module.h>
Arun Sharma600634972011-07-26 16:09:06 -070042#include <linux/atomic.h>
Thomas Hellstromba4e7d92009-06-10 15:20:19 +020043
44#define TTM_ASSERT_LOCKED(param)
45#define TTM_DEBUG(fmt, arg...)
46#define TTM_BO_HASH_ORDER 13
47
48static int ttm_bo_setup_vm(struct ttm_buffer_object *bo);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +020049static int ttm_bo_swapout(struct ttm_mem_shrink *shrink);
Thomas Hellstroma987fca2009-08-18 16:51:56 +020050static void ttm_bo_global_kobj_release(struct kobject *kobj);
51
52static struct attribute ttm_bo_count = {
53 .name = "bo_count",
54 .mode = S_IRUGO
55};
56
Jerome Glissefb53f862009-12-09 21:55:10 +010057static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type)
58{
59 int i;
60
61 for (i = 0; i <= TTM_PL_PRIV5; i++)
62 if (flags & (1 << i)) {
63 *mem_type = i;
64 return 0;
65 }
66 return -EINVAL;
67}
68
Jerome Glisse5012f502009-12-10 18:07:26 +010069static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
Jerome Glissefb53f862009-12-09 21:55:10 +010070{
Jerome Glisse5012f502009-12-10 18:07:26 +010071 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
72
Joe Perches25d04792012-03-16 21:43:50 -070073 pr_err(" has_type: %d\n", man->has_type);
74 pr_err(" use_type: %d\n", man->use_type);
75 pr_err(" flags: 0x%08X\n", man->flags);
76 pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset);
77 pr_err(" size: %llu\n", man->size);
78 pr_err(" available_caching: 0x%08X\n", man->available_caching);
79 pr_err(" default_caching: 0x%08X\n", man->default_caching);
Ben Skeggsd961db72010-08-05 10:48:18 +100080 if (mem_type != TTM_PL_SYSTEM)
81 (*man->func->debug)(man, TTM_PFX);
Jerome Glissefb53f862009-12-09 21:55:10 +010082}
83
84static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
85 struct ttm_placement *placement)
86{
Jerome Glissefb53f862009-12-09 21:55:10 +010087 int i, ret, mem_type;
88
Joe Perches25d04792012-03-16 21:43:50 -070089 pr_err("No space for %p (%lu pages, %luK, %luM)\n",
90 bo, bo->mem.num_pages, bo->mem.size >> 10,
91 bo->mem.size >> 20);
Jerome Glissefb53f862009-12-09 21:55:10 +010092 for (i = 0; i < placement->num_placement; i++) {
93 ret = ttm_mem_type_from_flags(placement->placement[i],
94 &mem_type);
95 if (ret)
96 return;
Joe Perches25d04792012-03-16 21:43:50 -070097 pr_err(" placement[%d]=0x%08X (%d)\n",
98 i, placement->placement[i], mem_type);
Jerome Glisse5012f502009-12-10 18:07:26 +010099 ttm_mem_type_debug(bo->bdev, mem_type);
Jerome Glissefb53f862009-12-09 21:55:10 +0100100 }
101}
102
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200103static ssize_t ttm_bo_global_show(struct kobject *kobj,
104 struct attribute *attr,
105 char *buffer)
106{
107 struct ttm_bo_global *glob =
108 container_of(kobj, struct ttm_bo_global, kobj);
109
110 return snprintf(buffer, PAGE_SIZE, "%lu\n",
111 (unsigned long) atomic_read(&glob->bo_count));
112}
113
114static struct attribute *ttm_bo_global_attrs[] = {
115 &ttm_bo_count,
116 NULL
117};
118
Emese Revfy52cf25d2010-01-19 02:58:23 +0100119static const struct sysfs_ops ttm_bo_global_ops = {
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200120 .show = &ttm_bo_global_show
121};
122
123static struct kobj_type ttm_bo_glob_kobj_type = {
124 .release = &ttm_bo_global_kobj_release,
125 .sysfs_ops = &ttm_bo_global_ops,
126 .default_attrs = ttm_bo_global_attrs
127};
128
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200129
130static inline uint32_t ttm_bo_type_flags(unsigned type)
131{
132 return 1 << (type);
133}
134
135static void ttm_bo_release_list(struct kref *list_kref)
136{
137 struct ttm_buffer_object *bo =
138 container_of(list_kref, struct ttm_buffer_object, list_kref);
139 struct ttm_bo_device *bdev = bo->bdev;
Jerome Glisse57de4ba2011-11-11 15:42:57 -0500140 size_t acc_size = bo->acc_size;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200141
142 BUG_ON(atomic_read(&bo->list_kref.refcount));
143 BUG_ON(atomic_read(&bo->kref.refcount));
144 BUG_ON(atomic_read(&bo->cpu_writers));
145 BUG_ON(bo->sync_obj != NULL);
146 BUG_ON(bo->mem.mm_node != NULL);
147 BUG_ON(!list_empty(&bo->lru));
148 BUG_ON(!list_empty(&bo->ddestroy));
149
150 if (bo->ttm)
151 ttm_tt_destroy(bo->ttm);
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200152 atomic_dec(&bo->glob->bo_count);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200153 if (bo->destroy)
154 bo->destroy(bo);
155 else {
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200156 kfree(bo);
157 }
Jerome Glisse57de4ba2011-11-11 15:42:57 -0500158 ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200159}
160
161int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
162{
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200163 if (interruptible) {
Jean Delvare965d3802010-10-09 12:36:45 +0000164 return wait_event_interruptible(bo->event_queue,
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200165 atomic_read(&bo->reserved) == 0);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200166 } else {
167 wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0);
Jean Delvare965d3802010-10-09 12:36:45 +0000168 return 0;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200169 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200170}
Ben Skeggsd1ede142009-12-11 15:13:00 +1000171EXPORT_SYMBOL(ttm_bo_wait_unreserved);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200172
Dave Airlied6ea8882010-11-22 13:24:40 +1000173void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200174{
175 struct ttm_bo_device *bdev = bo->bdev;
176 struct ttm_mem_type_manager *man;
177
178 BUG_ON(!atomic_read(&bo->reserved));
179
180 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
181
182 BUG_ON(!list_empty(&bo->lru));
183
184 man = &bdev->man[bo->mem.mem_type];
185 list_add_tail(&bo->lru, &man->lru);
186 kref_get(&bo->list_kref);
187
188 if (bo->ttm != NULL) {
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200189 list_add_tail(&bo->swap, &bo->glob->swap_lru);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200190 kref_get(&bo->list_kref);
191 }
192 }
193}
194
Dave Airlied6ea8882010-11-22 13:24:40 +1000195int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200196{
197 int put_count = 0;
198
199 if (!list_empty(&bo->swap)) {
200 list_del_init(&bo->swap);
201 ++put_count;
202 }
203 if (!list_empty(&bo->lru)) {
204 list_del_init(&bo->lru);
205 ++put_count;
206 }
207
208 /*
209 * TODO: Add a driver hook to delete from
210 * driver-specific LRU's here.
211 */
212
213 return put_count;
214}
215
216int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
217 bool interruptible,
218 bool no_wait, bool use_sequence, uint32_t sequence)
219{
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200220 struct ttm_bo_global *glob = bo->glob;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200221 int ret;
222
223 while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) {
Thomas Hellstrom95ccb0f2010-11-11 10:04:53 +0100224 /**
225 * Deadlock avoidance for multi-bo reserving.
226 */
Thomas Hellstrom96726fe2010-11-17 12:28:28 +0000227 if (use_sequence && bo->seq_valid) {
228 /**
229 * We've already reserved this one.
230 */
231 if (unlikely(sequence == bo->val_seq))
232 return -EDEADLK;
233 /**
234 * Already reserved by a thread that will not back
235 * off for us. We need to back off.
236 */
237 if (unlikely(sequence - bo->val_seq < (1 << 31)))
238 return -EAGAIN;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200239 }
240
241 if (no_wait)
242 return -EBUSY;
243
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200244 spin_unlock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200245 ret = ttm_bo_wait_unreserved(bo, interruptible);
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200246 spin_lock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200247
248 if (unlikely(ret))
249 return ret;
250 }
251
252 if (use_sequence) {
Thomas Hellstrom95ccb0f2010-11-11 10:04:53 +0100253 /**
254 * Wake up waiters that may need to recheck for deadlock,
255 * if we decreased the sequence number.
256 */
257 if (unlikely((bo->val_seq - sequence < (1 << 31))
258 || !bo->seq_valid))
259 wake_up_all(&bo->event_queue);
260
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200261 bo->val_seq = sequence;
262 bo->seq_valid = true;
263 } else {
264 bo->seq_valid = false;
265 }
266
267 return 0;
268}
269EXPORT_SYMBOL(ttm_bo_reserve);
270
271static void ttm_bo_ref_bug(struct kref *list_kref)
272{
273 BUG();
274}
275
Dave Airlied6ea8882010-11-22 13:24:40 +1000276void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
277 bool never_free)
278{
Thomas Hellstrom2357cbe2010-11-16 15:21:08 +0100279 kref_sub(&bo->list_kref, count,
280 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list);
Dave Airlied6ea8882010-11-22 13:24:40 +1000281}
282
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200283int ttm_bo_reserve(struct ttm_buffer_object *bo,
284 bool interruptible,
285 bool no_wait, bool use_sequence, uint32_t sequence)
286{
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200287 struct ttm_bo_global *glob = bo->glob;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200288 int put_count = 0;
289 int ret;
290
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200291 spin_lock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200292 ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence,
293 sequence);
294 if (likely(ret == 0))
295 put_count = ttm_bo_del_from_lru(bo);
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200296 spin_unlock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200297
Dave Airlied6ea8882010-11-22 13:24:40 +1000298 ttm_bo_list_ref_sub(bo, put_count, true);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200299
300 return ret;
301}
302
Thomas Hellstrom95762c22010-11-17 12:28:30 +0000303void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
304{
305 ttm_bo_add_to_lru(bo);
306 atomic_set(&bo->reserved, 0);
307 wake_up_all(&bo->event_queue);
308}
309
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200310void ttm_bo_unreserve(struct ttm_buffer_object *bo)
311{
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200312 struct ttm_bo_global *glob = bo->glob;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200313
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200314 spin_lock(&glob->lru_lock);
Thomas Hellstrom95762c22010-11-17 12:28:30 +0000315 ttm_bo_unreserve_locked(bo);
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200316 spin_unlock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200317}
318EXPORT_SYMBOL(ttm_bo_unreserve);
319
320/*
321 * Call bo->mutex locked.
322 */
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200323static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
324{
325 struct ttm_bo_device *bdev = bo->bdev;
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200326 struct ttm_bo_global *glob = bo->glob;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200327 int ret = 0;
328 uint32_t page_flags = 0;
329
330 TTM_ASSERT_LOCKED(&bo->mutex);
331 bo->ttm = NULL;
332
Dave Airliead49f502009-07-10 22:36:26 +1000333 if (bdev->need_dma32)
334 page_flags |= TTM_PAGE_FLAG_DMA32;
335
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200336 switch (bo->type) {
337 case ttm_bo_type_device:
338 if (zero_alloc)
339 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC;
340 case ttm_bo_type_kernel:
Jerome Glisse649bf3c2011-11-01 20:46:13 -0400341 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
342 page_flags, glob->dummy_read_page);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200343 if (unlikely(bo->ttm == NULL))
344 ret = -ENOMEM;
345 break;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200346 default:
Joe Perches25d04792012-03-16 21:43:50 -0700347 pr_err("Illegal buffer object type\n");
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200348 ret = -EINVAL;
349 break;
350 }
351
352 return ret;
353}
354
355static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
356 struct ttm_mem_reg *mem,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000357 bool evict, bool interruptible,
358 bool no_wait_reserve, bool no_wait_gpu)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200359{
360 struct ttm_bo_device *bdev = bo->bdev;
361 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
362 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
363 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
364 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
365 int ret = 0;
366
367 if (old_is_pci || new_is_pci ||
Thomas Hellstromeba67092010-11-11 09:41:57 +0100368 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
369 ret = ttm_mem_io_lock(old_man, true);
370 if (unlikely(ret != 0))
371 goto out_err;
372 ttm_bo_unmap_virtual_locked(bo);
373 ttm_mem_io_unlock(old_man);
374 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200375
376 /*
377 * Create and bind a ttm if required.
378 */
379
Ben Skeggs8d3bb232011-08-22 03:15:05 +0000380 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
381 if (bo->ttm == NULL) {
Ben Skeggsff02b132011-09-14 06:08:06 +1000382 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
383 ret = ttm_bo_add_ttm(bo, zero);
Ben Skeggs8d3bb232011-08-22 03:15:05 +0000384 if (ret)
385 goto out_err;
386 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200387
388 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
389 if (ret)
Thomas Hellstrom87ef9202009-06-17 12:29:57 +0200390 goto out_err;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200391
392 if (mem->mem_type != TTM_PL_SYSTEM) {
393 ret = ttm_tt_bind(bo->ttm, mem);
394 if (ret)
395 goto out_err;
396 }
397
398 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
Ben Skeggs82ef5942011-02-02 00:27:10 +0000399 if (bdev->driver->move_notify)
400 bdev->driver->move_notify(bo, mem);
Jerome Glisseca262a9992009-12-08 15:33:32 +0100401 bo->mem = *mem;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200402 mem->mm_node = NULL;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200403 goto moved;
404 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200405 }
406
Ben Skeggs9f1feed2012-01-25 15:34:22 +1000407 if (bdev->driver->move_notify)
408 bdev->driver->move_notify(bo, mem);
409
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200410 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
411 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000412 ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, mem);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200413 else if (bdev->driver->move)
414 ret = bdev->driver->move(bo, evict, interruptible,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000415 no_wait_reserve, no_wait_gpu, mem);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200416 else
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000417 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, mem);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200418
Ben Skeggs9f1feed2012-01-25 15:34:22 +1000419 if (ret) {
420 if (bdev->driver->move_notify) {
421 struct ttm_mem_reg tmp_mem = *mem;
422 *mem = bo->mem;
423 bo->mem = tmp_mem;
424 bdev->driver->move_notify(bo, mem);
425 bo->mem = *mem;
426 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200427
Ben Skeggs9f1feed2012-01-25 15:34:22 +1000428 goto out_err;
429 }
Jerome Glissedc97b342011-11-18 11:47:03 -0500430
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200431moved:
432 if (bo->evicted) {
433 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
434 if (ret)
Joe Perches25d04792012-03-16 21:43:50 -0700435 pr_err("Can not flush read caches\n");
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200436 bo->evicted = false;
437 }
438
439 if (bo->mem.mm_node) {
Ben Skeggsd961db72010-08-05 10:48:18 +1000440 bo->offset = (bo->mem.start << PAGE_SHIFT) +
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200441 bdev->man[bo->mem.mem_type].gpu_offset;
442 bo->cur_placement = bo->mem.placement;
Thomas Hellstrom354fb522010-01-13 22:28:45 +0100443 } else
444 bo->offset = 0;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200445
446 return 0;
447
448out_err:
449 new_man = &bdev->man[bo->mem.mem_type];
450 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) {
451 ttm_tt_unbind(bo->ttm);
452 ttm_tt_destroy(bo->ttm);
453 bo->ttm = NULL;
454 }
455
456 return ret;
457}
458
459/**
Thomas Hellstrom40d857b2010-10-19 09:01:00 +0200460 * Call bo::reserved.
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200461 * Will release GPU memory type usage on destruction.
Thomas Hellstrom40d857b2010-10-19 09:01:00 +0200462 * This is the place to put in driver specific hooks to release
463 * driver private resources.
464 * Will release the bo::reserved lock.
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200465 */
466
467static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
468{
Jerome Glissedc97b342011-11-18 11:47:03 -0500469 if (bo->bdev->driver->move_notify)
470 bo->bdev->driver->move_notify(bo, NULL);
471
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200472 if (bo->ttm) {
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200473 ttm_tt_unbind(bo->ttm);
474 ttm_tt_destroy(bo->ttm);
475 bo->ttm = NULL;
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200476 }
Thomas Hellstrom40d857b2010-10-19 09:01:00 +0200477 ttm_bo_mem_put(bo, &bo->mem);
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200478
479 atomic_set(&bo->reserved, 0);
Thomas Hellstrom06fba6d2010-10-29 10:46:48 +0200480
481 /*
482 * Make processes trying to reserve really pick it up.
483 */
484 smp_mb__after_atomic_dec();
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200485 wake_up_all(&bo->event_queue);
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200486}
487
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200488static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200489{
490 struct ttm_bo_device *bdev = bo->bdev;
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200491 struct ttm_bo_global *glob = bo->glob;
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200492 struct ttm_bo_driver *driver;
Thomas Hellstromaa123262010-11-02 13:21:47 +0000493 void *sync_obj = NULL;
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200494 void *sync_obj_arg;
495 int put_count;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200496 int ret;
497
Thomas Hellstrom702adba2010-11-17 12:28:29 +0000498 spin_lock(&bdev->fence_lock);
Dave Airlie1717c0e2011-10-27 18:28:37 +0200499 (void) ttm_bo_wait(bo, false, false, true);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200500 if (!bo->sync_obj) {
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200501
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200502 spin_lock(&glob->lru_lock);
Thomas Hellstromaaa20732009-12-02 18:33:45 +0100503
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200504 /**
Thomas Hellstrom702adba2010-11-17 12:28:29 +0000505 * Lock inversion between bo:reserve and bdev::fence_lock here,
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200506 * but that's OK, since we're only trylocking.
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200507 */
508
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200509 ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200510
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200511 if (unlikely(ret == -EBUSY))
512 goto queue;
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200513
Thomas Hellstrom702adba2010-11-17 12:28:29 +0000514 spin_unlock(&bdev->fence_lock);
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200515 put_count = ttm_bo_del_from_lru(bo);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200516
Thomas Hellstrom40d857b2010-10-19 09:01:00 +0200517 spin_unlock(&glob->lru_lock);
Thomas Hellstrom1df6a2e2010-09-30 12:36:45 +0200518 ttm_bo_cleanup_memtype_use(bo);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200519
Dave Airlied6ea8882010-11-22 13:24:40 +1000520 ttm_bo_list_ref_sub(bo, put_count, true);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200521
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200522 return;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200523 } else {
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200524 spin_lock(&glob->lru_lock);
525 }
526queue:
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200527 driver = bdev->driver;
Thomas Hellstromaa123262010-11-02 13:21:47 +0000528 if (bo->sync_obj)
529 sync_obj = driver->sync_obj_ref(bo->sync_obj);
530 sync_obj_arg = bo->sync_obj_arg;
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200531
532 kref_get(&bo->list_kref);
533 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
534 spin_unlock(&glob->lru_lock);
Thomas Hellstrom702adba2010-11-17 12:28:29 +0000535 spin_unlock(&bdev->fence_lock);
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200536
Thomas Hellstromaa123262010-11-02 13:21:47 +0000537 if (sync_obj) {
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200538 driver->sync_obj_flush(sync_obj, sync_obj_arg);
Thomas Hellstromaa123262010-11-02 13:21:47 +0000539 driver->sync_obj_unref(&sync_obj);
540 }
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200541 schedule_delayed_work(&bdev->wq,
542 ((HZ / 100) < 1) ? 1 : HZ / 100);
543}
544
545/**
546 * function ttm_bo_cleanup_refs
547 * If bo idle, remove from delayed- and lru lists, and unref.
548 * If not idle, do nothing.
549 *
550 * @interruptible Any sleeps should occur interruptibly.
551 * @no_wait_reserve Never wait for reserve. Return -EBUSY instead.
552 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead.
553 */
554
555static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
556 bool interruptible,
557 bool no_wait_reserve,
558 bool no_wait_gpu)
559{
Thomas Hellstrom702adba2010-11-17 12:28:29 +0000560 struct ttm_bo_device *bdev = bo->bdev;
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200561 struct ttm_bo_global *glob = bo->glob;
562 int put_count;
563 int ret = 0;
564
565retry:
Thomas Hellstrom702adba2010-11-17 12:28:29 +0000566 spin_lock(&bdev->fence_lock);
Dave Airlie1717c0e2011-10-27 18:28:37 +0200567 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
Thomas Hellstrom702adba2010-11-17 12:28:29 +0000568 spin_unlock(&bdev->fence_lock);
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200569
570 if (unlikely(ret != 0))
571 return ret;
572
573 spin_lock(&glob->lru_lock);
Thomas Hellstrom26cc40a2011-11-21 13:05:02 +0100574
575 if (unlikely(list_empty(&bo->ddestroy))) {
576 spin_unlock(&glob->lru_lock);
577 return 0;
578 }
579
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200580 ret = ttm_bo_reserve_locked(bo, interruptible,
581 no_wait_reserve, false, 0);
582
Thomas Hellstrom26cc40a2011-11-21 13:05:02 +0100583 if (unlikely(ret != 0)) {
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200584 spin_unlock(&glob->lru_lock);
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200585 return ret;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200586 }
587
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200588 /**
589 * We can re-check for sync object without taking
590 * the bo::lock since setting the sync object requires
591 * also bo::reserved. A busy object at this point may
592 * be caused by another thread recently starting an accelerated
593 * eviction.
594 */
595
596 if (unlikely(bo->sync_obj)) {
597 atomic_set(&bo->reserved, 0);
598 wake_up_all(&bo->event_queue);
599 spin_unlock(&glob->lru_lock);
600 goto retry;
601 }
602
603 put_count = ttm_bo_del_from_lru(bo);
604 list_del_init(&bo->ddestroy);
605 ++put_count;
606
607 spin_unlock(&glob->lru_lock);
608 ttm_bo_cleanup_memtype_use(bo);
609
Dave Airlied6ea8882010-11-22 13:24:40 +1000610 ttm_bo_list_ref_sub(bo, put_count, true);
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200611
612 return 0;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200613}
614
615/**
616 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
617 * encountered buffers.
618 */
619
620static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
621{
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200622 struct ttm_bo_global *glob = bdev->glob;
Luca Barbieri1a961ce2010-01-20 20:01:30 +0100623 struct ttm_buffer_object *entry = NULL;
624 int ret = 0;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200625
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200626 spin_lock(&glob->lru_lock);
Luca Barbieri1a961ce2010-01-20 20:01:30 +0100627 if (list_empty(&bdev->ddestroy))
628 goto out_unlock;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200629
Luca Barbieri1a961ce2010-01-20 20:01:30 +0100630 entry = list_first_entry(&bdev->ddestroy,
631 struct ttm_buffer_object, ddestroy);
632 kref_get(&entry->list_kref);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200633
Luca Barbieri1a961ce2010-01-20 20:01:30 +0100634 for (;;) {
635 struct ttm_buffer_object *nentry = NULL;
636
637 if (entry->ddestroy.next != &bdev->ddestroy) {
638 nentry = list_first_entry(&entry->ddestroy,
639 struct ttm_buffer_object, ddestroy);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200640 kref_get(&nentry->list_kref);
641 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200642
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200643 spin_unlock(&glob->lru_lock);
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200644 ret = ttm_bo_cleanup_refs(entry, false, !remove_all,
645 !remove_all);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200646 kref_put(&entry->list_kref, ttm_bo_release_list);
Luca Barbieri1a961ce2010-01-20 20:01:30 +0100647 entry = nentry;
648
649 if (ret || !entry)
650 goto out;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200651
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200652 spin_lock(&glob->lru_lock);
Luca Barbieri1a961ce2010-01-20 20:01:30 +0100653 if (list_empty(&entry->ddestroy))
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200654 break;
655 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200656
Luca Barbieri1a961ce2010-01-20 20:01:30 +0100657out_unlock:
658 spin_unlock(&glob->lru_lock);
659out:
660 if (entry)
661 kref_put(&entry->list_kref, ttm_bo_release_list);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200662 return ret;
663}
664
665static void ttm_bo_delayed_workqueue(struct work_struct *work)
666{
667 struct ttm_bo_device *bdev =
668 container_of(work, struct ttm_bo_device, wq.work);
669
670 if (ttm_bo_delayed_delete(bdev, false)) {
671 schedule_delayed_work(&bdev->wq,
672 ((HZ / 100) < 1) ? 1 : HZ / 100);
673 }
674}
675
676static void ttm_bo_release(struct kref *kref)
677{
678 struct ttm_buffer_object *bo =
679 container_of(kref, struct ttm_buffer_object, kref);
680 struct ttm_bo_device *bdev = bo->bdev;
Thomas Hellstromeba67092010-11-11 09:41:57 +0100681 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200682
683 if (likely(bo->vm_node != NULL)) {
684 rb_erase(&bo->vm_rb, &bdev->addr_space_rb);
685 drm_mm_put_block(bo->vm_node);
686 bo->vm_node = NULL;
687 }
688 write_unlock(&bdev->vm_lock);
Thomas Hellstromeba67092010-11-11 09:41:57 +0100689 ttm_mem_io_lock(man, false);
690 ttm_mem_io_free_vm(bo);
691 ttm_mem_io_unlock(man);
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200692 ttm_bo_cleanup_refs_or_queue(bo);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200693 kref_put(&bo->list_kref, ttm_bo_release_list);
694 write_lock(&bdev->vm_lock);
695}
696
697void ttm_bo_unref(struct ttm_buffer_object **p_bo)
698{
699 struct ttm_buffer_object *bo = *p_bo;
700 struct ttm_bo_device *bdev = bo->bdev;
701
702 *p_bo = NULL;
703 write_lock(&bdev->vm_lock);
704 kref_put(&bo->kref, ttm_bo_release);
705 write_unlock(&bdev->vm_lock);
706}
707EXPORT_SYMBOL(ttm_bo_unref);
708
Matthew Garrett7c5ee532010-04-26 16:00:09 -0400709int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
710{
711 return cancel_delayed_work_sync(&bdev->wq);
712}
713EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
714
715void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
716{
717 if (resched)
718 schedule_delayed_work(&bdev->wq,
719 ((HZ / 100) < 1) ? 1 : HZ / 100);
720}
721EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
722
Jerome Glisseca262a9992009-12-08 15:33:32 +0100723static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000724 bool no_wait_reserve, bool no_wait_gpu)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200725{
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200726 struct ttm_bo_device *bdev = bo->bdev;
727 struct ttm_mem_reg evict_mem;
Jerome Glisseca262a9992009-12-08 15:33:32 +0100728 struct ttm_placement placement;
729 int ret = 0;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200730
Thomas Hellstrom702adba2010-11-17 12:28:29 +0000731 spin_lock(&bdev->fence_lock);
Dave Airlie1717c0e2011-10-27 18:28:37 +0200732 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
Thomas Hellstrom702adba2010-11-17 12:28:29 +0000733 spin_unlock(&bdev->fence_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200734
Thomas Hellstrom78ecf092009-06-17 12:29:55 +0200735 if (unlikely(ret != 0)) {
Thomas Hellstrom98ffc4152009-12-07 18:36:18 +0100736 if (ret != -ERESTARTSYS) {
Joe Perches25d04792012-03-16 21:43:50 -0700737 pr_err("Failed to expire sync object before buffer eviction\n");
Thomas Hellstrom78ecf092009-06-17 12:29:55 +0200738 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200739 goto out;
740 }
741
742 BUG_ON(!atomic_read(&bo->reserved));
743
744 evict_mem = bo->mem;
745 evict_mem.mm_node = NULL;
Thomas Hellstromeba67092010-11-11 09:41:57 +0100746 evict_mem.bus.io_reserved_vm = false;
747 evict_mem.bus.io_reserved_count = 0;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200748
Jerome Glisse7cb7d1d2009-12-09 22:14:27 +0100749 placement.fpfn = 0;
750 placement.lpfn = 0;
751 placement.num_placement = 0;
752 placement.num_busy_placement = 0;
Jerome Glisseca262a9992009-12-08 15:33:32 +0100753 bdev->driver->evict_flags(bo, &placement);
754 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000755 no_wait_reserve, no_wait_gpu);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200756 if (ret) {
Jerome Glissefb53f862009-12-09 21:55:10 +0100757 if (ret != -ERESTARTSYS) {
Joe Perches25d04792012-03-16 21:43:50 -0700758 pr_err("Failed to find memory space for buffer 0x%p eviction\n",
759 bo);
Jerome Glissefb53f862009-12-09 21:55:10 +0100760 ttm_bo_mem_space_debug(bo, &placement);
761 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200762 goto out;
763 }
764
765 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000766 no_wait_reserve, no_wait_gpu);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200767 if (ret) {
Thomas Hellstrom98ffc4152009-12-07 18:36:18 +0100768 if (ret != -ERESTARTSYS)
Joe Perches25d04792012-03-16 21:43:50 -0700769 pr_err("Buffer eviction failed\n");
Ben Skeggs42311ff2010-08-04 12:07:08 +1000770 ttm_bo_mem_put(bo, &evict_mem);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200771 goto out;
772 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200773 bo->evicted = true;
774out:
775 return ret;
776}
777
Jerome Glisseca262a9992009-12-08 15:33:32 +0100778static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
779 uint32_t mem_type,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000780 bool interruptible, bool no_wait_reserve,
781 bool no_wait_gpu)
Jerome Glisseca262a9992009-12-08 15:33:32 +0100782{
783 struct ttm_bo_global *glob = bdev->glob;
784 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
785 struct ttm_buffer_object *bo;
786 int ret, put_count = 0;
787
Thomas Hellstrom9c51ba12009-12-02 18:33:46 +0100788retry:
Jerome Glisseca262a9992009-12-08 15:33:32 +0100789 spin_lock(&glob->lru_lock);
Thomas Hellstrom9c51ba12009-12-02 18:33:46 +0100790 if (list_empty(&man->lru)) {
791 spin_unlock(&glob->lru_lock);
792 return -EBUSY;
793 }
794
Jerome Glisseca262a9992009-12-08 15:33:32 +0100795 bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru);
796 kref_get(&bo->list_kref);
Thomas Hellstrom9c51ba12009-12-02 18:33:46 +0100797
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +0200798 if (!list_empty(&bo->ddestroy)) {
799 spin_unlock(&glob->lru_lock);
800 ret = ttm_bo_cleanup_refs(bo, interruptible,
801 no_wait_reserve, no_wait_gpu);
802 kref_put(&bo->list_kref, ttm_bo_release_list);
803
804 if (likely(ret == 0 || ret == -ERESTARTSYS))
805 return ret;
806
807 goto retry;
808 }
809
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000810 ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0);
Thomas Hellstrom9c51ba12009-12-02 18:33:46 +0100811
812 if (unlikely(ret == -EBUSY)) {
813 spin_unlock(&glob->lru_lock);
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000814 if (likely(!no_wait_gpu))
Thomas Hellstrom9c51ba12009-12-02 18:33:46 +0100815 ret = ttm_bo_wait_unreserved(bo, interruptible);
816
817 kref_put(&bo->list_kref, ttm_bo_release_list);
818
819 /**
820 * We *need* to retry after releasing the lru lock.
821 */
822
823 if (unlikely(ret != 0))
824 return ret;
825 goto retry;
826 }
827
828 put_count = ttm_bo_del_from_lru(bo);
Jerome Glisseca262a9992009-12-08 15:33:32 +0100829 spin_unlock(&glob->lru_lock);
Thomas Hellstrom9c51ba12009-12-02 18:33:46 +0100830
831 BUG_ON(ret != 0);
832
Dave Airlied6ea8882010-11-22 13:24:40 +1000833 ttm_bo_list_ref_sub(bo, put_count, true);
Thomas Hellstrom9c51ba12009-12-02 18:33:46 +0100834
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000835 ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu);
Jerome Glisseca262a9992009-12-08 15:33:32 +0100836 ttm_bo_unreserve(bo);
Thomas Hellstrom9c51ba12009-12-02 18:33:46 +0100837
Jerome Glisseca262a9992009-12-08 15:33:32 +0100838 kref_put(&bo->list_kref, ttm_bo_release_list);
839 return ret;
840}
841
Ben Skeggs42311ff2010-08-04 12:07:08 +1000842void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
843{
Ben Skeggsd961db72010-08-05 10:48:18 +1000844 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
Ben Skeggs42311ff2010-08-04 12:07:08 +1000845
Ben Skeggsd961db72010-08-05 10:48:18 +1000846 if (mem->mm_node)
847 (*man->func->put_node)(man, mem);
Ben Skeggs42311ff2010-08-04 12:07:08 +1000848}
849EXPORT_SYMBOL(ttm_bo_mem_put);
850
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200851/**
852 * Repeatedly evict memory from the LRU for @mem_type until we create enough
853 * space, or we've evicted everything and there isn't enough space.
854 */
Jerome Glisseca262a9992009-12-08 15:33:32 +0100855static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
856 uint32_t mem_type,
857 struct ttm_placement *placement,
858 struct ttm_mem_reg *mem,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000859 bool interruptible,
860 bool no_wait_reserve,
861 bool no_wait_gpu)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200862{
Jerome Glisseca262a9992009-12-08 15:33:32 +0100863 struct ttm_bo_device *bdev = bo->bdev;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200864 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200865 int ret;
866
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200867 do {
Ben Skeggsd961db72010-08-05 10:48:18 +1000868 ret = (*man->func->get_node)(man, bo, placement, mem);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200869 if (unlikely(ret != 0))
870 return ret;
Ben Skeggsd961db72010-08-05 10:48:18 +1000871 if (mem->mm_node)
Jerome Glisseca262a9992009-12-08 15:33:32 +0100872 break;
Jerome Glisseca262a9992009-12-08 15:33:32 +0100873 ret = ttm_mem_evict_first(bdev, mem_type, interruptible,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000874 no_wait_reserve, no_wait_gpu);
Jerome Glisseca262a9992009-12-08 15:33:32 +0100875 if (unlikely(ret != 0))
876 return ret;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200877 } while (1);
Ben Skeggsd961db72010-08-05 10:48:18 +1000878 if (mem->mm_node == NULL)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200879 return -ENOMEM;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200880 mem->mem_type = mem_type;
881 return 0;
882}
883
Thomas Hellstromae3e8122009-06-24 19:57:34 +0200884static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
885 uint32_t cur_placement,
886 uint32_t proposed_placement)
887{
888 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
889 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
890
891 /**
892 * Keep current caching if possible.
893 */
894
895 if ((cur_placement & caching) != 0)
896 result |= (cur_placement & caching);
897 else if ((man->default_caching & caching) != 0)
898 result |= man->default_caching;
899 else if ((TTM_PL_FLAG_CACHED & caching) != 0)
900 result |= TTM_PL_FLAG_CACHED;
901 else if ((TTM_PL_FLAG_WC & caching) != 0)
902 result |= TTM_PL_FLAG_WC;
903 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
904 result |= TTM_PL_FLAG_UNCACHED;
905
906 return result;
907}
908
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200909static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200910 uint32_t mem_type,
Thomas Hellstromae3e8122009-06-24 19:57:34 +0200911 uint32_t proposed_placement,
912 uint32_t *masked_placement)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200913{
914 uint32_t cur_flags = ttm_bo_type_flags(mem_type);
915
Thomas Hellstromae3e8122009-06-24 19:57:34 +0200916 if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200917 return false;
918
Thomas Hellstromae3e8122009-06-24 19:57:34 +0200919 if ((proposed_placement & man->available_caching) == 0)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200920 return false;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200921
Thomas Hellstromae3e8122009-06-24 19:57:34 +0200922 cur_flags |= (proposed_placement & man->available_caching);
923
924 *masked_placement = cur_flags;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200925 return true;
926}
927
928/**
929 * Creates space for memory region @mem according to its type.
930 *
931 * This function first searches for free space in compatible memory types in
932 * the priority order defined by the driver. If free space isn't found, then
933 * ttm_bo_mem_force_space is attempted in priority order to evict and find
934 * space.
935 */
936int ttm_bo_mem_space(struct ttm_buffer_object *bo,
Jerome Glisseca262a9992009-12-08 15:33:32 +0100937 struct ttm_placement *placement,
938 struct ttm_mem_reg *mem,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000939 bool interruptible, bool no_wait_reserve,
940 bool no_wait_gpu)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200941{
942 struct ttm_bo_device *bdev = bo->bdev;
943 struct ttm_mem_type_manager *man;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200944 uint32_t mem_type = TTM_PL_SYSTEM;
945 uint32_t cur_flags = 0;
946 bool type_found = false;
947 bool type_ok = false;
Thomas Hellstrom98ffc4152009-12-07 18:36:18 +0100948 bool has_erestartsys = false;
Jerome Glisseca262a9992009-12-08 15:33:32 +0100949 int i, ret;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200950
951 mem->mm_node = NULL;
Dave Airlieb6637522009-12-14 14:51:35 +1000952 for (i = 0; i < placement->num_placement; ++i) {
Jerome Glisseca262a9992009-12-08 15:33:32 +0100953 ret = ttm_mem_type_from_flags(placement->placement[i],
954 &mem_type);
955 if (ret)
956 return ret;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200957 man = &bdev->man[mem_type];
958
959 type_ok = ttm_bo_mt_compatible(man,
Jerome Glisseca262a9992009-12-08 15:33:32 +0100960 mem_type,
961 placement->placement[i],
962 &cur_flags);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200963
964 if (!type_ok)
965 continue;
966
Thomas Hellstromae3e8122009-06-24 19:57:34 +0200967 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
968 cur_flags);
Jerome Glisseca262a9992009-12-08 15:33:32 +0100969 /*
970 * Use the access and other non-mapping-related flag bits from
971 * the memory placement flags to the current flags
972 */
973 ttm_flag_masked(&cur_flags, placement->placement[i],
974 ~TTM_PL_MASK_MEMTYPE);
Thomas Hellstromae3e8122009-06-24 19:57:34 +0200975
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200976 if (mem_type == TTM_PL_SYSTEM)
977 break;
978
979 if (man->has_type && man->use_type) {
980 type_found = true;
Ben Skeggsd961db72010-08-05 10:48:18 +1000981 ret = (*man->func->get_node)(man, bo, placement, mem);
Jerome Glisseca262a9992009-12-08 15:33:32 +0100982 if (unlikely(ret))
983 return ret;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200984 }
Ben Skeggsd961db72010-08-05 10:48:18 +1000985 if (mem->mm_node)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200986 break;
987 }
988
Ben Skeggsd961db72010-08-05 10:48:18 +1000989 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
Thomas Hellstromba4e7d92009-06-10 15:20:19 +0200990 mem->mem_type = mem_type;
991 mem->placement = cur_flags;
992 return 0;
993 }
994
995 if (!type_found)
996 return -EINVAL;
997
Dave Airlieb6637522009-12-14 14:51:35 +1000998 for (i = 0; i < placement->num_busy_placement; ++i) {
999 ret = ttm_mem_type_from_flags(placement->busy_placement[i],
Jerome Glisseca262a9992009-12-08 15:33:32 +01001000 &mem_type);
1001 if (ret)
1002 return ret;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001003 man = &bdev->man[mem_type];
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001004 if (!man->has_type)
1005 continue;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001006 if (!ttm_bo_mt_compatible(man,
Jerome Glisseca262a9992009-12-08 15:33:32 +01001007 mem_type,
Dave Airlieb6637522009-12-14 14:51:35 +10001008 placement->busy_placement[i],
Jerome Glisseca262a9992009-12-08 15:33:32 +01001009 &cur_flags))
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001010 continue;
1011
Thomas Hellstromae3e8122009-06-24 19:57:34 +02001012 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
1013 cur_flags);
Jerome Glisseca262a9992009-12-08 15:33:32 +01001014 /*
1015 * Use the access and other non-mapping-related flag bits from
1016 * the memory placement flags to the current flags
1017 */
Dave Airlieb6637522009-12-14 14:51:35 +10001018 ttm_flag_masked(&cur_flags, placement->busy_placement[i],
Jerome Glisseca262a9992009-12-08 15:33:32 +01001019 ~TTM_PL_MASK_MEMTYPE);
Thomas Hellstromae3e8122009-06-24 19:57:34 +02001020
Thomas Hellstrom0eaddb22010-01-16 16:05:04 +01001021
1022 if (mem_type == TTM_PL_SYSTEM) {
1023 mem->mem_type = mem_type;
1024 mem->placement = cur_flags;
1025 mem->mm_node = NULL;
1026 return 0;
1027 }
1028
Jerome Glisseca262a9992009-12-08 15:33:32 +01001029 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem,
Jerome Glisse9d87fa22010-04-07 10:21:19 +00001030 interruptible, no_wait_reserve, no_wait_gpu);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001031 if (ret == 0 && mem->mm_node) {
1032 mem->placement = cur_flags;
1033 return 0;
1034 }
Thomas Hellstrom98ffc4152009-12-07 18:36:18 +01001035 if (ret == -ERESTARTSYS)
1036 has_erestartsys = true;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001037 }
Thomas Hellstrom98ffc4152009-12-07 18:36:18 +01001038 ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001039 return ret;
1040}
1041EXPORT_SYMBOL(ttm_bo_mem_space);
1042
1043int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait)
1044{
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001045 if ((atomic_read(&bo->cpu_writers) > 0) && no_wait)
1046 return -EBUSY;
1047
Thomas Hellstrom98ffc4152009-12-07 18:36:18 +01001048 return wait_event_interruptible(bo->event_queue,
1049 atomic_read(&bo->cpu_writers) == 0);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001050}
Ben Skeggsd1ede142009-12-11 15:13:00 +10001051EXPORT_SYMBOL(ttm_bo_wait_cpu);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001052
1053int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
Jerome Glisseca262a9992009-12-08 15:33:32 +01001054 struct ttm_placement *placement,
Jerome Glisse9d87fa22010-04-07 10:21:19 +00001055 bool interruptible, bool no_wait_reserve,
1056 bool no_wait_gpu)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001057{
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001058 int ret = 0;
1059 struct ttm_mem_reg mem;
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001060 struct ttm_bo_device *bdev = bo->bdev;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001061
1062 BUG_ON(!atomic_read(&bo->reserved));
1063
1064 /*
1065 * FIXME: It's possible to pipeline buffer moves.
1066 * Have the driver move function wait for idle when necessary,
1067 * instead of doing it here.
1068 */
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001069 spin_lock(&bdev->fence_lock);
Dave Airlie1717c0e2011-10-27 18:28:37 +02001070 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001071 spin_unlock(&bdev->fence_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001072 if (ret)
1073 return ret;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001074 mem.num_pages = bo->num_pages;
1075 mem.size = mem.num_pages << PAGE_SHIFT;
1076 mem.page_alignment = bo->mem.page_alignment;
Thomas Hellstromeba67092010-11-11 09:41:57 +01001077 mem.bus.io_reserved_vm = false;
1078 mem.bus.io_reserved_count = 0;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001079 /*
1080 * Determine where to move the buffer.
1081 */
Jerome Glisse9d87fa22010-04-07 10:21:19 +00001082 ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait_reserve, no_wait_gpu);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001083 if (ret)
1084 goto out_unlock;
Jerome Glisse9d87fa22010-04-07 10:21:19 +00001085 ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001086out_unlock:
Ben Skeggsd961db72010-08-05 10:48:18 +10001087 if (ret && mem.mm_node)
1088 ttm_bo_mem_put(bo, &mem);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001089 return ret;
1090}
1091
Jerome Glisseca262a9992009-12-08 15:33:32 +01001092static int ttm_bo_mem_compat(struct ttm_placement *placement,
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001093 struct ttm_mem_reg *mem)
1094{
Jerome Glisseca262a9992009-12-08 15:33:32 +01001095 int i;
Thomas Hellstrome22238e2010-02-12 00:18:00 +01001096
Ben Skeggsd961db72010-08-05 10:48:18 +10001097 if (mem->mm_node && placement->lpfn != 0 &&
1098 (mem->start < placement->fpfn ||
1099 mem->start + mem->num_pages > placement->lpfn))
Thomas Hellstrome22238e2010-02-12 00:18:00 +01001100 return -1;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001101
Jerome Glisseca262a9992009-12-08 15:33:32 +01001102 for (i = 0; i < placement->num_placement; i++) {
1103 if ((placement->placement[i] & mem->placement &
1104 TTM_PL_MASK_CACHING) &&
1105 (placement->placement[i] & mem->placement &
1106 TTM_PL_MASK_MEM))
1107 return i;
1108 }
1109 return -1;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001110}
1111
Jerome Glisse09855ac2009-12-10 17:16:27 +01001112int ttm_bo_validate(struct ttm_buffer_object *bo,
1113 struct ttm_placement *placement,
Jerome Glisse9d87fa22010-04-07 10:21:19 +00001114 bool interruptible, bool no_wait_reserve,
1115 bool no_wait_gpu)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001116{
1117 int ret;
1118
1119 BUG_ON(!atomic_read(&bo->reserved));
Jerome Glisseca262a9992009-12-08 15:33:32 +01001120 /* Check that range is valid */
1121 if (placement->lpfn || placement->fpfn)
1122 if (placement->fpfn > placement->lpfn ||
1123 (placement->lpfn - placement->fpfn) < bo->num_pages)
1124 return -EINVAL;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001125 /*
1126 * Check whether we need to move buffer.
1127 */
Jerome Glisseca262a9992009-12-08 15:33:32 +01001128 ret = ttm_bo_mem_compat(placement, &bo->mem);
1129 if (ret < 0) {
Jerome Glisse9d87fa22010-04-07 10:21:19 +00001130 ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu);
Jerome Glisseca262a9992009-12-08 15:33:32 +01001131 if (ret)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001132 return ret;
Jerome Glisseca262a9992009-12-08 15:33:32 +01001133 } else {
1134 /*
1135 * Use the access and other non-mapping-related flag bits from
1136 * the compatible memory placement flags to the active flags
1137 */
1138 ttm_flag_masked(&bo->mem.placement, placement->placement[ret],
1139 ~TTM_PL_MASK_MEMTYPE);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001140 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001141 /*
1142 * We might need to add a TTM.
1143 */
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001144 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1145 ret = ttm_bo_add_ttm(bo, true);
1146 if (ret)
1147 return ret;
1148 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001149 return 0;
1150}
Jerome Glisse09855ac2009-12-10 17:16:27 +01001151EXPORT_SYMBOL(ttm_bo_validate);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001152
Jerome Glisse09855ac2009-12-10 17:16:27 +01001153int ttm_bo_check_placement(struct ttm_buffer_object *bo,
1154 struct ttm_placement *placement)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001155{
Thomas Hellstrom29e190e2010-11-02 13:21:48 +00001156 BUG_ON((placement->fpfn || placement->lpfn) &&
1157 (bo->mem.num_pages > (placement->lpfn - placement->fpfn)));
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001158
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001159 return 0;
1160}
1161
Jerome Glisse09855ac2009-12-10 17:16:27 +01001162int ttm_bo_init(struct ttm_bo_device *bdev,
1163 struct ttm_buffer_object *bo,
1164 unsigned long size,
1165 enum ttm_bo_type type,
1166 struct ttm_placement *placement,
1167 uint32_t page_alignment,
1168 unsigned long buffer_start,
1169 bool interruptible,
Jan Engelhardt5df23972011-04-04 01:25:18 +02001170 struct file *persistent_swap_storage,
Jerome Glisse09855ac2009-12-10 17:16:27 +01001171 size_t acc_size,
1172 void (*destroy) (struct ttm_buffer_object *))
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001173{
Jerome Glisse09855ac2009-12-10 17:16:27 +01001174 int ret = 0;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001175 unsigned long num_pages;
Jerome Glisse57de4ba2011-11-11 15:42:57 -05001176 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1177
1178 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
1179 if (ret) {
Joe Perches25d04792012-03-16 21:43:50 -07001180 pr_err("Out of kernel memory\n");
Jerome Glisse57de4ba2011-11-11 15:42:57 -05001181 if (destroy)
1182 (*destroy)(bo);
1183 else
1184 kfree(bo);
1185 return -ENOMEM;
1186 }
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001187
1188 size += buffer_start & ~PAGE_MASK;
1189 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1190 if (num_pages == 0) {
Joe Perches25d04792012-03-16 21:43:50 -07001191 pr_err("Illegal buffer object size\n");
Thomas Hellstrom7dfbbdc2010-11-09 21:31:44 +01001192 if (destroy)
1193 (*destroy)(bo);
1194 else
1195 kfree(bo);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001196 return -EINVAL;
1197 }
1198 bo->destroy = destroy;
1199
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001200 kref_init(&bo->kref);
1201 kref_init(&bo->list_kref);
1202 atomic_set(&bo->cpu_writers, 0);
1203 atomic_set(&bo->reserved, 1);
1204 init_waitqueue_head(&bo->event_queue);
1205 INIT_LIST_HEAD(&bo->lru);
1206 INIT_LIST_HEAD(&bo->ddestroy);
1207 INIT_LIST_HEAD(&bo->swap);
Thomas Hellstromeba67092010-11-11 09:41:57 +01001208 INIT_LIST_HEAD(&bo->io_reserve_lru);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001209 bo->bdev = bdev;
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001210 bo->glob = bdev->glob;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001211 bo->type = type;
1212 bo->num_pages = num_pages;
Jerome Glisseeb6d2c32009-12-10 16:15:52 +01001213 bo->mem.size = num_pages << PAGE_SHIFT;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001214 bo->mem.mem_type = TTM_PL_SYSTEM;
1215 bo->mem.num_pages = bo->num_pages;
1216 bo->mem.mm_node = NULL;
1217 bo->mem.page_alignment = page_alignment;
Thomas Hellstromeba67092010-11-11 09:41:57 +01001218 bo->mem.bus.io_reserved_vm = false;
1219 bo->mem.bus.io_reserved_count = 0;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001220 bo->buffer_start = buffer_start & PAGE_MASK;
1221 bo->priv_flags = 0;
1222 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1223 bo->seq_valid = false;
Jan Engelhardt5df23972011-04-04 01:25:18 +02001224 bo->persistent_swap_storage = persistent_swap_storage;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001225 bo->acc_size = acc_size;
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001226 atomic_inc(&bo->glob->bo_count);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001227
Jerome Glisse09855ac2009-12-10 17:16:27 +01001228 ret = ttm_bo_check_placement(bo, placement);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001229 if (unlikely(ret != 0))
1230 goto out_err;
1231
1232 /*
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001233 * For ttm_bo_type_device buffers, allocate
1234 * address space from the device.
1235 */
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001236 if (bo->type == ttm_bo_type_device) {
1237 ret = ttm_bo_setup_vm(bo);
1238 if (ret)
1239 goto out_err;
1240 }
1241
Jerome Glisse9d87fa22010-04-07 10:21:19 +00001242 ret = ttm_bo_validate(bo, placement, interruptible, false, false);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001243 if (ret)
1244 goto out_err;
1245
1246 ttm_bo_unreserve(bo);
1247 return 0;
1248
1249out_err:
1250 ttm_bo_unreserve(bo);
1251 ttm_bo_unref(&bo);
1252
1253 return ret;
1254}
Jerome Glisse09855ac2009-12-10 17:16:27 +01001255EXPORT_SYMBOL(ttm_bo_init);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001256
Jerome Glisse57de4ba2011-11-11 15:42:57 -05001257size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1258 unsigned long bo_size,
1259 unsigned struct_size)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001260{
Jerome Glisse57de4ba2011-11-11 15:42:57 -05001261 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1262 size_t size = 0;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001263
Jerome Glisse57de4ba2011-11-11 15:42:57 -05001264 size += ttm_round_pot(struct_size);
1265 size += PAGE_ALIGN(npages * sizeof(void *));
1266 size += ttm_round_pot(sizeof(struct ttm_tt));
1267 return size;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001268}
Jerome Glisse57de4ba2011-11-11 15:42:57 -05001269EXPORT_SYMBOL(ttm_bo_acc_size);
1270
1271size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1272 unsigned long bo_size,
1273 unsigned struct_size)
1274{
1275 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1276 size_t size = 0;
1277
1278 size += ttm_round_pot(struct_size);
1279 size += PAGE_ALIGN(npages * sizeof(void *));
1280 size += PAGE_ALIGN(npages * sizeof(dma_addr_t));
1281 size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1282 return size;
1283}
1284EXPORT_SYMBOL(ttm_bo_dma_acc_size);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001285
Jerome Glisse09855ac2009-12-10 17:16:27 +01001286int ttm_bo_create(struct ttm_bo_device *bdev,
1287 unsigned long size,
1288 enum ttm_bo_type type,
1289 struct ttm_placement *placement,
1290 uint32_t page_alignment,
1291 unsigned long buffer_start,
1292 bool interruptible,
Jan Engelhardt5df23972011-04-04 01:25:18 +02001293 struct file *persistent_swap_storage,
Jerome Glisse09855ac2009-12-10 17:16:27 +01001294 struct ttm_buffer_object **p_bo)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001295{
1296 struct ttm_buffer_object *bo;
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001297 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
Jerome Glisse57de4ba2011-11-11 15:42:57 -05001298 size_t acc_size;
Jerome Glisseca262a9992009-12-08 15:33:32 +01001299 int ret;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001300
Jerome Glisse57de4ba2011-11-11 15:42:57 -05001301 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
Thomas Hellstrom5fd9cba2009-08-17 16:28:39 +02001302 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001303 if (unlikely(ret != 0))
1304 return ret;
1305
1306 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1307
1308 if (unlikely(bo == NULL)) {
Thomas Hellstrom5fd9cba2009-08-17 16:28:39 +02001309 ttm_mem_global_free(mem_glob, acc_size);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001310 return -ENOMEM;
1311 }
1312
Jerome Glisse09855ac2009-12-10 17:16:27 +01001313 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1314 buffer_start, interruptible,
Jan Engelhardt5df23972011-04-04 01:25:18 +02001315 persistent_swap_storage, acc_size, NULL);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001316 if (likely(ret == 0))
1317 *p_bo = bo;
1318
1319 return ret;
1320}
Thomas Hellstrom4d798932011-10-04 20:13:11 +02001321EXPORT_SYMBOL(ttm_bo_create);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001322
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001323static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
Jerome Glisseca262a9992009-12-08 15:33:32 +01001324 unsigned mem_type, bool allow_errors)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001325{
Jerome Glisseca262a9992009-12-08 15:33:32 +01001326 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001327 struct ttm_bo_global *glob = bdev->glob;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001328 int ret;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001329
1330 /*
1331 * Can't use standard list traversal since we're unlocking.
1332 */
1333
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001334 spin_lock(&glob->lru_lock);
Jerome Glisseca262a9992009-12-08 15:33:32 +01001335 while (!list_empty(&man->lru)) {
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001336 spin_unlock(&glob->lru_lock);
Jerome Glisse9d87fa22010-04-07 10:21:19 +00001337 ret = ttm_mem_evict_first(bdev, mem_type, false, false, false);
Jerome Glisseca262a9992009-12-08 15:33:32 +01001338 if (ret) {
1339 if (allow_errors) {
1340 return ret;
1341 } else {
Joe Perches25d04792012-03-16 21:43:50 -07001342 pr_err("Cleanup eviction failed\n");
Jerome Glisseca262a9992009-12-08 15:33:32 +01001343 }
1344 }
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001345 spin_lock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001346 }
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001347 spin_unlock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001348 return 0;
1349}
1350
1351int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1352{
Roel Kluinc96e7c72009-08-03 14:22:53 +02001353 struct ttm_mem_type_manager *man;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001354 int ret = -EINVAL;
1355
1356 if (mem_type >= TTM_NUM_MEM_TYPES) {
Joe Perches25d04792012-03-16 21:43:50 -07001357 pr_err("Illegal memory type %d\n", mem_type);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001358 return ret;
1359 }
Roel Kluinc96e7c72009-08-03 14:22:53 +02001360 man = &bdev->man[mem_type];
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001361
1362 if (!man->has_type) {
Joe Perches25d04792012-03-16 21:43:50 -07001363 pr_err("Trying to take down uninitialized memory manager type %u\n",
1364 mem_type);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001365 return ret;
1366 }
1367
1368 man->use_type = false;
1369 man->has_type = false;
1370
1371 ret = 0;
1372 if (mem_type > 0) {
Jerome Glisseca262a9992009-12-08 15:33:32 +01001373 ttm_bo_force_list_clean(bdev, mem_type, false);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001374
Ben Skeggsd961db72010-08-05 10:48:18 +10001375 ret = (*man->func->takedown)(man);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001376 }
1377
1378 return ret;
1379}
1380EXPORT_SYMBOL(ttm_bo_clean_mm);
1381
1382int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1383{
1384 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1385
1386 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
Joe Perches25d04792012-03-16 21:43:50 -07001387 pr_err("Illegal memory manager memory type %u\n", mem_type);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001388 return -EINVAL;
1389 }
1390
1391 if (!man->has_type) {
Joe Perches25d04792012-03-16 21:43:50 -07001392 pr_err("Memory type %u has not been initialized\n", mem_type);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001393 return 0;
1394 }
1395
Jerome Glisseca262a9992009-12-08 15:33:32 +01001396 return ttm_bo_force_list_clean(bdev, mem_type, true);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001397}
1398EXPORT_SYMBOL(ttm_bo_evict_mm);
1399
1400int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
Jerome Glisseca262a9992009-12-08 15:33:32 +01001401 unsigned long p_size)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001402{
1403 int ret = -EINVAL;
1404 struct ttm_mem_type_manager *man;
1405
Thomas Hellstromdbc4a5b2010-10-29 10:46:47 +02001406 BUG_ON(type >= TTM_NUM_MEM_TYPES);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001407 man = &bdev->man[type];
Thomas Hellstromdbc4a5b2010-10-29 10:46:47 +02001408 BUG_ON(man->has_type);
Thomas Hellstromeba67092010-11-11 09:41:57 +01001409 man->io_reserve_fastpath = true;
1410 man->use_io_reserve_lru = false;
1411 mutex_init(&man->io_reserve_mutex);
1412 INIT_LIST_HEAD(&man->io_reserve_lru);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001413
1414 ret = bdev->driver->init_mem_type(bdev, type, man);
1415 if (ret)
1416 return ret;
Ben Skeggsd961db72010-08-05 10:48:18 +10001417 man->bdev = bdev;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001418
1419 ret = 0;
1420 if (type != TTM_PL_SYSTEM) {
Ben Skeggsd961db72010-08-05 10:48:18 +10001421 ret = (*man->func->init)(man, p_size);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001422 if (ret)
1423 return ret;
1424 }
1425 man->has_type = true;
1426 man->use_type = true;
1427 man->size = p_size;
1428
1429 INIT_LIST_HEAD(&man->lru);
1430
1431 return 0;
1432}
1433EXPORT_SYMBOL(ttm_bo_init_mm);
1434
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001435static void ttm_bo_global_kobj_release(struct kobject *kobj)
1436{
1437 struct ttm_bo_global *glob =
1438 container_of(kobj, struct ttm_bo_global, kobj);
1439
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001440 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink);
1441 __free_page(glob->dummy_read_page);
1442 kfree(glob);
1443}
1444
Dave Airlieba4420c2010-03-09 10:56:52 +10001445void ttm_bo_global_release(struct drm_global_reference *ref)
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001446{
1447 struct ttm_bo_global *glob = ref->object;
1448
1449 kobject_del(&glob->kobj);
1450 kobject_put(&glob->kobj);
1451}
1452EXPORT_SYMBOL(ttm_bo_global_release);
1453
Dave Airlieba4420c2010-03-09 10:56:52 +10001454int ttm_bo_global_init(struct drm_global_reference *ref)
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001455{
1456 struct ttm_bo_global_ref *bo_ref =
1457 container_of(ref, struct ttm_bo_global_ref, ref);
1458 struct ttm_bo_global *glob = ref->object;
1459 int ret;
1460
1461 mutex_init(&glob->device_list_mutex);
1462 spin_lock_init(&glob->lru_lock);
1463 glob->mem_glob = bo_ref->mem_glob;
1464 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1465
1466 if (unlikely(glob->dummy_read_page == NULL)) {
1467 ret = -ENOMEM;
1468 goto out_no_drp;
1469 }
1470
1471 INIT_LIST_HEAD(&glob->swap_lru);
1472 INIT_LIST_HEAD(&glob->device_list);
1473
1474 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout);
1475 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink);
1476 if (unlikely(ret != 0)) {
Joe Perches25d04792012-03-16 21:43:50 -07001477 pr_err("Could not register buffer object swapout\n");
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001478 goto out_no_shrink;
1479 }
1480
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001481 atomic_set(&glob->bo_count, 0);
1482
Robert P. J. Dayb642ed02010-03-13 10:36:32 +00001483 ret = kobject_init_and_add(
1484 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001485 if (unlikely(ret != 0))
1486 kobject_put(&glob->kobj);
1487 return ret;
1488out_no_shrink:
1489 __free_page(glob->dummy_read_page);
1490out_no_drp:
1491 kfree(glob);
1492 return ret;
1493}
1494EXPORT_SYMBOL(ttm_bo_global_init);
1495
1496
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001497int ttm_bo_device_release(struct ttm_bo_device *bdev)
1498{
1499 int ret = 0;
1500 unsigned i = TTM_NUM_MEM_TYPES;
1501 struct ttm_mem_type_manager *man;
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001502 struct ttm_bo_global *glob = bdev->glob;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001503
1504 while (i--) {
1505 man = &bdev->man[i];
1506 if (man->has_type) {
1507 man->use_type = false;
1508 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1509 ret = -EBUSY;
Joe Perches25d04792012-03-16 21:43:50 -07001510 pr_err("DRM memory manager type %d is not clean\n",
1511 i);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001512 }
1513 man->has_type = false;
1514 }
1515 }
1516
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001517 mutex_lock(&glob->device_list_mutex);
1518 list_del(&bdev->device_list);
1519 mutex_unlock(&glob->device_list_mutex);
1520
Tejun Heof094cfc2010-12-24 15:59:06 +01001521 cancel_delayed_work_sync(&bdev->wq);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001522
1523 while (ttm_bo_delayed_delete(bdev, true))
1524 ;
1525
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001526 spin_lock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001527 if (list_empty(&bdev->ddestroy))
1528 TTM_DEBUG("Delayed destroy list was clean\n");
1529
1530 if (list_empty(&bdev->man[0].lru))
1531 TTM_DEBUG("Swap list was clean\n");
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001532 spin_unlock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001533
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001534 BUG_ON(!drm_mm_clean(&bdev->addr_space_mm));
1535 write_lock(&bdev->vm_lock);
1536 drm_mm_takedown(&bdev->addr_space_mm);
1537 write_unlock(&bdev->vm_lock);
1538
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001539 return ret;
1540}
1541EXPORT_SYMBOL(ttm_bo_device_release);
1542
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001543int ttm_bo_device_init(struct ttm_bo_device *bdev,
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001544 struct ttm_bo_global *glob,
1545 struct ttm_bo_driver *driver,
Dave Airlie51c8b402009-08-20 13:38:04 +10001546 uint64_t file_page_offset,
Dave Airliead49f502009-07-10 22:36:26 +10001547 bool need_dma32)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001548{
1549 int ret = -EINVAL;
1550
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001551 rwlock_init(&bdev->vm_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001552 bdev->driver = driver;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001553
1554 memset(bdev->man, 0, sizeof(bdev->man));
1555
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001556 /*
1557 * Initialize the system memory buffer type.
1558 * Other types need to be driver / IOCTL initialized.
1559 */
Jerome Glisseca262a9992009-12-08 15:33:32 +01001560 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001561 if (unlikely(ret != 0))
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001562 goto out_no_sys;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001563
1564 bdev->addr_space_rb = RB_ROOT;
1565 ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000);
1566 if (unlikely(ret != 0))
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001567 goto out_no_addr_mm;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001568
1569 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1570 bdev->nice_mode = true;
1571 INIT_LIST_HEAD(&bdev->ddestroy);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001572 bdev->dev_mapping = NULL;
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001573 bdev->glob = glob;
Dave Airliead49f502009-07-10 22:36:26 +10001574 bdev->need_dma32 = need_dma32;
Thomas Hellstrom65705962010-11-17 12:28:31 +00001575 bdev->val_seq = 0;
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001576 spin_lock_init(&bdev->fence_lock);
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001577 mutex_lock(&glob->device_list_mutex);
1578 list_add_tail(&bdev->device_list, &glob->device_list);
1579 mutex_unlock(&glob->device_list_mutex);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001580
1581 return 0;
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001582out_no_addr_mm:
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001583 ttm_bo_clean_mm(bdev, 0);
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001584out_no_sys:
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001585 return ret;
1586}
1587EXPORT_SYMBOL(ttm_bo_device_init);
1588
1589/*
1590 * buffer object vm functions.
1591 */
1592
1593bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1594{
1595 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1596
1597 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1598 if (mem->mem_type == TTM_PL_SYSTEM)
1599 return false;
1600
1601 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1602 return false;
1603
1604 if (mem->placement & TTM_PL_FLAG_CACHED)
1605 return false;
1606 }
1607 return true;
1608}
1609
Thomas Hellstromeba67092010-11-11 09:41:57 +01001610void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001611{
1612 struct ttm_bo_device *bdev = bo->bdev;
1613 loff_t offset = (loff_t) bo->addr_space_offset;
1614 loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT;
1615
1616 if (!bdev->dev_mapping)
1617 return;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001618 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1);
Thomas Hellstromeba67092010-11-11 09:41:57 +01001619 ttm_mem_io_free_vm(bo);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001620}
Thomas Hellstromeba67092010-11-11 09:41:57 +01001621
1622void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1623{
1624 struct ttm_bo_device *bdev = bo->bdev;
1625 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1626
1627 ttm_mem_io_lock(man, false);
1628 ttm_bo_unmap_virtual_locked(bo);
1629 ttm_mem_io_unlock(man);
1630}
1631
1632
Dave Airliee024e112009-06-24 09:48:08 +10001633EXPORT_SYMBOL(ttm_bo_unmap_virtual);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001634
1635static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
1636{
1637 struct ttm_bo_device *bdev = bo->bdev;
1638 struct rb_node **cur = &bdev->addr_space_rb.rb_node;
1639 struct rb_node *parent = NULL;
1640 struct ttm_buffer_object *cur_bo;
1641 unsigned long offset = bo->vm_node->start;
1642 unsigned long cur_offset;
1643
1644 while (*cur) {
1645 parent = *cur;
1646 cur_bo = rb_entry(parent, struct ttm_buffer_object, vm_rb);
1647 cur_offset = cur_bo->vm_node->start;
1648 if (offset < cur_offset)
1649 cur = &parent->rb_left;
1650 else if (offset > cur_offset)
1651 cur = &parent->rb_right;
1652 else
1653 BUG();
1654 }
1655
1656 rb_link_node(&bo->vm_rb, parent, cur);
1657 rb_insert_color(&bo->vm_rb, &bdev->addr_space_rb);
1658}
1659
1660/**
1661 * ttm_bo_setup_vm:
1662 *
1663 * @bo: the buffer to allocate address space for
1664 *
1665 * Allocate address space in the drm device so that applications
1666 * can mmap the buffer and access the contents. This only
1667 * applies to ttm_bo_type_device objects as others are not
1668 * placed in the drm device address space.
1669 */
1670
1671static int ttm_bo_setup_vm(struct ttm_buffer_object *bo)
1672{
1673 struct ttm_bo_device *bdev = bo->bdev;
1674 int ret;
1675
1676retry_pre_get:
1677 ret = drm_mm_pre_get(&bdev->addr_space_mm);
1678 if (unlikely(ret != 0))
1679 return ret;
1680
1681 write_lock(&bdev->vm_lock);
1682 bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm,
1683 bo->mem.num_pages, 0, 0);
1684
1685 if (unlikely(bo->vm_node == NULL)) {
1686 ret = -ENOMEM;
1687 goto out_unlock;
1688 }
1689
1690 bo->vm_node = drm_mm_get_block_atomic(bo->vm_node,
1691 bo->mem.num_pages, 0);
1692
1693 if (unlikely(bo->vm_node == NULL)) {
1694 write_unlock(&bdev->vm_lock);
1695 goto retry_pre_get;
1696 }
1697
1698 ttm_bo_vm_insert_rb(bo);
1699 write_unlock(&bdev->vm_lock);
1700 bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT;
1701
1702 return 0;
1703out_unlock:
1704 write_unlock(&bdev->vm_lock);
1705 return ret;
1706}
1707
1708int ttm_bo_wait(struct ttm_buffer_object *bo,
Dave Airlie1717c0e2011-10-27 18:28:37 +02001709 bool lazy, bool interruptible, bool no_wait)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001710{
1711 struct ttm_bo_driver *driver = bo->bdev->driver;
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001712 struct ttm_bo_device *bdev = bo->bdev;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001713 void *sync_obj;
1714 void *sync_obj_arg;
1715 int ret = 0;
1716
Dave Airlie1717c0e2011-10-27 18:28:37 +02001717 if (likely(bo->sync_obj == NULL))
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001718 return 0;
1719
Dave Airlie1717c0e2011-10-27 18:28:37 +02001720 while (bo->sync_obj) {
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001721
Dave Airlie1717c0e2011-10-27 18:28:37 +02001722 if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) {
1723 void *tmp_obj = bo->sync_obj;
1724 bo->sync_obj = NULL;
1725 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
1726 spin_unlock(&bdev->fence_lock);
1727 driver->sync_obj_unref(&tmp_obj);
1728 spin_lock(&bdev->fence_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001729 continue;
1730 }
1731
1732 if (no_wait)
1733 return -EBUSY;
1734
Dave Airlie1717c0e2011-10-27 18:28:37 +02001735 sync_obj = driver->sync_obj_ref(bo->sync_obj);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001736 sync_obj_arg = bo->sync_obj_arg;
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001737 spin_unlock(&bdev->fence_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001738 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg,
1739 lazy, interruptible);
1740 if (unlikely(ret != 0)) {
1741 driver->sync_obj_unref(&sync_obj);
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001742 spin_lock(&bdev->fence_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001743 return ret;
1744 }
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001745 spin_lock(&bdev->fence_lock);
Dave Airlie1717c0e2011-10-27 18:28:37 +02001746 if (likely(bo->sync_obj == sync_obj &&
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001747 bo->sync_obj_arg == sync_obj_arg)) {
Dave Airlie1717c0e2011-10-27 18:28:37 +02001748 void *tmp_obj = bo->sync_obj;
1749 bo->sync_obj = NULL;
1750 clear_bit(TTM_BO_PRIV_FLAG_MOVING,
1751 &bo->priv_flags);
1752 spin_unlock(&bdev->fence_lock);
1753 driver->sync_obj_unref(&sync_obj);
1754 driver->sync_obj_unref(&tmp_obj);
1755 spin_lock(&bdev->fence_lock);
Thomas Hellstromfee280d2009-08-03 12:39:06 +02001756 } else {
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001757 spin_unlock(&bdev->fence_lock);
Thomas Hellstromfee280d2009-08-03 12:39:06 +02001758 driver->sync_obj_unref(&sync_obj);
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001759 spin_lock(&bdev->fence_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001760 }
1761 }
1762 return 0;
1763}
1764EXPORT_SYMBOL(ttm_bo_wait);
1765
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001766int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1767{
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001768 struct ttm_bo_device *bdev = bo->bdev;
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001769 int ret = 0;
1770
1771 /*
Thomas Hellstrom8cfe92d2010-04-28 11:33:25 +02001772 * Using ttm_bo_reserve makes sure the lru lists are updated.
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001773 */
1774
1775 ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
1776 if (unlikely(ret != 0))
1777 return ret;
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001778 spin_lock(&bdev->fence_lock);
Dave Airlie1717c0e2011-10-27 18:28:37 +02001779 ret = ttm_bo_wait(bo, false, true, no_wait);
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001780 spin_unlock(&bdev->fence_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001781 if (likely(ret == 0))
1782 atomic_inc(&bo->cpu_writers);
1783 ttm_bo_unreserve(bo);
1784 return ret;
1785}
Ben Skeggsd1ede142009-12-11 15:13:00 +10001786EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001787
1788void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
1789{
1790 if (atomic_dec_and_test(&bo->cpu_writers))
1791 wake_up_all(&bo->event_queue);
1792}
Ben Skeggsd1ede142009-12-11 15:13:00 +10001793EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001794
1795/**
1796 * A buffer object shrink method that tries to swap out the first
1797 * buffer object on the bo_global::swap_lru list.
1798 */
1799
1800static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
1801{
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001802 struct ttm_bo_global *glob =
1803 container_of(shrink, struct ttm_bo_global, shrink);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001804 struct ttm_buffer_object *bo;
1805 int ret = -EBUSY;
1806 int put_count;
1807 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM);
1808
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001809 spin_lock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001810 while (ret == -EBUSY) {
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001811 if (unlikely(list_empty(&glob->swap_lru))) {
1812 spin_unlock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001813 return -EBUSY;
1814 }
1815
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001816 bo = list_first_entry(&glob->swap_lru,
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001817 struct ttm_buffer_object, swap);
1818 kref_get(&bo->list_kref);
1819
Thomas Hellstrome1efc9b2010-10-19 09:01:01 +02001820 if (!list_empty(&bo->ddestroy)) {
1821 spin_unlock(&glob->lru_lock);
1822 (void) ttm_bo_cleanup_refs(bo, false, false, false);
1823 kref_put(&bo->list_kref, ttm_bo_release_list);
1824 continue;
1825 }
1826
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001827 /**
1828 * Reserve buffer. Since we unlock while sleeping, we need
1829 * to re-check that nobody removed us from the swap-list while
1830 * we slept.
1831 */
1832
1833 ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
1834 if (unlikely(ret == -EBUSY)) {
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001835 spin_unlock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001836 ttm_bo_wait_unreserved(bo, false);
1837 kref_put(&bo->list_kref, ttm_bo_release_list);
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001838 spin_lock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001839 }
1840 }
1841
1842 BUG_ON(ret != 0);
1843 put_count = ttm_bo_del_from_lru(bo);
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001844 spin_unlock(&glob->lru_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001845
Dave Airlied6ea8882010-11-22 13:24:40 +10001846 ttm_bo_list_ref_sub(bo, put_count, true);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001847
1848 /**
1849 * Wait for GPU, then move to system cached.
1850 */
1851
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001852 spin_lock(&bo->bdev->fence_lock);
Dave Airlie1717c0e2011-10-27 18:28:37 +02001853 ret = ttm_bo_wait(bo, false, false, false);
Thomas Hellstrom702adba2010-11-17 12:28:29 +00001854 spin_unlock(&bo->bdev->fence_lock);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001855
1856 if (unlikely(ret != 0))
1857 goto out;
1858
1859 if ((bo->mem.placement & swap_placement) != swap_placement) {
1860 struct ttm_mem_reg evict_mem;
1861
1862 evict_mem = bo->mem;
1863 evict_mem.mm_node = NULL;
1864 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1865 evict_mem.mem_type = TTM_PL_SYSTEM;
1866
1867 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true,
Jerome Glisse9d87fa22010-04-07 10:21:19 +00001868 false, false, false);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001869 if (unlikely(ret != 0))
1870 goto out;
1871 }
1872
1873 ttm_bo_unmap_virtual(bo);
1874
1875 /**
1876 * Swap out. Buffer will be swapped in again as soon as
1877 * anyone tries to access a ttm page.
1878 */
1879
Thomas Hellstrom3f09ea42010-01-13 22:28:40 +01001880 if (bo->bdev->driver->swap_notify)
1881 bo->bdev->driver->swap_notify(bo);
1882
Jan Engelhardt5df23972011-04-04 01:25:18 +02001883 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001884out:
1885
1886 /**
1887 *
1888 * Unreserve without putting on LRU to avoid swapping out an
1889 * already swapped buffer.
1890 */
1891
1892 atomic_set(&bo->reserved, 0);
1893 wake_up_all(&bo->event_queue);
1894 kref_put(&bo->list_kref, ttm_bo_release_list);
1895 return ret;
1896}
1897
1898void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1899{
Thomas Hellstroma987fca2009-08-18 16:51:56 +02001900 while (ttm_bo_swapout(&bdev->glob->shrink) == 0)
Thomas Hellstromba4e7d92009-06-10 15:20:19 +02001901 ;
1902}
Thomas Hellstrome99e1e72010-01-13 22:28:42 +01001903EXPORT_SYMBOL(ttm_bo_swapout_all);