Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. |
Jack Morgenstein | 51a379d | 2008-07-25 10:32:52 -0700 | [diff] [blame] | 3 | * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 4 | * |
| 5 | * This software is available to you under a choice of one of two |
| 6 | * licenses. You may choose to be licensed under the terms of the GNU |
| 7 | * General Public License (GPL) Version 2, available from the file |
| 8 | * COPYING in the main directory of this source tree, or the |
| 9 | * OpenIB.org BSD license below: |
| 10 | * |
| 11 | * Redistribution and use in source and binary forms, with or |
| 12 | * without modification, are permitted provided that the following |
| 13 | * conditions are met: |
| 14 | * |
| 15 | * - Redistributions of source code must retain the above |
| 16 | * copyright notice, this list of conditions and the following |
| 17 | * disclaimer. |
| 18 | * |
| 19 | * - Redistributions in binary form must reproduce the above |
| 20 | * copyright notice, this list of conditions and the following |
| 21 | * disclaimer in the documentation and/or other materials |
| 22 | * provided with the distribution. |
| 23 | * |
| 24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 31 | * SOFTWARE. |
| 32 | */ |
| 33 | |
| 34 | #include <linux/errno.h> |
| 35 | #include <linux/slab.h> |
Andrew Morton | 6526128 | 2008-09-05 14:04:07 -0700 | [diff] [blame] | 36 | #include <linux/mm.h> |
Paul Gortmaker | ee40fa0 | 2011-05-27 16:14:23 -0400 | [diff] [blame] | 37 | #include <linux/export.h> |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 38 | #include <linux/bitmap.h> |
Al Viro | 9cbe05c | 2007-05-15 20:36:30 +0100 | [diff] [blame] | 39 | #include <linux/dma-mapping.h> |
Olof Johansson | 29c2711 | 2008-02-10 20:22:57 -0600 | [diff] [blame] | 40 | #include <linux/vmalloc.h> |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 41 | |
| 42 | #include "mlx4.h" |
| 43 | |
| 44 | u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) |
| 45 | { |
| 46 | u32 obj; |
| 47 | |
| 48 | spin_lock(&bitmap->lock); |
| 49 | |
| 50 | obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); |
| 51 | if (obj >= bitmap->max) { |
Yevgeny Petrilin | 93fc9e1 | 2008-10-22 10:25:29 -0700 | [diff] [blame] | 52 | bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
| 53 | & bitmap->mask; |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 54 | obj = find_first_zero_bit(bitmap->table, bitmap->max); |
| 55 | } |
| 56 | |
| 57 | if (obj < bitmap->max) { |
| 58 | set_bit(obj, bitmap->table); |
Yevgeny Petrilin | 93fc9e1 | 2008-10-22 10:25:29 -0700 | [diff] [blame] | 59 | bitmap->last = (obj + 1); |
| 60 | if (bitmap->last == bitmap->max) |
| 61 | bitmap->last = 0; |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 62 | obj |= bitmap->top; |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 63 | } else |
| 64 | obj = -1; |
| 65 | |
Eli Cohen | 42d1e01 | 2011-03-22 22:38:45 +0000 | [diff] [blame] | 66 | if (obj != -1) |
| 67 | --bitmap->avail; |
| 68 | |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 69 | spin_unlock(&bitmap->lock); |
| 70 | |
| 71 | return obj; |
| 72 | } |
| 73 | |
Jack Morgenstein | 7c6d74d | 2013-12-08 16:50:17 +0200 | [diff] [blame] | 74 | void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 75 | { |
Jack Morgenstein | 7c6d74d | 2013-12-08 16:50:17 +0200 | [diff] [blame] | 76 | mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); |
Yevgeny Petrilin | a3cdcbf | 2008-10-10 12:01:37 -0700 | [diff] [blame] | 77 | } |
| 78 | |
Eugenia Emantayev | ddae034 | 2014-12-11 10:57:54 +0200 | [diff] [blame] | 79 | static unsigned long find_aligned_range(unsigned long *bitmap, |
| 80 | u32 start, u32 nbits, |
| 81 | int len, int align, u32 skip_mask) |
| 82 | { |
| 83 | unsigned long end, i; |
| 84 | |
| 85 | again: |
| 86 | start = ALIGN(start, align); |
| 87 | |
| 88 | while ((start < nbits) && (test_bit(start, bitmap) || |
| 89 | (start & skip_mask))) |
| 90 | start += align; |
| 91 | |
| 92 | if (start >= nbits) |
| 93 | return -1; |
| 94 | |
| 95 | end = start+len; |
| 96 | if (end > nbits) |
| 97 | return -1; |
| 98 | |
| 99 | for (i = start + 1; i < end; i++) { |
| 100 | if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { |
| 101 | start = i + 1; |
| 102 | goto again; |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | return start; |
| 107 | } |
| 108 | |
| 109 | u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, |
| 110 | int align, u32 skip_mask) |
Yevgeny Petrilin | a3cdcbf | 2008-10-10 12:01:37 -0700 | [diff] [blame] | 111 | { |
Akinobu Mita | e27cd4f | 2010-08-27 19:08:13 +0000 | [diff] [blame] | 112 | u32 obj; |
Yevgeny Petrilin | a3cdcbf | 2008-10-10 12:01:37 -0700 | [diff] [blame] | 113 | |
Eugenia Emantayev | ddae034 | 2014-12-11 10:57:54 +0200 | [diff] [blame] | 114 | if (likely(cnt == 1 && align == 1 && !skip_mask)) |
Yevgeny Petrilin | a3cdcbf | 2008-10-10 12:01:37 -0700 | [diff] [blame] | 115 | return mlx4_bitmap_alloc(bitmap); |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 116 | |
| 117 | spin_lock(&bitmap->lock); |
Yevgeny Petrilin | a3cdcbf | 2008-10-10 12:01:37 -0700 | [diff] [blame] | 118 | |
Eugenia Emantayev | ddae034 | 2014-12-11 10:57:54 +0200 | [diff] [blame] | 119 | obj = find_aligned_range(bitmap->table, bitmap->last, |
| 120 | bitmap->max, cnt, align, skip_mask); |
Yevgeny Petrilin | a3cdcbf | 2008-10-10 12:01:37 -0700 | [diff] [blame] | 121 | if (obj >= bitmap->max) { |
Yevgeny Petrilin | 93fc9e1 | 2008-10-22 10:25:29 -0700 | [diff] [blame] | 122 | bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
| 123 | & bitmap->mask; |
Eugenia Emantayev | ddae034 | 2014-12-11 10:57:54 +0200 | [diff] [blame] | 124 | obj = find_aligned_range(bitmap->table, 0, bitmap->max, |
| 125 | cnt, align, skip_mask); |
Yevgeny Petrilin | a3cdcbf | 2008-10-10 12:01:37 -0700 | [diff] [blame] | 126 | } |
| 127 | |
| 128 | if (obj < bitmap->max) { |
Akinobu Mita | e27cd4f | 2010-08-27 19:08:13 +0000 | [diff] [blame] | 129 | bitmap_set(bitmap->table, obj, cnt); |
Yevgeny Petrilin | a3cdcbf | 2008-10-10 12:01:37 -0700 | [diff] [blame] | 130 | if (obj == bitmap->last) { |
| 131 | bitmap->last = (obj + cnt); |
| 132 | if (bitmap->last >= bitmap->max) |
| 133 | bitmap->last = 0; |
| 134 | } |
| 135 | obj |= bitmap->top; |
| 136 | } else |
| 137 | obj = -1; |
| 138 | |
Eli Cohen | 42d1e01 | 2011-03-22 22:38:45 +0000 | [diff] [blame] | 139 | if (obj != -1) |
| 140 | bitmap->avail -= cnt; |
| 141 | |
Yevgeny Petrilin | a3cdcbf | 2008-10-10 12:01:37 -0700 | [diff] [blame] | 142 | spin_unlock(&bitmap->lock); |
| 143 | |
| 144 | return obj; |
| 145 | } |
| 146 | |
Eli Cohen | 42d1e01 | 2011-03-22 22:38:45 +0000 | [diff] [blame] | 147 | u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) |
| 148 | { |
| 149 | return bitmap->avail; |
| 150 | } |
| 151 | |
Matan Barak | 7a89399 | 2014-12-11 10:57:56 +0200 | [diff] [blame] | 152 | static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj) |
| 153 | { |
| 154 | return obj & (bitmap->max + bitmap->reserved_top - 1); |
| 155 | } |
| 156 | |
Jack Morgenstein | 7c6d74d | 2013-12-08 16:50:17 +0200 | [diff] [blame] | 157 | void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, |
| 158 | int use_rr) |
Yevgeny Petrilin | a3cdcbf | 2008-10-10 12:01:37 -0700 | [diff] [blame] | 159 | { |
Yevgeny Petrilin | 93fc9e1 | 2008-10-22 10:25:29 -0700 | [diff] [blame] | 160 | obj &= bitmap->max + bitmap->reserved_top - 1; |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 161 | |
| 162 | spin_lock(&bitmap->lock); |
Jack Morgenstein | 7c6d74d | 2013-12-08 16:50:17 +0200 | [diff] [blame] | 163 | if (!use_rr) { |
| 164 | bitmap->last = min(bitmap->last, obj); |
| 165 | bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
| 166 | & bitmap->mask; |
| 167 | } |
Akinobu Mita | e27cd4f | 2010-08-27 19:08:13 +0000 | [diff] [blame] | 168 | bitmap_clear(bitmap->table, obj, cnt); |
Eli Cohen | 42d1e01 | 2011-03-22 22:38:45 +0000 | [diff] [blame] | 169 | bitmap->avail += cnt; |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 170 | spin_unlock(&bitmap->lock); |
| 171 | } |
| 172 | |
Yevgeny Petrilin | 93fc9e1 | 2008-10-22 10:25:29 -0700 | [diff] [blame] | 173 | int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, |
| 174 | u32 reserved_bot, u32 reserved_top) |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 175 | { |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 176 | /* num must be a power of 2 */ |
| 177 | if (num != roundup_pow_of_two(num)) |
| 178 | return -EINVAL; |
| 179 | |
| 180 | bitmap->last = 0; |
| 181 | bitmap->top = 0; |
Yevgeny Petrilin | 93fc9e1 | 2008-10-22 10:25:29 -0700 | [diff] [blame] | 182 | bitmap->max = num - reserved_top; |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 183 | bitmap->mask = mask; |
Yevgeny Petrilin | 93fc9e1 | 2008-10-22 10:25:29 -0700 | [diff] [blame] | 184 | bitmap->reserved_top = reserved_top; |
Eli Cohen | 42d1e01 | 2011-03-22 22:38:45 +0000 | [diff] [blame] | 185 | bitmap->avail = num - reserved_top - reserved_bot; |
Matan Barak | 7a89399 | 2014-12-11 10:57:56 +0200 | [diff] [blame] | 186 | bitmap->effective_len = bitmap->avail; |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 187 | spin_lock_init(&bitmap->lock); |
Kees Cook | 6396bb2 | 2018-06-12 14:03:40 -0700 | [diff] [blame] | 188 | bitmap->table = kcalloc(BITS_TO_LONGS(bitmap->max), sizeof(long), |
| 189 | GFP_KERNEL); |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 190 | if (!bitmap->table) |
| 191 | return -ENOMEM; |
| 192 | |
Akinobu Mita | e27cd4f | 2010-08-27 19:08:13 +0000 | [diff] [blame] | 193 | bitmap_set(bitmap->table, 0, reserved_bot); |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 194 | |
| 195 | return 0; |
| 196 | } |
| 197 | |
| 198 | void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) |
| 199 | { |
| 200 | kfree(bitmap->table); |
| 201 | } |
| 202 | |
Matan Barak | 7a89399 | 2014-12-11 10:57:56 +0200 | [diff] [blame] | 203 | struct mlx4_zone_allocator { |
| 204 | struct list_head entries; |
| 205 | struct list_head prios; |
| 206 | u32 last_uid; |
| 207 | u32 mask; |
| 208 | /* protect the zone_allocator from concurrent accesses */ |
| 209 | spinlock_t lock; |
| 210 | enum mlx4_zone_alloc_flags flags; |
| 211 | }; |
| 212 | |
| 213 | struct mlx4_zone_entry { |
| 214 | struct list_head list; |
| 215 | struct list_head prio_list; |
| 216 | u32 uid; |
| 217 | struct mlx4_zone_allocator *allocator; |
| 218 | struct mlx4_bitmap *bitmap; |
| 219 | int use_rr; |
| 220 | int priority; |
| 221 | int offset; |
| 222 | enum mlx4_zone_flags flags; |
| 223 | }; |
| 224 | |
| 225 | struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags) |
| 226 | { |
| 227 | struct mlx4_zone_allocator *zones = kmalloc(sizeof(*zones), GFP_KERNEL); |
| 228 | |
| 229 | if (NULL == zones) |
| 230 | return NULL; |
| 231 | |
| 232 | INIT_LIST_HEAD(&zones->entries); |
| 233 | INIT_LIST_HEAD(&zones->prios); |
| 234 | spin_lock_init(&zones->lock); |
| 235 | zones->last_uid = 0; |
| 236 | zones->mask = 0; |
| 237 | zones->flags = flags; |
| 238 | |
| 239 | return zones; |
| 240 | } |
| 241 | |
| 242 | int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc, |
| 243 | struct mlx4_bitmap *bitmap, |
| 244 | u32 flags, |
| 245 | int priority, |
| 246 | int offset, |
| 247 | u32 *puid) |
| 248 | { |
| 249 | u32 mask = mlx4_bitmap_masked_value(bitmap, (u32)-1); |
| 250 | struct mlx4_zone_entry *it; |
| 251 | struct mlx4_zone_entry *zone = kmalloc(sizeof(*zone), GFP_KERNEL); |
| 252 | |
| 253 | if (NULL == zone) |
| 254 | return -ENOMEM; |
| 255 | |
| 256 | zone->flags = flags; |
| 257 | zone->bitmap = bitmap; |
| 258 | zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0; |
| 259 | zone->priority = priority; |
| 260 | zone->offset = offset; |
| 261 | |
| 262 | spin_lock(&zone_alloc->lock); |
| 263 | |
| 264 | zone->uid = zone_alloc->last_uid++; |
| 265 | zone->allocator = zone_alloc; |
| 266 | |
| 267 | if (zone_alloc->mask < mask) |
| 268 | zone_alloc->mask = mask; |
| 269 | |
| 270 | list_for_each_entry(it, &zone_alloc->prios, prio_list) |
| 271 | if (it->priority >= priority) |
| 272 | break; |
| 273 | |
| 274 | if (&it->prio_list == &zone_alloc->prios || it->priority > priority) |
| 275 | list_add_tail(&zone->prio_list, &it->prio_list); |
| 276 | list_add_tail(&zone->list, &it->list); |
| 277 | |
| 278 | spin_unlock(&zone_alloc->lock); |
| 279 | |
| 280 | *puid = zone->uid; |
| 281 | |
| 282 | return 0; |
| 283 | } |
| 284 | |
| 285 | /* Should be called under a lock */ |
Zhu Yanjun | e36fef6 | 2017-07-13 23:01:27 -0400 | [diff] [blame] | 286 | static void __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry) |
Matan Barak | 7a89399 | 2014-12-11 10:57:56 +0200 | [diff] [blame] | 287 | { |
| 288 | struct mlx4_zone_allocator *zone_alloc = entry->allocator; |
| 289 | |
| 290 | if (!list_empty(&entry->prio_list)) { |
| 291 | /* Check if we need to add an alternative node to the prio list */ |
| 292 | if (!list_is_last(&entry->list, &zone_alloc->entries)) { |
| 293 | struct mlx4_zone_entry *next = list_first_entry(&entry->list, |
| 294 | typeof(*next), |
| 295 | list); |
| 296 | |
| 297 | if (next->priority == entry->priority) |
| 298 | list_add_tail(&next->prio_list, &entry->prio_list); |
| 299 | } |
| 300 | |
| 301 | list_del(&entry->prio_list); |
| 302 | } |
| 303 | |
| 304 | list_del(&entry->list); |
| 305 | |
| 306 | if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) { |
| 307 | u32 mask = 0; |
| 308 | struct mlx4_zone_entry *it; |
| 309 | |
| 310 | list_for_each_entry(it, &zone_alloc->prios, prio_list) { |
| 311 | u32 cur_mask = mlx4_bitmap_masked_value(it->bitmap, (u32)-1); |
| 312 | |
| 313 | if (mask < cur_mask) |
| 314 | mask = cur_mask; |
| 315 | } |
| 316 | zone_alloc->mask = mask; |
| 317 | } |
Matan Barak | 7a89399 | 2014-12-11 10:57:56 +0200 | [diff] [blame] | 318 | } |
| 319 | |
| 320 | void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc) |
| 321 | { |
| 322 | struct mlx4_zone_entry *zone, *tmp; |
| 323 | |
| 324 | spin_lock(&zone_alloc->lock); |
| 325 | |
| 326 | list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) { |
| 327 | list_del(&zone->list); |
| 328 | list_del(&zone->prio_list); |
| 329 | kfree(zone); |
| 330 | } |
| 331 | |
| 332 | spin_unlock(&zone_alloc->lock); |
| 333 | kfree(zone_alloc); |
| 334 | } |
| 335 | |
| 336 | /* Should be called under a lock */ |
| 337 | static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count, |
| 338 | int align, u32 skip_mask, u32 *puid) |
| 339 | { |
Tariq Toukan | 3ea7e7e | 2018-11-15 18:05:14 +0200 | [diff] [blame] | 340 | u32 uid = 0; |
Matan Barak | 7a89399 | 2014-12-11 10:57:56 +0200 | [diff] [blame] | 341 | u32 res; |
| 342 | struct mlx4_zone_allocator *zone_alloc = zone->allocator; |
| 343 | struct mlx4_zone_entry *curr_node; |
| 344 | |
| 345 | res = mlx4_bitmap_alloc_range(zone->bitmap, count, |
| 346 | align, skip_mask); |
| 347 | |
| 348 | if (res != (u32)-1) { |
| 349 | res += zone->offset; |
| 350 | uid = zone->uid; |
| 351 | goto out; |
| 352 | } |
| 353 | |
| 354 | list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) { |
| 355 | if (unlikely(curr_node->priority == zone->priority)) |
| 356 | break; |
| 357 | } |
| 358 | |
| 359 | if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) { |
| 360 | struct mlx4_zone_entry *it = curr_node; |
| 361 | |
| 362 | list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) { |
| 363 | res = mlx4_bitmap_alloc_range(it->bitmap, count, |
| 364 | align, skip_mask); |
| 365 | if (res != (u32)-1) { |
| 366 | res += it->offset; |
| 367 | uid = it->uid; |
| 368 | goto out; |
| 369 | } |
| 370 | } |
| 371 | } |
| 372 | |
| 373 | if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) { |
| 374 | struct mlx4_zone_entry *it = curr_node; |
| 375 | |
| 376 | list_for_each_entry_from(it, &zone_alloc->entries, list) { |
| 377 | if (unlikely(it == zone)) |
| 378 | continue; |
| 379 | |
| 380 | if (unlikely(it->priority != curr_node->priority)) |
| 381 | break; |
| 382 | |
| 383 | res = mlx4_bitmap_alloc_range(it->bitmap, count, |
| 384 | align, skip_mask); |
| 385 | if (res != (u32)-1) { |
| 386 | res += it->offset; |
| 387 | uid = it->uid; |
| 388 | goto out; |
| 389 | } |
| 390 | } |
| 391 | } |
| 392 | |
| 393 | if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) { |
| 394 | if (list_is_last(&curr_node->prio_list, &zone_alloc->prios)) |
| 395 | goto out; |
| 396 | |
| 397 | curr_node = list_first_entry(&curr_node->prio_list, |
| 398 | typeof(*curr_node), |
| 399 | prio_list); |
| 400 | |
| 401 | list_for_each_entry_from(curr_node, &zone_alloc->entries, list) { |
| 402 | res = mlx4_bitmap_alloc_range(curr_node->bitmap, count, |
| 403 | align, skip_mask); |
| 404 | if (res != (u32)-1) { |
| 405 | res += curr_node->offset; |
| 406 | uid = curr_node->uid; |
| 407 | goto out; |
| 408 | } |
| 409 | } |
| 410 | } |
| 411 | |
| 412 | out: |
| 413 | if (NULL != puid && res != (u32)-1) |
| 414 | *puid = uid; |
| 415 | return res; |
| 416 | } |
| 417 | |
| 418 | /* Should be called under a lock */ |
| 419 | static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj, |
| 420 | u32 count) |
| 421 | { |
| 422 | mlx4_bitmap_free_range(zone->bitmap, obj - zone->offset, count, zone->use_rr); |
| 423 | } |
| 424 | |
| 425 | /* Should be called under a lock */ |
| 426 | static struct mlx4_zone_entry *__mlx4_find_zone_by_uid( |
| 427 | struct mlx4_zone_allocator *zones, u32 uid) |
| 428 | { |
| 429 | struct mlx4_zone_entry *zone; |
| 430 | |
| 431 | list_for_each_entry(zone, &zones->entries, list) { |
| 432 | if (zone->uid == uid) |
| 433 | return zone; |
| 434 | } |
| 435 | |
| 436 | return NULL; |
| 437 | } |
| 438 | |
| 439 | struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid) |
| 440 | { |
| 441 | struct mlx4_zone_entry *zone; |
| 442 | struct mlx4_bitmap *bitmap; |
| 443 | |
| 444 | spin_lock(&zones->lock); |
| 445 | |
| 446 | zone = __mlx4_find_zone_by_uid(zones, uid); |
| 447 | |
| 448 | bitmap = zone == NULL ? NULL : zone->bitmap; |
| 449 | |
| 450 | spin_unlock(&zones->lock); |
| 451 | |
| 452 | return bitmap; |
| 453 | } |
| 454 | |
| 455 | int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid) |
| 456 | { |
| 457 | struct mlx4_zone_entry *zone; |
Zhu Yanjun | e36fef6 | 2017-07-13 23:01:27 -0400 | [diff] [blame] | 458 | int res = 0; |
Matan Barak | 7a89399 | 2014-12-11 10:57:56 +0200 | [diff] [blame] | 459 | |
| 460 | spin_lock(&zones->lock); |
| 461 | |
| 462 | zone = __mlx4_find_zone_by_uid(zones, uid); |
| 463 | |
| 464 | if (NULL == zone) { |
| 465 | res = -1; |
| 466 | goto out; |
| 467 | } |
| 468 | |
Zhu Yanjun | e36fef6 | 2017-07-13 23:01:27 -0400 | [diff] [blame] | 469 | __mlx4_zone_remove_one_entry(zone); |
Matan Barak | 7a89399 | 2014-12-11 10:57:56 +0200 | [diff] [blame] | 470 | |
| 471 | out: |
| 472 | spin_unlock(&zones->lock); |
| 473 | kfree(zone); |
| 474 | |
| 475 | return res; |
| 476 | } |
| 477 | |
| 478 | /* Should be called under a lock */ |
| 479 | static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique( |
| 480 | struct mlx4_zone_allocator *zones, u32 obj) |
| 481 | { |
| 482 | struct mlx4_zone_entry *zone, *zone_candidate = NULL; |
| 483 | u32 dist = (u32)-1; |
| 484 | |
| 485 | /* Search for the smallest zone that this obj could be |
| 486 | * allocated from. This is done in order to handle |
| 487 | * situations when small bitmaps are allocated from bigger |
| 488 | * bitmaps (and the allocated space is marked as reserved in |
| 489 | * the bigger bitmap. |
| 490 | */ |
| 491 | list_for_each_entry(zone, &zones->entries, list) { |
| 492 | if (obj >= zone->offset) { |
| 493 | u32 mobj = (obj - zone->offset) & zones->mask; |
| 494 | |
| 495 | if (mobj < zone->bitmap->max) { |
| 496 | u32 curr_dist = zone->bitmap->effective_len; |
| 497 | |
| 498 | if (curr_dist < dist) { |
| 499 | dist = curr_dist; |
| 500 | zone_candidate = zone; |
| 501 | } |
| 502 | } |
| 503 | } |
| 504 | } |
| 505 | |
| 506 | return zone_candidate; |
| 507 | } |
| 508 | |
| 509 | u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count, |
| 510 | int align, u32 skip_mask, u32 *puid) |
| 511 | { |
| 512 | struct mlx4_zone_entry *zone; |
| 513 | int res = -1; |
| 514 | |
| 515 | spin_lock(&zones->lock); |
| 516 | |
| 517 | zone = __mlx4_find_zone_by_uid(zones, uid); |
| 518 | |
| 519 | if (NULL == zone) |
| 520 | goto out; |
| 521 | |
| 522 | res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid); |
| 523 | |
| 524 | out: |
| 525 | spin_unlock(&zones->lock); |
| 526 | |
| 527 | return res; |
| 528 | } |
| 529 | |
| 530 | u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count) |
| 531 | { |
| 532 | struct mlx4_zone_entry *zone; |
| 533 | int res = 0; |
| 534 | |
| 535 | spin_lock(&zones->lock); |
| 536 | |
| 537 | zone = __mlx4_find_zone_by_uid(zones, uid); |
| 538 | |
| 539 | if (NULL == zone) { |
| 540 | res = -1; |
| 541 | goto out; |
| 542 | } |
| 543 | |
| 544 | __mlx4_free_from_zone(zone, obj, count); |
| 545 | |
| 546 | out: |
| 547 | spin_unlock(&zones->lock); |
| 548 | |
| 549 | return res; |
| 550 | } |
| 551 | |
| 552 | u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count) |
| 553 | { |
| 554 | struct mlx4_zone_entry *zone; |
| 555 | int res; |
| 556 | |
| 557 | if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP)) |
| 558 | return -EFAULT; |
| 559 | |
| 560 | spin_lock(&zones->lock); |
| 561 | |
| 562 | zone = __mlx4_find_zone_by_uid_unique(zones, obj); |
| 563 | |
| 564 | if (NULL == zone) { |
| 565 | res = -1; |
| 566 | goto out; |
| 567 | } |
| 568 | |
| 569 | __mlx4_free_from_zone(zone, obj, count); |
| 570 | res = 0; |
| 571 | |
| 572 | out: |
| 573 | spin_unlock(&zones->lock); |
| 574 | |
| 575 | return res; |
| 576 | } |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 577 | |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 578 | static int mlx4_buf_direct_alloc(struct mlx4_dev *dev, int size, |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 579 | struct mlx4_buf *buf) |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 580 | { |
| 581 | dma_addr_t t; |
| 582 | |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 583 | buf->nbufs = 1; |
| 584 | buf->npages = 1; |
| 585 | buf->page_shift = get_order(size) + PAGE_SHIFT; |
| 586 | buf->direct.buf = |
| 587 | dma_zalloc_coherent(&dev->persist->pdev->dev, |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 588 | size, &t, GFP_KERNEL); |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 589 | if (!buf->direct.buf) |
| 590 | return -ENOMEM; |
| 591 | |
| 592 | buf->direct.map = t; |
| 593 | |
| 594 | while (t & ((1 << buf->page_shift) - 1)) { |
| 595 | --buf->page_shift; |
| 596 | buf->npages *= 2; |
| 597 | } |
| 598 | |
| 599 | return 0; |
| 600 | } |
| 601 | |
| 602 | /* Handling for queue buffers -- we allocate a bunch of memory and |
| 603 | * register it in a memory region at HCA virtual address 0. If the |
| 604 | * requested size is > max_direct, we split the allocation into |
| 605 | * multiple pages, so we don't require too much contiguous memory. |
| 606 | */ |
| 607 | int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 608 | struct mlx4_buf *buf) |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 609 | { |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 610 | if (size <= max_direct) { |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 611 | return mlx4_buf_direct_alloc(dev, size, buf); |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 612 | } else { |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 613 | dma_addr_t t; |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 614 | int i; |
| 615 | |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 616 | buf->direct.buf = NULL; |
zhong jiang | f8a1988 | 2018-09-11 21:08:15 +0800 | [diff] [blame] | 617 | buf->nbufs = DIV_ROUND_UP(size, PAGE_SIZE); |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 618 | buf->npages = buf->nbufs; |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 619 | buf->page_shift = PAGE_SHIFT; |
Joe Perches | baeb2ff | 2010-08-11 07:02:48 +0000 | [diff] [blame] | 620 | buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 621 | GFP_KERNEL); |
Roland Dreier | b57aacf | 2008-02-06 21:17:59 -0800 | [diff] [blame] | 622 | if (!buf->page_list) |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 623 | return -ENOMEM; |
| 624 | |
| 625 | for (i = 0; i < buf->nbufs; ++i) { |
Roland Dreier | b57aacf | 2008-02-06 21:17:59 -0800 | [diff] [blame] | 626 | buf->page_list[i].buf = |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 627 | dma_zalloc_coherent(&dev->persist->pdev->dev, |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 628 | PAGE_SIZE, &t, GFP_KERNEL); |
Roland Dreier | b57aacf | 2008-02-06 21:17:59 -0800 | [diff] [blame] | 629 | if (!buf->page_list[i].buf) |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 630 | goto err_free; |
| 631 | |
Roland Dreier | b57aacf | 2008-02-06 21:17:59 -0800 | [diff] [blame] | 632 | buf->page_list[i].map = t; |
Jack Morgenstein | 313abe5 | 2008-01-28 10:40:51 +0200 | [diff] [blame] | 633 | } |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 634 | } |
| 635 | |
| 636 | return 0; |
| 637 | |
| 638 | err_free: |
| 639 | mlx4_buf_free(dev, size, buf); |
| 640 | |
| 641 | return -ENOMEM; |
| 642 | } |
| 643 | EXPORT_SYMBOL_GPL(mlx4_buf_alloc); |
| 644 | |
| 645 | void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) |
| 646 | { |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 647 | if (buf->nbufs == 1) { |
Yishai Hadas | 872bf2f | 2015-01-25 16:59:35 +0200 | [diff] [blame] | 648 | dma_free_coherent(&dev->persist->pdev->dev, size, |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 649 | buf->direct.buf, buf->direct.map); |
| 650 | } else { |
| 651 | int i; |
Jack Morgenstein | 313abe5 | 2008-01-28 10:40:51 +0200 | [diff] [blame] | 652 | |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 653 | for (i = 0; i < buf->nbufs; ++i) |
Roland Dreier | b57aacf | 2008-02-06 21:17:59 -0800 | [diff] [blame] | 654 | if (buf->page_list[i].buf) |
Yishai Hadas | 872bf2f | 2015-01-25 16:59:35 +0200 | [diff] [blame] | 655 | dma_free_coherent(&dev->persist->pdev->dev, |
| 656 | PAGE_SIZE, |
Roland Dreier | b57aacf | 2008-02-06 21:17:59 -0800 | [diff] [blame] | 657 | buf->page_list[i].buf, |
| 658 | buf->page_list[i].map); |
| 659 | kfree(buf->page_list); |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 660 | } |
| 661 | } |
| 662 | EXPORT_SYMBOL_GPL(mlx4_buf_free); |
Yevgeny Petrilin | 6296883 | 2008-04-23 11:55:45 -0700 | [diff] [blame] | 663 | |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 664 | static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device) |
Yevgeny Petrilin | 6296883 | 2008-04-23 11:55:45 -0700 | [diff] [blame] | 665 | { |
| 666 | struct mlx4_db_pgdir *pgdir; |
| 667 | |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 668 | pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); |
Yevgeny Petrilin | 6296883 | 2008-04-23 11:55:45 -0700 | [diff] [blame] | 669 | if (!pgdir) |
| 670 | return NULL; |
| 671 | |
| 672 | bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); |
| 673 | pgdir->bits[0] = pgdir->order0; |
| 674 | pgdir->bits[1] = pgdir->order1; |
| 675 | pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 676 | &pgdir->db_dma, GFP_KERNEL); |
Yevgeny Petrilin | 6296883 | 2008-04-23 11:55:45 -0700 | [diff] [blame] | 677 | if (!pgdir->db_page) { |
| 678 | kfree(pgdir); |
| 679 | return NULL; |
| 680 | } |
| 681 | |
| 682 | return pgdir; |
| 683 | } |
| 684 | |
| 685 | static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, |
| 686 | struct mlx4_db *db, int order) |
| 687 | { |
| 688 | int o; |
| 689 | int i; |
| 690 | |
| 691 | for (o = order; o <= 1; ++o) { |
| 692 | i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); |
| 693 | if (i < MLX4_DB_PER_PAGE >> o) |
| 694 | goto found; |
| 695 | } |
| 696 | |
| 697 | return -ENOMEM; |
| 698 | |
| 699 | found: |
| 700 | clear_bit(i, pgdir->bits[o]); |
| 701 | |
| 702 | i <<= o; |
| 703 | |
| 704 | if (o > order) |
| 705 | set_bit(i ^ 1, pgdir->bits[order]); |
| 706 | |
| 707 | db->u.pgdir = pgdir; |
| 708 | db->index = i; |
| 709 | db->db = pgdir->db_page + db->index; |
| 710 | db->dma = pgdir->db_dma + db->index * 4; |
| 711 | db->order = order; |
| 712 | |
| 713 | return 0; |
| 714 | } |
| 715 | |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 716 | int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) |
Yevgeny Petrilin | 6296883 | 2008-04-23 11:55:45 -0700 | [diff] [blame] | 717 | { |
| 718 | struct mlx4_priv *priv = mlx4_priv(dev); |
| 719 | struct mlx4_db_pgdir *pgdir; |
| 720 | int ret = 0; |
| 721 | |
| 722 | mutex_lock(&priv->pgdir_mutex); |
| 723 | |
| 724 | list_for_each_entry(pgdir, &priv->pgdir_list, list) |
| 725 | if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) |
| 726 | goto out; |
| 727 | |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 728 | pgdir = mlx4_alloc_db_pgdir(&dev->persist->pdev->dev); |
Yevgeny Petrilin | 6296883 | 2008-04-23 11:55:45 -0700 | [diff] [blame] | 729 | if (!pgdir) { |
| 730 | ret = -ENOMEM; |
| 731 | goto out; |
| 732 | } |
| 733 | |
| 734 | list_add(&pgdir->list, &priv->pgdir_list); |
| 735 | |
| 736 | /* This should never fail -- we just allocated an empty page: */ |
| 737 | WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); |
| 738 | |
| 739 | out: |
| 740 | mutex_unlock(&priv->pgdir_mutex); |
| 741 | |
| 742 | return ret; |
| 743 | } |
| 744 | EXPORT_SYMBOL_GPL(mlx4_db_alloc); |
| 745 | |
| 746 | void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) |
| 747 | { |
| 748 | struct mlx4_priv *priv = mlx4_priv(dev); |
| 749 | int o; |
| 750 | int i; |
| 751 | |
| 752 | mutex_lock(&priv->pgdir_mutex); |
| 753 | |
| 754 | o = db->order; |
| 755 | i = db->index; |
| 756 | |
| 757 | if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { |
| 758 | clear_bit(i ^ 1, db->u.pgdir->order0); |
| 759 | ++o; |
| 760 | } |
| 761 | i >>= o; |
| 762 | set_bit(i, db->u.pgdir->bits[o]); |
| 763 | |
| 764 | if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { |
Yishai Hadas | 872bf2f | 2015-01-25 16:59:35 +0200 | [diff] [blame] | 765 | dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, |
Yevgeny Petrilin | 6296883 | 2008-04-23 11:55:45 -0700 | [diff] [blame] | 766 | db->u.pgdir->db_page, db->u.pgdir->db_dma); |
| 767 | list_del(&db->u.pgdir->list); |
| 768 | kfree(db->u.pgdir); |
| 769 | } |
| 770 | |
| 771 | mutex_unlock(&priv->pgdir_mutex); |
| 772 | } |
| 773 | EXPORT_SYMBOL_GPL(mlx4_db_free); |
Yevgeny Petrilin | 38ae6a5 | 2008-04-25 14:27:08 -0700 | [diff] [blame] | 774 | |
| 775 | int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, |
Haggai Abramovsky | 73898db | 2016-05-04 14:50:15 +0300 | [diff] [blame] | 776 | int size) |
Yevgeny Petrilin | 38ae6a5 | 2008-04-25 14:27:08 -0700 | [diff] [blame] | 777 | { |
| 778 | int err; |
| 779 | |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 780 | err = mlx4_db_alloc(dev, &wqres->db, 1); |
Yevgeny Petrilin | 38ae6a5 | 2008-04-25 14:27:08 -0700 | [diff] [blame] | 781 | if (err) |
| 782 | return err; |
| 783 | |
| 784 | *wqres->db.db = 0; |
| 785 | |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 786 | err = mlx4_buf_direct_alloc(dev, size, &wqres->buf); |
Yevgeny Petrilin | 38ae6a5 | 2008-04-25 14:27:08 -0700 | [diff] [blame] | 787 | if (err) |
| 788 | goto err_db; |
| 789 | |
| 790 | err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, |
| 791 | &wqres->mtt); |
| 792 | if (err) |
| 793 | goto err_buf; |
| 794 | |
Leon Romanovsky | 8900b89 | 2017-05-23 14:38:15 +0300 | [diff] [blame] | 795 | err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf); |
Yevgeny Petrilin | 38ae6a5 | 2008-04-25 14:27:08 -0700 | [diff] [blame] | 796 | if (err) |
| 797 | goto err_mtt; |
| 798 | |
| 799 | return 0; |
| 800 | |
| 801 | err_mtt: |
| 802 | mlx4_mtt_cleanup(dev, &wqres->mtt); |
| 803 | err_buf: |
| 804 | mlx4_buf_free(dev, size, &wqres->buf); |
| 805 | err_db: |
| 806 | mlx4_db_free(dev, &wqres->db); |
| 807 | |
| 808 | return err; |
| 809 | } |
| 810 | EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); |
| 811 | |
| 812 | void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, |
| 813 | int size) |
| 814 | { |
| 815 | mlx4_mtt_cleanup(dev, &wqres->mtt); |
| 816 | mlx4_buf_free(dev, size, &wqres->buf); |
| 817 | mlx4_db_free(dev, &wqres->db); |
| 818 | } |
| 819 | EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); |