blob: da2a14d096d29cabbbd2d551da4989047061a08b [file] [log] [blame]
Dan Williams51cf7842017-07-12 17:58:21 -07001// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */
Dan Williams89ec9f22018-10-29 15:52:42 -07003#include <linux/memremap.h>
Dan Williams51cf7842017-07-12 17:58:21 -07004#include <linux/device.h>
Dan Williamsd2007812018-11-07 15:31:23 -08005#include <linux/mutex.h>
6#include <linux/list.h>
Dan Williams51cf7842017-07-12 17:58:21 -07007#include <linux/slab.h>
8#include <linux/dax.h>
Dan Williamsfcffb6a2020-10-13 16:50:24 -07009#include <linux/io.h>
Dan Williams51cf7842017-07-12 17:58:21 -070010#include "dax-private.h"
11#include "bus.h"
12
Dan Williamsd2007812018-11-07 15:31:23 -080013static DEFINE_MUTEX(dax_bus_lock);
14
15#define DAX_NAME_LEN 30
16struct dax_id {
17 struct list_head list;
18 char dev_name[DAX_NAME_LEN];
19};
20
Dan Williams9567da02017-07-12 17:58:21 -070021static int dax_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
22{
23 /*
24 * We only ever expect to handle device-dax instances, i.e. the
25 * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
26 */
27 return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
28}
29
Dan Williamsd2007812018-11-07 15:31:23 -080030static struct dax_device_driver *to_dax_drv(struct device_driver *drv)
31{
32 return container_of(drv, struct dax_device_driver, drv);
33}
34
35static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv,
36 const char *dev_name)
37{
38 struct dax_id *dax_id;
39
40 lockdep_assert_held(&dax_bus_lock);
41
42 list_for_each_entry(dax_id, &dax_drv->ids, list)
43 if (sysfs_streq(dax_id->dev_name, dev_name))
44 return dax_id;
45 return NULL;
46}
47
48static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev)
49{
50 int match;
51
52 mutex_lock(&dax_bus_lock);
53 match = !!__dax_match_id(dax_drv, dev_name(dev));
54 mutex_unlock(&dax_bus_lock);
55
56 return match;
57}
58
Dan Williams664525b2019-01-24 13:12:04 -080059enum id_action {
60 ID_REMOVE,
61 ID_ADD,
62};
63
Dan Williamsd2007812018-11-07 15:31:23 -080064static ssize_t do_id_store(struct device_driver *drv, const char *buf,
Dan Williams664525b2019-01-24 13:12:04 -080065 size_t count, enum id_action action)
Dan Williamsd2007812018-11-07 15:31:23 -080066{
67 struct dax_device_driver *dax_drv = to_dax_drv(drv);
68 unsigned int region_id, id;
69 char devname[DAX_NAME_LEN];
70 struct dax_id *dax_id;
71 ssize_t rc = count;
72 int fields;
73
74 fields = sscanf(buf, "dax%d.%d", &region_id, &id);
75 if (fields != 2)
76 return -EINVAL;
77 sprintf(devname, "dax%d.%d", region_id, id);
78 if (!sysfs_streq(buf, devname))
79 return -EINVAL;
80
81 mutex_lock(&dax_bus_lock);
82 dax_id = __dax_match_id(dax_drv, buf);
83 if (!dax_id) {
Dan Williams664525b2019-01-24 13:12:04 -080084 if (action == ID_ADD) {
Dan Williamsd2007812018-11-07 15:31:23 -080085 dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL);
86 if (dax_id) {
87 strncpy(dax_id->dev_name, buf, DAX_NAME_LEN);
88 list_add(&dax_id->list, &dax_drv->ids);
89 } else
90 rc = -ENOMEM;
Arnd Bergmann9de82ca2021-03-22 12:44:58 +010091 }
Dan Williams664525b2019-01-24 13:12:04 -080092 } else if (action == ID_REMOVE) {
Dan Williamsd2007812018-11-07 15:31:23 -080093 list_del(&dax_id->list);
94 kfree(dax_id);
Arnd Bergmann9de82ca2021-03-22 12:44:58 +010095 }
Dan Williamsd2007812018-11-07 15:31:23 -080096 mutex_unlock(&dax_bus_lock);
Dan Williams664525b2019-01-24 13:12:04 -080097
98 if (rc < 0)
99 return rc;
100 if (action == ID_ADD)
101 rc = driver_attach(drv);
102 if (rc)
103 return rc;
104 return count;
Dan Williamsd2007812018-11-07 15:31:23 -0800105}
106
107static ssize_t new_id_store(struct device_driver *drv, const char *buf,
108 size_t count)
109{
Dan Williams664525b2019-01-24 13:12:04 -0800110 return do_id_store(drv, buf, count, ID_ADD);
Dan Williamsd2007812018-11-07 15:31:23 -0800111}
112static DRIVER_ATTR_WO(new_id);
113
114static ssize_t remove_id_store(struct device_driver *drv, const char *buf,
115 size_t count)
116{
Dan Williams664525b2019-01-24 13:12:04 -0800117 return do_id_store(drv, buf, count, ID_REMOVE);
Dan Williamsd2007812018-11-07 15:31:23 -0800118}
119static DRIVER_ATTR_WO(remove_id);
120
121static struct attribute *dax_drv_attrs[] = {
122 &driver_attr_new_id.attr,
123 &driver_attr_remove_id.attr,
124 NULL,
125};
126ATTRIBUTE_GROUPS(dax_drv);
127
Dan Williams9567da02017-07-12 17:58:21 -0700128static int dax_bus_match(struct device *dev, struct device_driver *drv);
129
Dan Williamsc2f30112020-10-13 16:50:03 -0700130static bool is_static(struct dax_region *dax_region)
131{
132 return (dax_region->res.flags & IORESOURCE_DAX_STATIC) != 0;
133}
134
Dan Williams60e93dc2020-10-13 16:50:39 -0700135static u64 dev_dax_size(struct dev_dax *dev_dax)
136{
137 u64 size = 0;
138 int i;
139
140 device_lock_assert(&dev_dax->dev);
141
142 for (i = 0; i < dev_dax->nr_range; i++)
143 size += range_len(&dev_dax->ranges[i].range);
144
145 return size;
146}
147
Dan Williamsf11cf812020-10-13 16:50:08 -0700148static int dax_bus_probe(struct device *dev)
149{
150 struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
151 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williams0f3da142020-10-13 16:50:13 -0700152 struct dax_region *dax_region = dev_dax->region;
Dan Williams0f3da142020-10-13 16:50:13 -0700153 int rc;
Dan Williamsf11cf812020-10-13 16:50:08 -0700154
Dan Williams60e93dc2020-10-13 16:50:39 -0700155 if (dev_dax_size(dev_dax) == 0 || dev_dax->id < 0)
Dan Williams0f3da142020-10-13 16:50:13 -0700156 return -ENXIO;
157
158 rc = dax_drv->probe(dev_dax);
159
160 if (rc || is_static(dax_region))
161 return rc;
162
163 /*
164 * Track new seed creation only after successful probe of the
165 * previous seed.
166 */
167 if (dax_region->seed == dev)
168 dax_region->seed = NULL;
169
170 return 0;
Dan Williamsf11cf812020-10-13 16:50:08 -0700171}
172
Uwe Kleine-Königfc7a6202021-07-13 21:35:22 +0200173static void dax_bus_remove(struct device *dev)
Dan Williamsf11cf812020-10-13 16:50:08 -0700174{
175 struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
176 struct dev_dax *dev_dax = to_dev_dax(dev);
177
Uwe Kleine-König80299682021-02-05 23:28:39 +0100178 if (dax_drv->remove)
Uwe Kleine-König0d519e02021-02-05 23:28:42 +0100179 dax_drv->remove(dev_dax);
Dan Williamsf11cf812020-10-13 16:50:08 -0700180}
181
Dan Williams9567da02017-07-12 17:58:21 -0700182static struct bus_type dax_bus_type = {
183 .name = "dax",
184 .uevent = dax_bus_uevent,
185 .match = dax_bus_match,
Dan Williamsf11cf812020-10-13 16:50:08 -0700186 .probe = dax_bus_probe,
187 .remove = dax_bus_remove,
Dan Williamsd2007812018-11-07 15:31:23 -0800188 .drv_groups = dax_drv_groups,
Dan Williams9567da02017-07-12 17:58:21 -0700189};
190
191static int dax_bus_match(struct device *dev, struct device_driver *drv)
192{
Dan Williamsd2007812018-11-07 15:31:23 -0800193 struct dax_device_driver *dax_drv = to_dax_drv(drv);
194
Dan Williams9567da02017-07-12 17:58:21 -0700195 /*
Dan Williamsd2007812018-11-07 15:31:23 -0800196 * All but the 'device-dax' driver, which has 'match_always'
197 * set, requires an exact id match.
Dan Williams9567da02017-07-12 17:58:21 -0700198 */
Dan Williamsd2007812018-11-07 15:31:23 -0800199 if (dax_drv->match_always)
200 return 1;
201
202 return dax_match_id(dax_drv, dev);
Dan Williams9567da02017-07-12 17:58:21 -0700203}
204
Dan Williams51cf7842017-07-12 17:58:21 -0700205/*
206 * Rely on the fact that drvdata is set before the attributes are
207 * registered, and that the attributes are unregistered before drvdata
208 * is cleared to assume that drvdata is always valid.
209 */
210static ssize_t id_show(struct device *dev,
211 struct device_attribute *attr, char *buf)
212{
213 struct dax_region *dax_region = dev_get_drvdata(dev);
214
215 return sprintf(buf, "%d\n", dax_region->id);
216}
217static DEVICE_ATTR_RO(id);
218
219static ssize_t region_size_show(struct device *dev,
220 struct device_attribute *attr, char *buf)
221{
222 struct dax_region *dax_region = dev_get_drvdata(dev);
223
224 return sprintf(buf, "%llu\n", (unsigned long long)
225 resource_size(&dax_region->res));
226}
227static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
228 region_size_show, NULL);
229
Dan Williams6d821202020-10-13 16:50:55 -0700230static ssize_t region_align_show(struct device *dev,
Dan Williams51cf7842017-07-12 17:58:21 -0700231 struct device_attribute *attr, char *buf)
232{
233 struct dax_region *dax_region = dev_get_drvdata(dev);
234
235 return sprintf(buf, "%u\n", dax_region->align);
236}
Dan Williams6d821202020-10-13 16:50:55 -0700237static struct device_attribute dev_attr_region_align =
238 __ATTR(align, 0400, region_align_show, NULL);
Dan Williams51cf7842017-07-12 17:58:21 -0700239
Dan Williamsc2f30112020-10-13 16:50:03 -0700240#define for_each_dax_region_resource(dax_region, res) \
241 for (res = (dax_region)->res.child; res; res = res->sibling)
242
243static unsigned long long dax_region_avail_size(struct dax_region *dax_region)
244{
245 resource_size_t size = resource_size(&dax_region->res);
246 struct resource *res;
247
248 device_lock_assert(dax_region->dev);
249
250 for_each_dax_region_resource(dax_region, res)
251 size -= resource_size(res);
252 return size;
253}
254
255static ssize_t available_size_show(struct device *dev,
256 struct device_attribute *attr, char *buf)
257{
258 struct dax_region *dax_region = dev_get_drvdata(dev);
259 unsigned long long size;
260
261 device_lock(dev);
262 size = dax_region_avail_size(dax_region);
263 device_unlock(dev);
264
265 return sprintf(buf, "%llu\n", size);
266}
267static DEVICE_ATTR_RO(available_size);
268
Dan Williams0f3da142020-10-13 16:50:13 -0700269static ssize_t seed_show(struct device *dev,
270 struct device_attribute *attr, char *buf)
271{
272 struct dax_region *dax_region = dev_get_drvdata(dev);
273 struct device *seed;
274 ssize_t rc;
275
276 if (is_static(dax_region))
277 return -EINVAL;
278
279 device_lock(dev);
280 seed = dax_region->seed;
281 rc = sprintf(buf, "%s\n", seed ? dev_name(seed) : "");
282 device_unlock(dev);
283
284 return rc;
285}
286static DEVICE_ATTR_RO(seed);
287
288static ssize_t create_show(struct device *dev,
289 struct device_attribute *attr, char *buf)
290{
291 struct dax_region *dax_region = dev_get_drvdata(dev);
292 struct device *youngest;
293 ssize_t rc;
294
295 if (is_static(dax_region))
296 return -EINVAL;
297
298 device_lock(dev);
299 youngest = dax_region->youngest;
300 rc = sprintf(buf, "%s\n", youngest ? dev_name(youngest) : "");
301 device_unlock(dev);
302
303 return rc;
304}
305
306static ssize_t create_store(struct device *dev, struct device_attribute *attr,
307 const char *buf, size_t len)
308{
309 struct dax_region *dax_region = dev_get_drvdata(dev);
310 unsigned long long avail;
311 ssize_t rc;
312 int val;
313
314 if (is_static(dax_region))
315 return -EINVAL;
316
317 rc = kstrtoint(buf, 0, &val);
318 if (rc)
319 return rc;
320 if (val != 1)
321 return -EINVAL;
322
323 device_lock(dev);
324 avail = dax_region_avail_size(dax_region);
325 if (avail == 0)
326 rc = -ENOSPC;
327 else {
328 struct dev_dax_data data = {
329 .dax_region = dax_region,
330 .size = 0,
331 .id = -1,
332 };
333 struct dev_dax *dev_dax = devm_create_dev_dax(&data);
334
335 if (IS_ERR(dev_dax))
336 rc = PTR_ERR(dev_dax);
337 else {
338 /*
339 * In support of crafting multiple new devices
340 * simultaneously multiple seeds can be created,
341 * but only the first one that has not been
342 * successfully bound is tracked as the region
343 * seed.
344 */
345 if (!dax_region->seed)
346 dax_region->seed = &dev_dax->dev;
347 dax_region->youngest = &dev_dax->dev;
348 rc = len;
349 }
350 }
351 device_unlock(dev);
352
353 return rc;
354}
355static DEVICE_ATTR_RW(create);
356
357void kill_dev_dax(struct dev_dax *dev_dax)
358{
359 struct dax_device *dax_dev = dev_dax->dax_dev;
360 struct inode *inode = dax_inode(dax_dev);
361
362 kill_dax(dax_dev);
363 unmap_mapping_range(inode->i_mapping, 0, 0, 1);
364}
365EXPORT_SYMBOL_GPL(kill_dev_dax);
366
Dan Williams6268d7d2020-12-18 18:41:41 -0800367static void trim_dev_dax_range(struct dev_dax *dev_dax)
Dan Williams0f3da142020-10-13 16:50:13 -0700368{
Dan Williams6268d7d2020-12-18 18:41:41 -0800369 int i = dev_dax->nr_range - 1;
370 struct range *range = &dev_dax->ranges[i].range;
Dan Williams0f3da142020-10-13 16:50:13 -0700371 struct dax_region *dax_region = dev_dax->region;
Dan Williams0f3da142020-10-13 16:50:13 -0700372
373 device_lock_assert(dax_region->dev);
Dan Williams6268d7d2020-12-18 18:41:41 -0800374 dev_dbg(&dev_dax->dev, "delete range[%d]: %#llx:%#llx\n", i,
375 (unsigned long long)range->start,
376 (unsigned long long)range->end);
Dan Williams60e93dc2020-10-13 16:50:39 -0700377
Dan Williams6268d7d2020-12-18 18:41:41 -0800378 __release_region(&dax_region->res, range->start, range_len(range));
379 if (--dev_dax->nr_range == 0) {
380 kfree(dev_dax->ranges);
381 dev_dax->ranges = NULL;
Dan Williams60e93dc2020-10-13 16:50:39 -0700382 }
Dan Williams6268d7d2020-12-18 18:41:41 -0800383}
384
385static void free_dev_dax_ranges(struct dev_dax *dev_dax)
386{
387 while (dev_dax->nr_range)
388 trim_dev_dax_range(dev_dax);
Dan Williams0f3da142020-10-13 16:50:13 -0700389}
390
391static void unregister_dev_dax(void *dev)
392{
393 struct dev_dax *dev_dax = to_dev_dax(dev);
394
395 dev_dbg(dev, "%s\n", __func__);
396
397 kill_dev_dax(dev_dax);
Dan Williams60e93dc2020-10-13 16:50:39 -0700398 free_dev_dax_ranges(dev_dax);
Dan Williams0f3da142020-10-13 16:50:13 -0700399 device_del(dev);
400 put_device(dev);
401}
402
403/* a return value >= 0 indicates this invocation invalidated the id */
404static int __free_dev_dax_id(struct dev_dax *dev_dax)
405{
406 struct dax_region *dax_region = dev_dax->region;
407 struct device *dev = &dev_dax->dev;
408 int rc = dev_dax->id;
409
410 device_lock_assert(dev);
411
412 if (is_static(dax_region) || dev_dax->id < 0)
413 return -1;
414 ida_free(&dax_region->ida, dev_dax->id);
415 dev_dax->id = -1;
416 return rc;
417}
418
419static int free_dev_dax_id(struct dev_dax *dev_dax)
420{
421 struct device *dev = &dev_dax->dev;
422 int rc;
423
424 device_lock(dev);
425 rc = __free_dev_dax_id(dev_dax);
426 device_unlock(dev);
427 return rc;
428}
429
430static ssize_t delete_store(struct device *dev, struct device_attribute *attr,
431 const char *buf, size_t len)
432{
433 struct dax_region *dax_region = dev_get_drvdata(dev);
434 struct dev_dax *dev_dax;
435 struct device *victim;
436 bool do_del = false;
437 int rc;
438
439 if (is_static(dax_region))
440 return -EINVAL;
441
442 victim = device_find_child_by_name(dax_region->dev, buf);
443 if (!victim)
444 return -ENXIO;
445
446 device_lock(dev);
447 device_lock(victim);
448 dev_dax = to_dev_dax(victim);
Dan Williams60e93dc2020-10-13 16:50:39 -0700449 if (victim->driver || dev_dax_size(dev_dax))
Dan Williams0f3da142020-10-13 16:50:13 -0700450 rc = -EBUSY;
451 else {
452 /*
453 * Invalidate the device so it does not become active
454 * again, but always preserve device-id-0 so that
455 * /sys/bus/dax/ is guaranteed to be populated while any
456 * dax_region is registered.
457 */
458 if (dev_dax->id > 0) {
459 do_del = __free_dev_dax_id(dev_dax) >= 0;
460 rc = len;
461 if (dax_region->seed == victim)
462 dax_region->seed = NULL;
463 if (dax_region->youngest == victim)
464 dax_region->youngest = NULL;
465 } else
466 rc = -EBUSY;
467 }
468 device_unlock(victim);
469
470 /* won the race to invalidate the device, clean it up */
471 if (do_del)
472 devm_release_action(dev, unregister_dev_dax, victim);
473 device_unlock(dev);
474 put_device(victim);
475
476 return rc;
477}
478static DEVICE_ATTR_WO(delete);
479
Dan Williamsc2f30112020-10-13 16:50:03 -0700480static umode_t dax_region_visible(struct kobject *kobj, struct attribute *a,
481 int n)
482{
483 struct device *dev = container_of(kobj, struct device, kobj);
484 struct dax_region *dax_region = dev_get_drvdata(dev);
485
Dan Williams0f3da142020-10-13 16:50:13 -0700486 if (is_static(dax_region))
487 if (a == &dev_attr_available_size.attr
488 || a == &dev_attr_create.attr
489 || a == &dev_attr_seed.attr
490 || a == &dev_attr_delete.attr)
491 return 0;
Dan Williamsc2f30112020-10-13 16:50:03 -0700492 return a->mode;
493}
494
Dan Williams51cf7842017-07-12 17:58:21 -0700495static struct attribute *dax_region_attributes[] = {
Dan Williamsc2f30112020-10-13 16:50:03 -0700496 &dev_attr_available_size.attr,
Dan Williams51cf7842017-07-12 17:58:21 -0700497 &dev_attr_region_size.attr,
Dan Williams6d821202020-10-13 16:50:55 -0700498 &dev_attr_region_align.attr,
Dan Williams0f3da142020-10-13 16:50:13 -0700499 &dev_attr_create.attr,
500 &dev_attr_seed.attr,
501 &dev_attr_delete.attr,
Dan Williams51cf7842017-07-12 17:58:21 -0700502 &dev_attr_id.attr,
503 NULL,
504};
505
506static const struct attribute_group dax_region_attribute_group = {
507 .name = "dax_region",
508 .attrs = dax_region_attributes,
Dan Williamsc2f30112020-10-13 16:50:03 -0700509 .is_visible = dax_region_visible,
Dan Williams51cf7842017-07-12 17:58:21 -0700510};
511
512static const struct attribute_group *dax_region_attribute_groups[] = {
513 &dax_region_attribute_group,
514 NULL,
515};
516
517static void dax_region_free(struct kref *kref)
518{
519 struct dax_region *dax_region;
520
521 dax_region = container_of(kref, struct dax_region, kref);
522 kfree(dax_region);
523}
524
525void dax_region_put(struct dax_region *dax_region)
526{
527 kref_put(&dax_region->kref, dax_region_free);
528}
529EXPORT_SYMBOL_GPL(dax_region_put);
530
531static void dax_region_unregister(void *region)
532{
533 struct dax_region *dax_region = region;
534
535 sysfs_remove_groups(&dax_region->dev->kobj,
536 dax_region_attribute_groups);
537 dax_region_put(dax_region);
538}
539
540struct dax_region *alloc_dax_region(struct device *parent, int region_id,
Dan Williamsa4574f62020-10-13 16:50:29 -0700541 struct range *range, int target_node, unsigned int align,
Dan Williamsc2f30112020-10-13 16:50:03 -0700542 unsigned long flags)
Dan Williams51cf7842017-07-12 17:58:21 -0700543{
544 struct dax_region *dax_region;
545
546 /*
547 * The DAX core assumes that it can store its private data in
548 * parent->driver_data. This WARN is a reminder / safeguard for
549 * developers of device-dax drivers.
550 */
551 if (dev_get_drvdata(parent)) {
552 dev_WARN(parent, "dax core failed to setup private data\n");
553 return NULL;
554 }
555
Dan Williamsa4574f62020-10-13 16:50:29 -0700556 if (!IS_ALIGNED(range->start, align)
557 || !IS_ALIGNED(range_len(range), align))
Dan Williams51cf7842017-07-12 17:58:21 -0700558 return NULL;
559
560 dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
561 if (!dax_region)
562 return NULL;
563
564 dev_set_drvdata(parent, dax_region);
Dan Williams51cf7842017-07-12 17:58:21 -0700565 kref_init(&dax_region->kref);
566 dax_region->id = region_id;
567 dax_region->align = align;
568 dax_region->dev = parent;
Dan Williams8fc5c732018-11-09 12:43:07 -0800569 dax_region->target_node = target_node;
Dan Williams0f3da142020-10-13 16:50:13 -0700570 ida_init(&dax_region->ida);
Dan Williamsc2f30112020-10-13 16:50:03 -0700571 dax_region->res = (struct resource) {
Dan Williamsa4574f62020-10-13 16:50:29 -0700572 .start = range->start,
573 .end = range->end,
Dan Williamsc2f30112020-10-13 16:50:03 -0700574 .flags = IORESOURCE_MEM | flags,
575 };
576
Dan Williams51cf7842017-07-12 17:58:21 -0700577 if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
578 kfree(dax_region);
579 return NULL;
580 }
581
582 kref_get(&dax_region->kref);
583 if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
584 return NULL;
585 return dax_region;
586}
587EXPORT_SYMBOL_GPL(alloc_dax_region);
588
Dan Williams0b07ce82020-10-13 16:50:45 -0700589static void dax_mapping_release(struct device *dev)
590{
591 struct dax_mapping *mapping = to_dax_mapping(dev);
592 struct dev_dax *dev_dax = to_dev_dax(dev->parent);
593
594 ida_free(&dev_dax->ida, mapping->id);
595 kfree(mapping);
596}
597
598static void unregister_dax_mapping(void *data)
599{
600 struct device *dev = data;
601 struct dax_mapping *mapping = to_dax_mapping(dev);
602 struct dev_dax *dev_dax = to_dev_dax(dev->parent);
603 struct dax_region *dax_region = dev_dax->region;
604
605 dev_dbg(dev, "%s\n", __func__);
606
607 device_lock_assert(dax_region->dev);
608
609 dev_dax->ranges[mapping->range_id].mapping = NULL;
610 mapping->range_id = -1;
611
612 device_del(dev);
613 put_device(dev);
614}
615
616static struct dev_dax_range *get_dax_range(struct device *dev)
617{
618 struct dax_mapping *mapping = to_dax_mapping(dev);
619 struct dev_dax *dev_dax = to_dev_dax(dev->parent);
620 struct dax_region *dax_region = dev_dax->region;
621
622 device_lock(dax_region->dev);
623 if (mapping->range_id < 0) {
624 device_unlock(dax_region->dev);
625 return NULL;
626 }
627
628 return &dev_dax->ranges[mapping->range_id];
629}
630
631static void put_dax_range(struct dev_dax_range *dax_range)
632{
633 struct dax_mapping *mapping = dax_range->mapping;
634 struct dev_dax *dev_dax = to_dev_dax(mapping->dev.parent);
635 struct dax_region *dax_region = dev_dax->region;
636
637 device_unlock(dax_region->dev);
638}
639
640static ssize_t start_show(struct device *dev,
641 struct device_attribute *attr, char *buf)
642{
643 struct dev_dax_range *dax_range;
644 ssize_t rc;
645
646 dax_range = get_dax_range(dev);
647 if (!dax_range)
648 return -ENXIO;
649 rc = sprintf(buf, "%#llx\n", dax_range->range.start);
650 put_dax_range(dax_range);
651
652 return rc;
653}
654static DEVICE_ATTR(start, 0400, start_show, NULL);
655
656static ssize_t end_show(struct device *dev,
657 struct device_attribute *attr, char *buf)
658{
659 struct dev_dax_range *dax_range;
660 ssize_t rc;
661
662 dax_range = get_dax_range(dev);
663 if (!dax_range)
664 return -ENXIO;
665 rc = sprintf(buf, "%#llx\n", dax_range->range.end);
666 put_dax_range(dax_range);
667
668 return rc;
669}
670static DEVICE_ATTR(end, 0400, end_show, NULL);
671
672static ssize_t pgoff_show(struct device *dev,
673 struct device_attribute *attr, char *buf)
674{
675 struct dev_dax_range *dax_range;
676 ssize_t rc;
677
678 dax_range = get_dax_range(dev);
679 if (!dax_range)
680 return -ENXIO;
681 rc = sprintf(buf, "%#lx\n", dax_range->pgoff);
682 put_dax_range(dax_range);
683
684 return rc;
685}
686static DEVICE_ATTR(page_offset, 0400, pgoff_show, NULL);
687
688static struct attribute *dax_mapping_attributes[] = {
689 &dev_attr_start.attr,
690 &dev_attr_end.attr,
691 &dev_attr_page_offset.attr,
692 NULL,
693};
694
695static const struct attribute_group dax_mapping_attribute_group = {
696 .attrs = dax_mapping_attributes,
697};
698
699static const struct attribute_group *dax_mapping_attribute_groups[] = {
700 &dax_mapping_attribute_group,
701 NULL,
702};
703
704static struct device_type dax_mapping_type = {
705 .release = dax_mapping_release,
706 .groups = dax_mapping_attribute_groups,
707};
708
709static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id)
710{
711 struct dax_region *dax_region = dev_dax->region;
712 struct dax_mapping *mapping;
713 struct device *dev;
714 int rc;
715
716 device_lock_assert(dax_region->dev);
717
718 if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver,
719 "region disabled\n"))
720 return -ENXIO;
721
722 mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
723 if (!mapping)
724 return -ENOMEM;
725 mapping->range_id = range_id;
726 mapping->id = ida_alloc(&dev_dax->ida, GFP_KERNEL);
727 if (mapping->id < 0) {
728 kfree(mapping);
729 return -ENOMEM;
730 }
731 dev_dax->ranges[range_id].mapping = mapping;
732 dev = &mapping->dev;
733 device_initialize(dev);
734 dev->parent = &dev_dax->dev;
735 dev->type = &dax_mapping_type;
736 dev_set_name(dev, "mapping%d", mapping->id);
737 rc = device_add(dev);
738 if (rc) {
739 put_device(dev);
740 return rc;
741 }
742
743 rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_mapping,
744 dev);
745 if (rc)
746 return rc;
747 return 0;
748}
749
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700750static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start,
751 resource_size_t size)
Dan Williamsc2f30112020-10-13 16:50:03 -0700752{
753 struct dax_region *dax_region = dev_dax->region;
754 struct resource *res = &dax_region->res;
755 struct device *dev = &dev_dax->dev;
Dan Williams60e93dc2020-10-13 16:50:39 -0700756 struct dev_dax_range *ranges;
757 unsigned long pgoff = 0;
Dan Williamsc2f30112020-10-13 16:50:03 -0700758 struct resource *alloc;
Dan Williams0b07ce82020-10-13 16:50:45 -0700759 int i, rc;
Dan Williamsc2f30112020-10-13 16:50:03 -0700760
761 device_lock_assert(dax_region->dev);
762
Dan Williams0f3da142020-10-13 16:50:13 -0700763 /* handle the seed alloc special case */
764 if (!size) {
Dan Williams60e93dc2020-10-13 16:50:39 -0700765 if (dev_WARN_ONCE(dev, dev_dax->nr_range,
766 "0-size allocation must be first\n"))
767 return -EBUSY;
768 /* nr_range == 0 is elsewhere special cased as 0-size device */
Dan Williams0f3da142020-10-13 16:50:13 -0700769 return 0;
770 }
771
Zhen Leiff8da372020-12-19 16:18:40 +0800772 alloc = __request_region(res, start, size, dev_name(dev), 0);
773 if (!alloc)
Dan Williamsc2f30112020-10-13 16:50:03 -0700774 return -ENOMEM;
775
Zhen Leiff8da372020-12-19 16:18:40 +0800776 ranges = krealloc(dev_dax->ranges, sizeof(*ranges)
777 * (dev_dax->nr_range + 1), GFP_KERNEL);
778 if (!ranges) {
779 __release_region(res, alloc->start, resource_size(alloc));
Dan Williams60e93dc2020-10-13 16:50:39 -0700780 return -ENOMEM;
781 }
782
783 for (i = 0; i < dev_dax->nr_range; i++)
784 pgoff += PHYS_PFN(range_len(&ranges[i].range));
785 dev_dax->ranges = ranges;
786 ranges[dev_dax->nr_range++] = (struct dev_dax_range) {
787 .pgoff = pgoff,
788 .range = {
789 .start = alloc->start,
790 .end = alloc->end,
791 },
Dan Williamsc2f30112020-10-13 16:50:03 -0700792 };
793
Dan Williams60e93dc2020-10-13 16:50:39 -0700794 dev_dbg(dev, "alloc range[%d]: %pa:%pa\n", dev_dax->nr_range - 1,
795 &alloc->start, &alloc->end);
Dan Williams0b07ce82020-10-13 16:50:45 -0700796 /*
797 * A dev_dax instance must be registered before mapping device
798 * children can be added. Defer to devm_create_dev_dax() to add
799 * the initial mapping device.
800 */
801 if (!device_is_registered(&dev_dax->dev))
802 return 0;
803
804 rc = devm_register_dax_mapping(dev_dax, dev_dax->nr_range - 1);
Dan Williams6268d7d2020-12-18 18:41:41 -0800805 if (rc)
806 trim_dev_dax_range(dev_dax);
Dan Williams60e93dc2020-10-13 16:50:39 -0700807
Dan Williams6268d7d2020-12-18 18:41:41 -0800808 return rc;
Dan Williamsc2f30112020-10-13 16:50:03 -0700809}
810
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700811static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, resource_size_t size)
812{
Dan Williams60e93dc2020-10-13 16:50:39 -0700813 int last_range = dev_dax->nr_range - 1;
814 struct dev_dax_range *dax_range = &dev_dax->ranges[last_range];
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700815 struct dax_region *dax_region = dev_dax->region;
Dan Williams60e93dc2020-10-13 16:50:39 -0700816 bool is_shrink = resource_size(res) > size;
817 struct range *range = &dax_range->range;
818 struct device *dev = &dev_dax->dev;
819 int rc;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700820
821 device_lock_assert(dax_region->dev);
822
Dan Williams60e93dc2020-10-13 16:50:39 -0700823 if (dev_WARN_ONCE(dev, !size, "deletion is handled by dev_dax_shrink\n"))
824 return -EINVAL;
825
826 rc = adjust_resource(res, range->start, size);
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700827 if (rc)
828 return rc;
829
Dan Williams60e93dc2020-10-13 16:50:39 -0700830 *range = (struct range) {
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700831 .start = range->start,
832 .end = range->start + size - 1,
833 };
834
Dan Williams60e93dc2020-10-13 16:50:39 -0700835 dev_dbg(dev, "%s range[%d]: %#llx:%#llx\n", is_shrink ? "shrink" : "extend",
836 last_range, (unsigned long long) range->start,
837 (unsigned long long) range->end);
838
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700839 return 0;
840}
841
Dan Williams51cf7842017-07-12 17:58:21 -0700842static ssize_t size_show(struct device *dev,
843 struct device_attribute *attr, char *buf)
844{
845 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williams60e93dc2020-10-13 16:50:39 -0700846 unsigned long long size;
847
848 device_lock(dev);
849 size = dev_dax_size(dev_dax);
850 device_unlock(dev);
Dan Williams51cf7842017-07-12 17:58:21 -0700851
852 return sprintf(buf, "%llu\n", size);
853}
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700854
Dan Williams6d821202020-10-13 16:50:55 -0700855static bool alloc_is_aligned(struct dev_dax *dev_dax, resource_size_t size)
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700856{
857 /*
858 * The minimum mapping granularity for a device instance is a
859 * single subsection, unless the arch says otherwise.
860 */
Dan Williams6d821202020-10-13 16:50:55 -0700861 return IS_ALIGNED(size, max_t(unsigned long, dev_dax->align, memremap_compat_align()));
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700862}
863
864static int dev_dax_shrink(struct dev_dax *dev_dax, resource_size_t size)
865{
Dan Williams60e93dc2020-10-13 16:50:39 -0700866 resource_size_t to_shrink = dev_dax_size(dev_dax) - size;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700867 struct dax_region *dax_region = dev_dax->region;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700868 struct device *dev = &dev_dax->dev;
Dan Williams60e93dc2020-10-13 16:50:39 -0700869 int i;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700870
Dan Williams60e93dc2020-10-13 16:50:39 -0700871 for (i = dev_dax->nr_range - 1; i >= 0; i--) {
872 struct range *range = &dev_dax->ranges[i].range;
Dan Williams0b07ce82020-10-13 16:50:45 -0700873 struct dax_mapping *mapping = dev_dax->ranges[i].mapping;
Dan Williams60e93dc2020-10-13 16:50:39 -0700874 struct resource *adjust = NULL, *res;
875 resource_size_t shrink;
876
877 shrink = min_t(u64, to_shrink, range_len(range));
878 if (shrink >= range_len(range)) {
Dan Williams0b07ce82020-10-13 16:50:45 -0700879 devm_release_action(dax_region->dev,
880 unregister_dax_mapping, &mapping->dev);
Dan Williams6268d7d2020-12-18 18:41:41 -0800881 trim_dev_dax_range(dev_dax);
Dan Williams60e93dc2020-10-13 16:50:39 -0700882 to_shrink -= shrink;
883 if (!to_shrink)
884 break;
885 continue;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700886 }
887
Dan Williams60e93dc2020-10-13 16:50:39 -0700888 for_each_dax_region_resource(dax_region, res)
889 if (strcmp(res->name, dev_name(dev)) == 0
890 && res->start == range->start) {
891 adjust = res;
892 break;
893 }
894
895 if (dev_WARN_ONCE(dev, !adjust || i != dev_dax->nr_range - 1,
896 "failed to find matching resource\n"))
897 return -ENXIO;
898 return adjust_dev_dax_range(dev_dax, adjust, range_len(range)
899 - shrink);
900 }
901 return 0;
902}
903
904/*
905 * Only allow adjustments that preserve the relative pgoff of existing
906 * allocations. I.e. the dev_dax->ranges array is ordered by increasing pgoff.
907 */
908static bool adjust_ok(struct dev_dax *dev_dax, struct resource *res)
909{
910 struct dev_dax_range *last;
911 int i;
912
913 if (dev_dax->nr_range == 0)
914 return false;
915 if (strcmp(res->name, dev_name(&dev_dax->dev)) != 0)
916 return false;
917 last = &dev_dax->ranges[dev_dax->nr_range - 1];
918 if (last->range.start != res->start || last->range.end != res->end)
919 return false;
920 for (i = 0; i < dev_dax->nr_range - 1; i++) {
921 struct dev_dax_range *dax_range = &dev_dax->ranges[i];
922
923 if (dax_range->pgoff > last->pgoff)
924 return false;
925 }
926
927 return true;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700928}
929
930static ssize_t dev_dax_resize(struct dax_region *dax_region,
931 struct dev_dax *dev_dax, resource_size_t size)
932{
933 resource_size_t avail = dax_region_avail_size(dax_region), to_alloc;
Dan Williams60e93dc2020-10-13 16:50:39 -0700934 resource_size_t dev_size = dev_dax_size(dev_dax);
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700935 struct resource *region_res = &dax_region->res;
936 struct device *dev = &dev_dax->dev;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700937 struct resource *res, *first;
Dan Williams60e93dc2020-10-13 16:50:39 -0700938 resource_size_t alloc = 0;
939 int rc;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700940
941 if (dev->driver)
942 return -EBUSY;
943 if (size == dev_size)
944 return 0;
945 if (size > dev_size && size - dev_size > avail)
946 return -ENOSPC;
947 if (size < dev_size)
948 return dev_dax_shrink(dev_dax, size);
949
950 to_alloc = size - dev_size;
Dan Williams6d821202020-10-13 16:50:55 -0700951 if (dev_WARN_ONCE(dev, !alloc_is_aligned(dev_dax, to_alloc),
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700952 "resize of %pa misaligned\n", &to_alloc))
953 return -ENXIO;
954
955 /*
956 * Expand the device into the unused portion of the region. This
957 * may involve adjusting the end of an existing resource, or
958 * allocating a new resource.
959 */
Dan Williams60e93dc2020-10-13 16:50:39 -0700960retry:
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700961 first = region_res->child;
962 if (!first)
963 return alloc_dev_dax_range(dev_dax, dax_region->res.start, to_alloc);
Dan Williams60e93dc2020-10-13 16:50:39 -0700964
965 rc = -ENOSPC;
966 for (res = first; res; res = res->sibling) {
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700967 struct resource *next = res->sibling;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700968
969 /* space at the beginning of the region */
Dan Williams60e93dc2020-10-13 16:50:39 -0700970 if (res == first && res->start > dax_region->res.start) {
971 alloc = min(res->start - dax_region->res.start, to_alloc);
972 rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, alloc);
973 break;
974 }
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700975
Dan Williams60e93dc2020-10-13 16:50:39 -0700976 alloc = 0;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700977 /* space between allocations */
978 if (next && next->start > res->end + 1)
Dan Williams60e93dc2020-10-13 16:50:39 -0700979 alloc = min(next->start - (res->end + 1), to_alloc);
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700980
981 /* space at the end of the region */
Dan Williams60e93dc2020-10-13 16:50:39 -0700982 if (!alloc && !next && res->end < region_res->end)
983 alloc = min(region_res->end - res->end, to_alloc);
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700984
Dan Williams60e93dc2020-10-13 16:50:39 -0700985 if (!alloc)
986 continue;
987
988 if (adjust_ok(dev_dax, res)) {
989 rc = adjust_dev_dax_range(dev_dax, res, resource_size(res) + alloc);
990 break;
991 }
992 rc = alloc_dev_dax_range(dev_dax, res->end + 1, alloc);
993 break;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700994 }
Dan Williams60e93dc2020-10-13 16:50:39 -0700995 if (rc)
996 return rc;
997 to_alloc -= alloc;
998 if (to_alloc)
999 goto retry;
1000 return 0;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001001}
1002
1003static ssize_t size_store(struct device *dev, struct device_attribute *attr,
1004 const char *buf, size_t len)
1005{
1006 ssize_t rc;
1007 unsigned long long val;
1008 struct dev_dax *dev_dax = to_dev_dax(dev);
1009 struct dax_region *dax_region = dev_dax->region;
1010
1011 rc = kstrtoull(buf, 0, &val);
1012 if (rc)
1013 return rc;
1014
Dan Williams6d821202020-10-13 16:50:55 -07001015 if (!alloc_is_aligned(dev_dax, val)) {
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001016 dev_dbg(dev, "%s: size: %lld misaligned\n", __func__, val);
1017 return -EINVAL;
1018 }
1019
1020 device_lock(dax_region->dev);
1021 if (!dax_region->dev->driver) {
1022 device_unlock(dax_region->dev);
1023 return -ENXIO;
1024 }
1025 device_lock(dev);
1026 rc = dev_dax_resize(dax_region, dev_dax, val);
1027 device_unlock(dev);
1028 device_unlock(dax_region->dev);
1029
1030 return rc == 0 ? len : rc;
1031}
1032static DEVICE_ATTR_RW(size);
Dan Williams51cf7842017-07-12 17:58:21 -07001033
Joao Martins8490e2e2020-10-13 16:51:06 -07001034static ssize_t range_parse(const char *opt, size_t len, struct range *range)
1035{
1036 unsigned long long addr = 0;
1037 char *start, *end, *str;
Shiyang Ruan7323fb22021-01-26 10:13:31 +08001038 ssize_t rc = -EINVAL;
Joao Martins8490e2e2020-10-13 16:51:06 -07001039
1040 str = kstrdup(opt, GFP_KERNEL);
1041 if (!str)
1042 return rc;
1043
1044 end = str;
1045 start = strsep(&end, "-");
1046 if (!start || !end)
1047 goto err;
1048
1049 rc = kstrtoull(start, 16, &addr);
1050 if (rc)
1051 goto err;
1052 range->start = addr;
1053
1054 rc = kstrtoull(end, 16, &addr);
1055 if (rc)
1056 goto err;
1057 range->end = addr;
1058
1059err:
1060 kfree(str);
1061 return rc;
1062}
1063
1064static ssize_t mapping_store(struct device *dev, struct device_attribute *attr,
1065 const char *buf, size_t len)
1066{
1067 struct dev_dax *dev_dax = to_dev_dax(dev);
1068 struct dax_region *dax_region = dev_dax->region;
1069 size_t to_alloc;
1070 struct range r;
1071 ssize_t rc;
1072
1073 rc = range_parse(buf, len, &r);
1074 if (rc)
1075 return rc;
1076
1077 rc = -ENXIO;
1078 device_lock(dax_region->dev);
1079 if (!dax_region->dev->driver) {
1080 device_unlock(dax_region->dev);
1081 return rc;
1082 }
1083 device_lock(dev);
1084
1085 to_alloc = range_len(&r);
1086 if (alloc_is_aligned(dev_dax, to_alloc))
1087 rc = alloc_dev_dax_range(dev_dax, r.start, to_alloc);
1088 device_unlock(dev);
1089 device_unlock(dax_region->dev);
1090
1091 return rc == 0 ? len : rc;
1092}
1093static DEVICE_ATTR_WO(mapping);
1094
Dan Williams6d821202020-10-13 16:50:55 -07001095static ssize_t align_show(struct device *dev,
1096 struct device_attribute *attr, char *buf)
1097{
1098 struct dev_dax *dev_dax = to_dev_dax(dev);
1099
1100 return sprintf(buf, "%d\n", dev_dax->align);
1101}
1102
1103static ssize_t dev_dax_validate_align(struct dev_dax *dev_dax)
1104{
Dan Williams6d821202020-10-13 16:50:55 -07001105 struct device *dev = &dev_dax->dev;
1106 int i;
1107
Dan Williams6d821202020-10-13 16:50:55 -07001108 for (i = 0; i < dev_dax->nr_range; i++) {
1109 size_t len = range_len(&dev_dax->ranges[i].range);
1110
1111 if (!alloc_is_aligned(dev_dax, len)) {
1112 dev_dbg(dev, "%s: align %u invalid for range %d\n",
1113 __func__, dev_dax->align, i);
1114 return -EINVAL;
1115 }
1116 }
1117
1118 return 0;
1119}
1120
1121static ssize_t align_store(struct device *dev, struct device_attribute *attr,
1122 const char *buf, size_t len)
1123{
1124 struct dev_dax *dev_dax = to_dev_dax(dev);
1125 struct dax_region *dax_region = dev_dax->region;
1126 unsigned long val, align_save;
1127 ssize_t rc;
1128
1129 rc = kstrtoul(buf, 0, &val);
1130 if (rc)
1131 return -ENXIO;
1132
1133 if (!dax_align_valid(val))
1134 return -EINVAL;
1135
1136 device_lock(dax_region->dev);
1137 if (!dax_region->dev->driver) {
1138 device_unlock(dax_region->dev);
1139 return -ENXIO;
1140 }
1141
1142 device_lock(dev);
1143 if (dev->driver) {
1144 rc = -EBUSY;
1145 goto out_unlock;
1146 }
1147
1148 align_save = dev_dax->align;
1149 dev_dax->align = val;
1150 rc = dev_dax_validate_align(dev_dax);
1151 if (rc)
1152 dev_dax->align = align_save;
1153out_unlock:
1154 device_unlock(dev);
1155 device_unlock(dax_region->dev);
1156 return rc == 0 ? len : rc;
1157}
1158static DEVICE_ATTR_RW(align);
1159
Dan Williams21c75762019-02-20 11:39:36 -08001160static int dev_dax_target_node(struct dev_dax *dev_dax)
1161{
1162 struct dax_region *dax_region = dev_dax->region;
1163
1164 return dax_region->target_node;
1165}
1166
1167static ssize_t target_node_show(struct device *dev,
1168 struct device_attribute *attr, char *buf)
1169{
1170 struct dev_dax *dev_dax = to_dev_dax(dev);
1171
1172 return sprintf(buf, "%d\n", dev_dax_target_node(dev_dax));
1173}
1174static DEVICE_ATTR_RO(target_node);
1175
Vishal Verma40cdc602019-06-20 18:40:38 -06001176static ssize_t resource_show(struct device *dev,
1177 struct device_attribute *attr, char *buf)
1178{
1179 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williams60e93dc2020-10-13 16:50:39 -07001180 struct dax_region *dax_region = dev_dax->region;
1181 unsigned long long start;
Vishal Verma40cdc602019-06-20 18:40:38 -06001182
Dan Williams60e93dc2020-10-13 16:50:39 -07001183 if (dev_dax->nr_range < 1)
1184 start = dax_region->res.start;
1185 else
1186 start = dev_dax->ranges[0].range.start;
1187
1188 return sprintf(buf, "%#llx\n", start);
Vishal Verma40cdc602019-06-20 18:40:38 -06001189}
Dan Williams153dd282019-11-12 17:12:38 -08001190static DEVICE_ATTR(resource, 0400, resource_show, NULL);
Vishal Verma40cdc602019-06-20 18:40:38 -06001191
Vishal Vermac347bd72019-02-22 16:58:54 -07001192static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
1193 char *buf)
1194{
1195 /*
1196 * We only ever expect to handle device-dax instances, i.e. the
1197 * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
1198 */
1199 return sprintf(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0);
1200}
1201static DEVICE_ATTR_RO(modalias);
1202
Dan Williamscb4dd722019-11-12 17:13:31 -08001203static ssize_t numa_node_show(struct device *dev,
1204 struct device_attribute *attr, char *buf)
1205{
1206 return sprintf(buf, "%d\n", dev_to_node(dev));
1207}
1208static DEVICE_ATTR_RO(numa_node);
1209
Dan Williams21c75762019-02-20 11:39:36 -08001210static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n)
1211{
1212 struct device *dev = container_of(kobj, struct device, kobj);
1213 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001214 struct dax_region *dax_region = dev_dax->region;
Dan Williams21c75762019-02-20 11:39:36 -08001215
1216 if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
1217 return 0;
Dan Williamscb4dd722019-11-12 17:13:31 -08001218 if (a == &dev_attr_numa_node.attr && !IS_ENABLED(CONFIG_NUMA))
1219 return 0;
Joao Martins8490e2e2020-10-13 16:51:06 -07001220 if (a == &dev_attr_mapping.attr && is_static(dax_region))
1221 return 0;
Dan Williams6d821202020-10-13 16:50:55 -07001222 if ((a == &dev_attr_align.attr ||
1223 a == &dev_attr_size.attr) && is_static(dax_region))
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001224 return 0444;
Dan Williams21c75762019-02-20 11:39:36 -08001225 return a->mode;
1226}
1227
Dan Williams51cf7842017-07-12 17:58:21 -07001228static struct attribute *dev_dax_attributes[] = {
Vishal Vermac347bd72019-02-22 16:58:54 -07001229 &dev_attr_modalias.attr,
Dan Williams51cf7842017-07-12 17:58:21 -07001230 &dev_attr_size.attr,
Joao Martins8490e2e2020-10-13 16:51:06 -07001231 &dev_attr_mapping.attr,
Dan Williams21c75762019-02-20 11:39:36 -08001232 &dev_attr_target_node.attr,
Dan Williams6d821202020-10-13 16:50:55 -07001233 &dev_attr_align.attr,
Vishal Verma40cdc602019-06-20 18:40:38 -06001234 &dev_attr_resource.attr,
Dan Williamscb4dd722019-11-12 17:13:31 -08001235 &dev_attr_numa_node.attr,
Dan Williams51cf7842017-07-12 17:58:21 -07001236 NULL,
1237};
1238
1239static const struct attribute_group dev_dax_attribute_group = {
1240 .attrs = dev_dax_attributes,
Dan Williams21c75762019-02-20 11:39:36 -08001241 .is_visible = dev_dax_visible,
Dan Williams51cf7842017-07-12 17:58:21 -07001242};
1243
Dan Williams9567da02017-07-12 17:58:21 -07001244static const struct attribute_group *dax_attribute_groups[] = {
Dan Williams51cf7842017-07-12 17:58:21 -07001245 &dev_dax_attribute_group,
1246 NULL,
1247};
Dan Williams51cf7842017-07-12 17:58:21 -07001248
Dan Williams9567da02017-07-12 17:58:21 -07001249static void dev_dax_release(struct device *dev)
Dan Williams51cf7842017-07-12 17:58:21 -07001250{
1251 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williams9567da02017-07-12 17:58:21 -07001252 struct dax_region *dax_region = dev_dax->region;
Dan Williams51cf7842017-07-12 17:58:21 -07001253 struct dax_device *dax_dev = dev_dax->dax_dev;
Dan Williams51cf7842017-07-12 17:58:21 -07001254
Dan Williams9567da02017-07-12 17:58:21 -07001255 put_dax(dax_dev);
Dan Williams0f3da142020-10-13 16:50:13 -07001256 free_dev_dax_id(dev_dax);
1257 dax_region_put(dax_region);
Dan Williamsf5516ec2020-10-13 16:49:43 -07001258 kfree(dev_dax->pgmap);
Dan Williams9567da02017-07-12 17:58:21 -07001259 kfree(dev_dax);
1260}
1261
Dan Williams770619a2019-11-12 17:12:23 -08001262static const struct device_type dev_dax_type = {
1263 .release = dev_dax_release,
1264 .groups = dax_attribute_groups,
1265};
1266
Dan Williams174ebec2020-10-13 16:49:38 -07001267struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
Dan Williams9567da02017-07-12 17:58:21 -07001268{
Dan Williams174ebec2020-10-13 16:49:38 -07001269 struct dax_region *dax_region = data->dax_region;
Dan Williams9567da02017-07-12 17:58:21 -07001270 struct device *parent = dax_region->dev;
1271 struct dax_device *dax_dev;
1272 struct dev_dax *dev_dax;
1273 struct inode *inode;
1274 struct device *dev;
Dan Williamsc2f30112020-10-13 16:50:03 -07001275 int rc;
Dan Williams9567da02017-07-12 17:58:21 -07001276
Dan Williams9567da02017-07-12 17:58:21 -07001277 dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL);
1278 if (!dev_dax)
1279 return ERR_PTR(-ENOMEM);
1280
Dan Williams0f3da142020-10-13 16:50:13 -07001281 if (is_static(dax_region)) {
1282 if (dev_WARN_ONCE(parent, data->id < 0,
1283 "dynamic id specified to static region\n")) {
1284 rc = -EINVAL;
1285 goto err_id;
1286 }
1287
1288 dev_dax->id = data->id;
1289 } else {
1290 if (dev_WARN_ONCE(parent, data->id >= 0,
1291 "static id specified to dynamic region\n")) {
1292 rc = -EINVAL;
1293 goto err_id;
1294 }
1295
1296 rc = ida_alloc(&dax_region->ida, GFP_KERNEL);
1297 if (rc < 0)
1298 goto err_id;
1299 dev_dax->id = rc;
1300 }
1301
Dan Williamsc2f30112020-10-13 16:50:03 -07001302 dev_dax->region = dax_region;
1303 dev = &dev_dax->dev;
1304 device_initialize(dev);
Dan Williams0f3da142020-10-13 16:50:13 -07001305 dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
Dan Williamsc2f30112020-10-13 16:50:03 -07001306
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001307 rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, data->size);
Dan Williamsc2f30112020-10-13 16:50:03 -07001308 if (rc)
1309 goto err_range;
1310
Dan Williamsf5516ec2020-10-13 16:49:43 -07001311 if (data->pgmap) {
Dan Williamsc2f30112020-10-13 16:50:03 -07001312 dev_WARN_ONCE(parent, !is_static(dax_region),
1313 "custom dev_pagemap requires a static dax_region\n");
1314
Dan Williamsf5516ec2020-10-13 16:49:43 -07001315 dev_dax->pgmap = kmemdup(data->pgmap,
1316 sizeof(struct dev_pagemap), GFP_KERNEL);
Dan Williamsc2f30112020-10-13 16:50:03 -07001317 if (!dev_dax->pgmap) {
1318 rc = -ENOMEM;
Dan Williamsf5516ec2020-10-13 16:49:43 -07001319 goto err_pgmap;
Dan Williamsc2f30112020-10-13 16:50:03 -07001320 }
Dan Williamsf5516ec2020-10-13 16:49:43 -07001321 }
Dan Williams89ec9f22018-10-29 15:52:42 -07001322
Dan Williams9567da02017-07-12 17:58:21 -07001323 /*
Christoph Hellwigfb08a192021-11-29 11:21:38 +01001324 * No dax_operations since there is no access to this device outside of
1325 * mmap of the resulting character device.
Dan Williams9567da02017-07-12 17:58:21 -07001326 */
Christoph Hellwig30c68282021-12-15 09:45:07 +01001327 dax_dev = alloc_dax(dev_dax, NULL);
Vivek Goyal4e4ced92020-04-01 12:11:25 -04001328 if (IS_ERR(dax_dev)) {
1329 rc = PTR_ERR(dax_dev);
Dan Williamsf5516ec2020-10-13 16:49:43 -07001330 goto err_alloc_dax;
Vivek Goyal4e4ced92020-04-01 12:11:25 -04001331 }
Christoph Hellwig30c68282021-12-15 09:45:07 +01001332 set_dax_synchronous(dax_dev);
Dan Williams9567da02017-07-12 17:58:21 -07001333
1334 /* a device_dax instance is dead while the driver is not attached */
1335 kill_dax(dax_dev);
1336
Dan Williams9567da02017-07-12 17:58:21 -07001337 dev_dax->dax_dev = dax_dev;
Dan Williams8fc5c732018-11-09 12:43:07 -08001338 dev_dax->target_node = dax_region->target_node;
Joao Martins33cf94d2020-10-13 16:50:50 -07001339 dev_dax->align = dax_region->align;
Dan Williams0b07ce82020-10-13 16:50:45 -07001340 ida_init(&dev_dax->ida);
Dan Williams9567da02017-07-12 17:58:21 -07001341 kref_get(&dax_region->kref);
1342
1343 inode = dax_inode(dax_dev);
1344 dev->devt = inode->i_rdev;
Dan Williams83762cb2021-11-15 13:20:57 -08001345 dev->bus = &dax_bus_type;
Dan Williams9567da02017-07-12 17:58:21 -07001346 dev->parent = parent;
Dan Williams770619a2019-11-12 17:12:23 -08001347 dev->type = &dev_dax_type;
Dan Williams9567da02017-07-12 17:58:21 -07001348
1349 rc = device_add(dev);
1350 if (rc) {
1351 kill_dev_dax(dev_dax);
1352 put_device(dev);
1353 return ERR_PTR(rc);
1354 }
1355
1356 rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
1357 if (rc)
1358 return ERR_PTR(rc);
1359
Dan Williams0b07ce82020-10-13 16:50:45 -07001360 /* register mapping device for the initial allocation range */
1361 if (dev_dax->nr_range && range_len(&dev_dax->ranges[0].range)) {
1362 rc = devm_register_dax_mapping(dev_dax, 0);
1363 if (rc)
1364 return ERR_PTR(rc);
1365 }
1366
Dan Williams9567da02017-07-12 17:58:21 -07001367 return dev_dax;
Dan Williamsc2f30112020-10-13 16:50:03 -07001368
Dan Williamsf5516ec2020-10-13 16:49:43 -07001369err_alloc_dax:
1370 kfree(dev_dax->pgmap);
1371err_pgmap:
Dan Williams60e93dc2020-10-13 16:50:39 -07001372 free_dev_dax_ranges(dev_dax);
Dan Williamsc2f30112020-10-13 16:50:03 -07001373err_range:
Dan Williams0f3da142020-10-13 16:50:13 -07001374 free_dev_dax_id(dev_dax);
1375err_id:
Dan Williams9567da02017-07-12 17:58:21 -07001376 kfree(dev_dax);
1377
1378 return ERR_PTR(rc);
1379}
Dan Williams174ebec2020-10-13 16:49:38 -07001380EXPORT_SYMBOL_GPL(devm_create_dev_dax);
Dan Williams9567da02017-07-12 17:58:21 -07001381
Dan Williamsd2007812018-11-07 15:31:23 -08001382static int match_always_count;
1383
1384int __dax_driver_register(struct dax_device_driver *dax_drv,
Dan Williams9567da02017-07-12 17:58:21 -07001385 struct module *module, const char *mod_name)
1386{
Dan Williamsd2007812018-11-07 15:31:23 -08001387 struct device_driver *drv = &dax_drv->drv;
1388 int rc = 0;
1389
Uwe Kleine-König5b8e64f2021-02-05 23:28:38 +01001390 /*
1391 * dax_bus_probe() calls dax_drv->probe() unconditionally.
1392 * So better be safe than sorry and ensure it is provided.
1393 */
1394 if (!dax_drv->probe)
1395 return -EINVAL;
1396
Dan Williamsd2007812018-11-07 15:31:23 -08001397 INIT_LIST_HEAD(&dax_drv->ids);
Dan Williams9567da02017-07-12 17:58:21 -07001398 drv->owner = module;
1399 drv->name = mod_name;
1400 drv->mod_name = mod_name;
1401 drv->bus = &dax_bus_type;
Dan Williamsd2007812018-11-07 15:31:23 -08001402
1403 /* there can only be one default driver */
1404 mutex_lock(&dax_bus_lock);
1405 match_always_count += dax_drv->match_always;
1406 if (match_always_count > 1) {
1407 match_always_count--;
1408 WARN_ON(1);
1409 rc = -EINVAL;
1410 }
1411 mutex_unlock(&dax_bus_lock);
1412 if (rc)
1413 return rc;
Uwe Kleine-Könige307bf12021-02-05 23:28:40 +01001414
1415 rc = driver_register(drv);
1416 if (rc && dax_drv->match_always) {
1417 mutex_lock(&dax_bus_lock);
1418 match_always_count -= dax_drv->match_always;
1419 mutex_unlock(&dax_bus_lock);
1420 }
1421
1422 return rc;
Dan Williams9567da02017-07-12 17:58:21 -07001423}
1424EXPORT_SYMBOL_GPL(__dax_driver_register);
1425
Dan Williamsd2007812018-11-07 15:31:23 -08001426void dax_driver_unregister(struct dax_device_driver *dax_drv)
1427{
Dan Williams730926c2017-07-16 13:51:53 -07001428 struct device_driver *drv = &dax_drv->drv;
Dan Williamsd2007812018-11-07 15:31:23 -08001429 struct dax_id *dax_id, *_id;
1430
1431 mutex_lock(&dax_bus_lock);
1432 match_always_count -= dax_drv->match_always;
1433 list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) {
1434 list_del(&dax_id->list);
1435 kfree(dax_id);
1436 }
1437 mutex_unlock(&dax_bus_lock);
Dan Williams730926c2017-07-16 13:51:53 -07001438 driver_unregister(drv);
Dan Williamsd2007812018-11-07 15:31:23 -08001439}
1440EXPORT_SYMBOL_GPL(dax_driver_unregister);
1441
Dan Williams9567da02017-07-12 17:58:21 -07001442int __init dax_bus_init(void)
1443{
Dan Williams83762cb2021-11-15 13:20:57 -08001444 return bus_register(&dax_bus_type);
Dan Williams9567da02017-07-12 17:58:21 -07001445}
1446
1447void __exit dax_bus_exit(void)
1448{
1449 bus_unregister(&dax_bus_type);
1450}