Thomas Gleixner | d2912cb | 2019-06-04 10:11:33 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 2 | /* |
| 3 | * omap iommu: tlb and pagetable primitives |
| 4 | * |
Hiroshi DOYU | c127c7d | 2010-02-15 10:03:32 -0800 | [diff] [blame] | 5 | * Copyright (C) 2008-2010 Nokia Corporation |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 6 | * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/ |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 7 | * |
| 8 | * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>, |
| 9 | * Paul Mundt and Toshihiro Kobayashi |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 10 | */ |
| 11 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 12 | #include <linux/dma-mapping.h> |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 13 | #include <linux/err.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 14 | #include <linux/slab.h> |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 15 | #include <linux/interrupt.h> |
| 16 | #include <linux/ioport.h> |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 17 | #include <linux/platform_device.h> |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 18 | #include <linux/iommu.h> |
Tony Lindgren | c8d35c8 | 2012-11-02 12:24:03 -0700 | [diff] [blame] | 19 | #include <linux/omap-iommu.h> |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 20 | #include <linux/mutex.h> |
| 21 | #include <linux/spinlock.h> |
Tony Lindgren | ed1c7de | 2012-11-02 12:24:06 -0700 | [diff] [blame] | 22 | #include <linux/io.h> |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 23 | #include <linux/pm_runtime.h> |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 24 | #include <linux/of.h> |
| 25 | #include <linux/of_iommu.h> |
| 26 | #include <linux/of_irq.h> |
Suman Anna | 7d68277 | 2014-09-04 17:27:30 -0500 | [diff] [blame] | 27 | #include <linux/of_platform.h> |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 28 | #include <linux/regmap.h> |
| 29 | #include <linux/mfd/syscon.h> |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 30 | |
Tony Lindgren | 2ab7c84 | 2012-11-02 12:24:14 -0700 | [diff] [blame] | 31 | #include <linux/platform_data/iommu-omap.h> |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 32 | |
Ido Yariv | 2f7702a | 2012-11-02 12:24:00 -0700 | [diff] [blame] | 33 | #include "omap-iopgtable.h" |
Tony Lindgren | ed1c7de | 2012-11-02 12:24:06 -0700 | [diff] [blame] | 34 | #include "omap-iommu.h" |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 35 | |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 36 | static const struct iommu_ops omap_iommu_ops; |
| 37 | |
Kefeng Wang | 6e8b566 | 2019-04-23 15:50:08 +0800 | [diff] [blame] | 38 | #define to_iommu(dev) ((struct omap_iommu *)dev_get_drvdata(dev)) |
Suman Anna | 5acc97d | 2014-03-17 20:31:34 -0500 | [diff] [blame] | 39 | |
Ohad Ben-Cohen | 66bc8cf | 2011-11-10 11:32:27 +0200 | [diff] [blame] | 40 | /* bitmap of the page sizes currently supported */ |
| 41 | #define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) |
| 42 | |
Ido Yariv | 7bd9e25 | 2012-11-02 12:24:09 -0700 | [diff] [blame] | 43 | #define MMU_LOCK_BASE_SHIFT 10 |
| 44 | #define MMU_LOCK_BASE_MASK (0x1f << MMU_LOCK_BASE_SHIFT) |
| 45 | #define MMU_LOCK_BASE(x) \ |
| 46 | ((x & MMU_LOCK_BASE_MASK) >> MMU_LOCK_BASE_SHIFT) |
| 47 | |
| 48 | #define MMU_LOCK_VICT_SHIFT 4 |
| 49 | #define MMU_LOCK_VICT_MASK (0x1f << MMU_LOCK_VICT_SHIFT) |
| 50 | #define MMU_LOCK_VICT(x) \ |
| 51 | ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT) |
| 52 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 53 | static struct platform_driver omap_iommu_driver; |
| 54 | static struct kmem_cache *iopte_cachep; |
| 55 | |
| 56 | /** |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 57 | * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain |
| 58 | * @dom: generic iommu domain handle |
| 59 | **/ |
| 60 | static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom) |
| 61 | { |
| 62 | return container_of(dom, struct omap_iommu_domain, domain); |
| 63 | } |
| 64 | |
| 65 | /** |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 66 | * omap_iommu_save_ctx - Save registers for pm off-mode support |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 67 | * @dev: client device |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 68 | **/ |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 69 | void omap_iommu_save_ctx(struct device *dev) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 70 | { |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 71 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
| 72 | struct omap_iommu *obj; |
| 73 | u32 *p; |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 74 | int i; |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 75 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 76 | if (!arch_data) |
| 77 | return; |
| 78 | |
| 79 | while (arch_data->iommu_dev) { |
| 80 | obj = arch_data->iommu_dev; |
| 81 | p = obj->ctx; |
| 82 | for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) { |
| 83 | p[i] = iommu_read_reg(obj, i * sizeof(u32)); |
| 84 | dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, |
| 85 | p[i]); |
| 86 | } |
| 87 | arch_data++; |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 88 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 89 | } |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 90 | EXPORT_SYMBOL_GPL(omap_iommu_save_ctx); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 91 | |
| 92 | /** |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 93 | * omap_iommu_restore_ctx - Restore registers for pm off-mode support |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 94 | * @dev: client device |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 95 | **/ |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 96 | void omap_iommu_restore_ctx(struct device *dev) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 97 | { |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 98 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
| 99 | struct omap_iommu *obj; |
| 100 | u32 *p; |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 101 | int i; |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 102 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 103 | if (!arch_data) |
| 104 | return; |
| 105 | |
| 106 | while (arch_data->iommu_dev) { |
| 107 | obj = arch_data->iommu_dev; |
| 108 | p = obj->ctx; |
| 109 | for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) { |
| 110 | iommu_write_reg(obj, p[i], i * sizeof(u32)); |
| 111 | dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, |
| 112 | p[i]); |
| 113 | } |
| 114 | arch_data++; |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 115 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 116 | } |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 117 | EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 118 | |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 119 | static void dra7_cfg_dspsys_mmu(struct omap_iommu *obj, bool enable) |
| 120 | { |
| 121 | u32 val, mask; |
| 122 | |
| 123 | if (!obj->syscfg) |
| 124 | return; |
| 125 | |
| 126 | mask = (1 << (obj->id * DSP_SYS_MMU_CONFIG_EN_SHIFT)); |
| 127 | val = enable ? mask : 0; |
| 128 | regmap_update_bits(obj->syscfg, DSP_SYS_MMU_CONFIG, mask, val); |
| 129 | } |
| 130 | |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 131 | static void __iommu_set_twl(struct omap_iommu *obj, bool on) |
| 132 | { |
| 133 | u32 l = iommu_read_reg(obj, MMU_CNTL); |
| 134 | |
| 135 | if (on) |
| 136 | iommu_write_reg(obj, MMU_IRQ_TWL_MASK, MMU_IRQENABLE); |
| 137 | else |
| 138 | iommu_write_reg(obj, MMU_IRQ_TLB_MISS_MASK, MMU_IRQENABLE); |
| 139 | |
| 140 | l &= ~MMU_CNTL_MASK; |
| 141 | if (on) |
| 142 | l |= (MMU_CNTL_MMU_EN | MMU_CNTL_TWL_EN); |
| 143 | else |
| 144 | l |= (MMU_CNTL_MMU_EN); |
| 145 | |
| 146 | iommu_write_reg(obj, l, MMU_CNTL); |
| 147 | } |
| 148 | |
| 149 | static int omap2_iommu_enable(struct omap_iommu *obj) |
| 150 | { |
| 151 | u32 l, pa; |
| 152 | |
| 153 | if (!obj->iopgd || !IS_ALIGNED((u32)obj->iopgd, SZ_16K)) |
| 154 | return -EINVAL; |
| 155 | |
| 156 | pa = virt_to_phys(obj->iopgd); |
| 157 | if (!IS_ALIGNED(pa, SZ_16K)) |
| 158 | return -EINVAL; |
| 159 | |
| 160 | l = iommu_read_reg(obj, MMU_REVISION); |
| 161 | dev_info(obj->dev, "%s: version %d.%d\n", obj->name, |
| 162 | (l >> 4) & 0xf, l & 0xf); |
| 163 | |
| 164 | iommu_write_reg(obj, pa, MMU_TTB); |
| 165 | |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 166 | dra7_cfg_dspsys_mmu(obj, true); |
| 167 | |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 168 | if (obj->has_bus_err_back) |
| 169 | iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG); |
| 170 | |
| 171 | __iommu_set_twl(obj, true); |
| 172 | |
| 173 | return 0; |
| 174 | } |
| 175 | |
| 176 | static void omap2_iommu_disable(struct omap_iommu *obj) |
| 177 | { |
| 178 | u32 l = iommu_read_reg(obj, MMU_CNTL); |
| 179 | |
| 180 | l &= ~MMU_CNTL_MASK; |
| 181 | iommu_write_reg(obj, l, MMU_CNTL); |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 182 | dra7_cfg_dspsys_mmu(obj, false); |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 183 | |
| 184 | dev_dbg(obj->dev, "%s is shutting down\n", obj->name); |
| 185 | } |
| 186 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 187 | static int iommu_enable(struct omap_iommu *obj) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 188 | { |
Suman Anna | db8918f | 2019-08-07 11:26:47 +0300 | [diff] [blame^] | 189 | int ret; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 190 | |
Suman Anna | db8918f | 2019-08-07 11:26:47 +0300 | [diff] [blame^] | 191 | ret = pm_runtime_get_sync(obj->dev); |
| 192 | if (ret < 0) |
| 193 | pm_runtime_put_noidle(obj->dev); |
Suman Anna | 3846a3b9 | 2019-08-07 11:26:45 +0300 | [diff] [blame] | 194 | |
Suman Anna | db8918f | 2019-08-07 11:26:47 +0300 | [diff] [blame^] | 195 | return ret < 0 ? ret : 0; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 196 | } |
| 197 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 198 | static void iommu_disable(struct omap_iommu *obj) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 199 | { |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 200 | pm_runtime_put_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 201 | } |
| 202 | |
| 203 | /* |
| 204 | * TLB operations |
| 205 | */ |
Ohad Ben-Cohen | e1f2381 | 2011-08-16 14:58:14 +0300 | [diff] [blame] | 206 | static u32 iotlb_cr_to_virt(struct cr_regs *cr) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 207 | { |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 208 | u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK; |
| 209 | u32 mask = get_cam_va_mask(cr->cam & page_size); |
| 210 | |
| 211 | return cr->cam & mask; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 212 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 213 | |
| 214 | static u32 get_iopte_attr(struct iotlb_entry *e) |
| 215 | { |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 216 | u32 attr; |
| 217 | |
| 218 | attr = e->mixed << 5; |
| 219 | attr |= e->endian; |
| 220 | attr |= e->elsz >> 3; |
| 221 | attr <<= (((e->pgsz == MMU_CAM_PGSZ_4K) || |
| 222 | (e->pgsz == MMU_CAM_PGSZ_64K)) ? 0 : 6); |
| 223 | return attr; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 224 | } |
| 225 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 226 | static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 227 | { |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 228 | u32 status, fault_addr; |
| 229 | |
| 230 | status = iommu_read_reg(obj, MMU_IRQSTATUS); |
| 231 | status &= MMU_IRQ_MASK; |
| 232 | if (!status) { |
| 233 | *da = 0; |
| 234 | return 0; |
| 235 | } |
| 236 | |
| 237 | fault_addr = iommu_read_reg(obj, MMU_FAULT_AD); |
| 238 | *da = fault_addr; |
| 239 | |
| 240 | iommu_write_reg(obj, status, MMU_IRQSTATUS); |
| 241 | |
| 242 | return status; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 243 | } |
| 244 | |
Suman Anna | 69c2c19 | 2015-07-20 17:33:25 -0500 | [diff] [blame] | 245 | void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 246 | { |
| 247 | u32 val; |
| 248 | |
| 249 | val = iommu_read_reg(obj, MMU_LOCK); |
| 250 | |
| 251 | l->base = MMU_LOCK_BASE(val); |
| 252 | l->vict = MMU_LOCK_VICT(val); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 253 | } |
| 254 | |
Suman Anna | 69c2c19 | 2015-07-20 17:33:25 -0500 | [diff] [blame] | 255 | void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 256 | { |
| 257 | u32 val; |
| 258 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 259 | val = (l->base << MMU_LOCK_BASE_SHIFT); |
| 260 | val |= (l->vict << MMU_LOCK_VICT_SHIFT); |
| 261 | |
| 262 | iommu_write_reg(obj, val, MMU_LOCK); |
| 263 | } |
| 264 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 265 | static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 266 | { |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 267 | cr->cam = iommu_read_reg(obj, MMU_READ_CAM); |
| 268 | cr->ram = iommu_read_reg(obj, MMU_READ_RAM); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 269 | } |
| 270 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 271 | static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 272 | { |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 273 | iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM); |
| 274 | iommu_write_reg(obj, cr->ram, MMU_RAM); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 275 | |
| 276 | iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); |
| 277 | iommu_write_reg(obj, 1, MMU_LD_TLB); |
| 278 | } |
| 279 | |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 280 | /* only used in iotlb iteration for-loop */ |
Suman Anna | 69c2c19 | 2015-07-20 17:33:25 -0500 | [diff] [blame] | 281 | struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n) |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 282 | { |
| 283 | struct cr_regs cr; |
| 284 | struct iotlb_lock l; |
| 285 | |
| 286 | iotlb_lock_get(obj, &l); |
| 287 | l.vict = n; |
| 288 | iotlb_lock_set(obj, &l); |
| 289 | iotlb_read_cr(obj, &cr); |
| 290 | |
| 291 | return cr; |
| 292 | } |
| 293 | |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 294 | #ifdef PREFETCH_IOTLB |
| 295 | static struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj, |
| 296 | struct iotlb_entry *e) |
| 297 | { |
| 298 | struct cr_regs *cr; |
| 299 | |
| 300 | if (!e) |
| 301 | return NULL; |
| 302 | |
| 303 | if (e->da & ~(get_cam_va_mask(e->pgsz))) { |
| 304 | dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__, |
| 305 | e->da); |
| 306 | return ERR_PTR(-EINVAL); |
| 307 | } |
| 308 | |
| 309 | cr = kmalloc(sizeof(*cr), GFP_KERNEL); |
| 310 | if (!cr) |
| 311 | return ERR_PTR(-ENOMEM); |
| 312 | |
| 313 | cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid; |
| 314 | cr->ram = e->pa | e->endian | e->elsz | e->mixed; |
| 315 | |
| 316 | return cr; |
| 317 | } |
| 318 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 319 | /** |
| 320 | * load_iotlb_entry - Set an iommu tlb entry |
| 321 | * @obj: target iommu |
| 322 | * @e: an iommu tlb entry info |
| 323 | **/ |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 324 | static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 325 | { |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 326 | int err = 0; |
| 327 | struct iotlb_lock l; |
| 328 | struct cr_regs *cr; |
| 329 | |
| 330 | if (!obj || !obj->nr_tlb_entries || !e) |
| 331 | return -EINVAL; |
| 332 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 333 | pm_runtime_get_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 334 | |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 335 | iotlb_lock_get(obj, &l); |
| 336 | if (l.base == obj->nr_tlb_entries) { |
| 337 | dev_warn(obj->dev, "%s: preserve entries full\n", __func__); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 338 | err = -EBUSY; |
| 339 | goto out; |
| 340 | } |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 341 | if (!e->prsvd) { |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 342 | int i; |
| 343 | struct cr_regs tmp; |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 344 | |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 345 | for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp) |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 346 | if (!iotlb_cr_valid(&tmp)) |
| 347 | break; |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 348 | |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 349 | if (i == obj->nr_tlb_entries) { |
| 350 | dev_dbg(obj->dev, "%s: full: no entry\n", __func__); |
| 351 | err = -EBUSY; |
| 352 | goto out; |
| 353 | } |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 354 | |
| 355 | iotlb_lock_get(obj, &l); |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 356 | } else { |
| 357 | l.vict = l.base; |
| 358 | iotlb_lock_set(obj, &l); |
| 359 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 360 | |
| 361 | cr = iotlb_alloc_cr(obj, e); |
| 362 | if (IS_ERR(cr)) { |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 363 | pm_runtime_put_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 364 | return PTR_ERR(cr); |
| 365 | } |
| 366 | |
| 367 | iotlb_load_cr(obj, cr); |
| 368 | kfree(cr); |
| 369 | |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 370 | if (e->prsvd) |
| 371 | l.base++; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 372 | /* increment victim for next tlb load */ |
| 373 | if (++l.vict == obj->nr_tlb_entries) |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 374 | l.vict = l.base; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 375 | iotlb_lock_set(obj, &l); |
| 376 | out: |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 377 | pm_runtime_put_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 378 | return err; |
| 379 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 380 | |
Ohad Ben-Cohen | 5da14a4 | 2011-08-16 15:19:10 +0300 | [diff] [blame] | 381 | #else /* !PREFETCH_IOTLB */ |
| 382 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 383 | static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
Ohad Ben-Cohen | 5da14a4 | 2011-08-16 15:19:10 +0300 | [diff] [blame] | 384 | { |
| 385 | return 0; |
| 386 | } |
| 387 | |
| 388 | #endif /* !PREFETCH_IOTLB */ |
| 389 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 390 | static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
Ohad Ben-Cohen | 5da14a4 | 2011-08-16 15:19:10 +0300 | [diff] [blame] | 391 | { |
| 392 | return load_iotlb_entry(obj, e); |
| 393 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 394 | |
| 395 | /** |
| 396 | * flush_iotlb_page - Clear an iommu tlb entry |
| 397 | * @obj: target iommu |
| 398 | * @da: iommu device virtual address |
| 399 | * |
| 400 | * Clear an iommu tlb entry which includes 'da' address. |
| 401 | **/ |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 402 | static void flush_iotlb_page(struct omap_iommu *obj, u32 da) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 403 | { |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 404 | int i; |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 405 | struct cr_regs cr; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 406 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 407 | pm_runtime_get_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 408 | |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 409 | for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) { |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 410 | u32 start; |
| 411 | size_t bytes; |
| 412 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 413 | if (!iotlb_cr_valid(&cr)) |
| 414 | continue; |
| 415 | |
| 416 | start = iotlb_cr_to_virt(&cr); |
| 417 | bytes = iopgsz_to_bytes(cr.cam & 3); |
| 418 | |
| 419 | if ((start <= da) && (da < start + bytes)) { |
| 420 | dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n", |
| 421 | __func__, start, da, bytes); |
Hari Kanigeri | 0fa035e | 2010-08-20 13:50:18 +0000 | [diff] [blame] | 422 | iotlb_load_cr(obj, &cr); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 423 | iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); |
Laurent Pinchart | f7129a0 | 2014-03-07 23:47:03 +0100 | [diff] [blame] | 424 | break; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 425 | } |
| 426 | } |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 427 | pm_runtime_put_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 428 | |
| 429 | if (i == obj->nr_tlb_entries) |
| 430 | dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da); |
| 431 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 432 | |
| 433 | /** |
| 434 | * flush_iotlb_all - Clear all iommu tlb entries |
| 435 | * @obj: target iommu |
| 436 | **/ |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 437 | static void flush_iotlb_all(struct omap_iommu *obj) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 438 | { |
| 439 | struct iotlb_lock l; |
| 440 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 441 | pm_runtime_get_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 442 | |
| 443 | l.base = 0; |
| 444 | l.vict = 0; |
| 445 | iotlb_lock_set(obj, &l); |
| 446 | |
| 447 | iommu_write_reg(obj, 1, MMU_GFLUSH); |
| 448 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 449 | pm_runtime_put_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 450 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 451 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 452 | /* |
| 453 | * H/W pagetable operations |
| 454 | */ |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 455 | static void flush_iopte_range(struct device *dev, dma_addr_t dma, |
| 456 | unsigned long offset, int num_entries) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 457 | { |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 458 | size_t size = num_entries * sizeof(u32); |
| 459 | |
| 460 | dma_sync_single_range_for_device(dev, dma, offset, size, DMA_TO_DEVICE); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 461 | } |
| 462 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 463 | static void iopte_free(struct omap_iommu *obj, u32 *iopte, bool dma_valid) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 464 | { |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 465 | dma_addr_t pt_dma; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 466 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 467 | /* Note: freed iopte's must be clean ready for re-use */ |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 468 | if (iopte) { |
| 469 | if (dma_valid) { |
| 470 | pt_dma = virt_to_phys(iopte); |
| 471 | dma_unmap_single(obj->dev, pt_dma, IOPTE_TABLE_SIZE, |
| 472 | DMA_TO_DEVICE); |
| 473 | } |
| 474 | |
Zhouyi Zhou | e28045a | 2014-03-05 18:20:19 +0800 | [diff] [blame] | 475 | kmem_cache_free(iopte_cachep, iopte); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 476 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 477 | } |
| 478 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 479 | static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, |
| 480 | dma_addr_t *pt_dma, u32 da) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 481 | { |
| 482 | u32 *iopte; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 483 | unsigned long offset = iopgd_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 484 | |
| 485 | /* a table has already existed */ |
| 486 | if (*iopgd) |
| 487 | goto pte_ready; |
| 488 | |
| 489 | /* |
| 490 | * do the allocation outside the page table lock |
| 491 | */ |
| 492 | spin_unlock(&obj->page_table_lock); |
| 493 | iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL); |
| 494 | spin_lock(&obj->page_table_lock); |
| 495 | |
| 496 | if (!*iopgd) { |
| 497 | if (!iopte) |
| 498 | return ERR_PTR(-ENOMEM); |
| 499 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 500 | *pt_dma = dma_map_single(obj->dev, iopte, IOPTE_TABLE_SIZE, |
| 501 | DMA_TO_DEVICE); |
| 502 | if (dma_mapping_error(obj->dev, *pt_dma)) { |
| 503 | dev_err(obj->dev, "DMA map error for L2 table\n"); |
| 504 | iopte_free(obj, iopte, false); |
| 505 | return ERR_PTR(-ENOMEM); |
| 506 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 507 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 508 | /* |
| 509 | * we rely on dma address and the physical address to be |
| 510 | * the same for mapping the L2 table |
| 511 | */ |
| 512 | if (WARN_ON(*pt_dma != virt_to_phys(iopte))) { |
| 513 | dev_err(obj->dev, "DMA translation error for L2 table\n"); |
| 514 | dma_unmap_single(obj->dev, *pt_dma, IOPTE_TABLE_SIZE, |
| 515 | DMA_TO_DEVICE); |
| 516 | iopte_free(obj, iopte, false); |
| 517 | return ERR_PTR(-ENOMEM); |
| 518 | } |
| 519 | |
| 520 | *iopgd = virt_to_phys(iopte) | IOPGD_TABLE; |
| 521 | |
| 522 | flush_iopte_range(obj->dev, obj->pd_dma, offset, 1); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 523 | dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte); |
| 524 | } else { |
| 525 | /* We raced, free the reduniovant table */ |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 526 | iopte_free(obj, iopte, false); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 527 | } |
| 528 | |
| 529 | pte_ready: |
| 530 | iopte = iopte_offset(iopgd, da); |
Ralf Goebel | 04c532a | 2018-08-06 17:00:36 +0200 | [diff] [blame] | 531 | *pt_dma = iopgd_page_paddr(iopgd); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 532 | dev_vdbg(obj->dev, |
| 533 | "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n", |
| 534 | __func__, da, iopgd, *iopgd, iopte, *iopte); |
| 535 | |
| 536 | return iopte; |
| 537 | } |
| 538 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 539 | static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 540 | { |
| 541 | u32 *iopgd = iopgd_offset(obj, da); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 542 | unsigned long offset = iopgd_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 543 | |
Hiroshi DOYU | 4abb761 | 2010-05-06 18:24:04 +0300 | [diff] [blame] | 544 | if ((da | pa) & ~IOSECTION_MASK) { |
| 545 | dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", |
| 546 | __func__, da, pa, IOSECTION_SIZE); |
| 547 | return -EINVAL; |
| 548 | } |
| 549 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 550 | *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 551 | flush_iopte_range(obj->dev, obj->pd_dma, offset, 1); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 552 | return 0; |
| 553 | } |
| 554 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 555 | static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 556 | { |
| 557 | u32 *iopgd = iopgd_offset(obj, da); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 558 | unsigned long offset = iopgd_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 559 | int i; |
| 560 | |
Hiroshi DOYU | 4abb761 | 2010-05-06 18:24:04 +0300 | [diff] [blame] | 561 | if ((da | pa) & ~IOSUPER_MASK) { |
| 562 | dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", |
| 563 | __func__, da, pa, IOSUPER_SIZE); |
| 564 | return -EINVAL; |
| 565 | } |
| 566 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 567 | for (i = 0; i < 16; i++) |
| 568 | *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 569 | flush_iopte_range(obj->dev, obj->pd_dma, offset, 16); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 570 | return 0; |
| 571 | } |
| 572 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 573 | static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 574 | { |
| 575 | u32 *iopgd = iopgd_offset(obj, da); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 576 | dma_addr_t pt_dma; |
| 577 | u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da); |
| 578 | unsigned long offset = iopte_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 579 | |
| 580 | if (IS_ERR(iopte)) |
| 581 | return PTR_ERR(iopte); |
| 582 | |
| 583 | *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 584 | flush_iopte_range(obj->dev, pt_dma, offset, 1); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 585 | |
| 586 | dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n", |
| 587 | __func__, da, pa, iopte, *iopte); |
| 588 | |
| 589 | return 0; |
| 590 | } |
| 591 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 592 | static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 593 | { |
| 594 | u32 *iopgd = iopgd_offset(obj, da); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 595 | dma_addr_t pt_dma; |
| 596 | u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da); |
| 597 | unsigned long offset = iopte_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 598 | int i; |
| 599 | |
Hiroshi DOYU | 4abb761 | 2010-05-06 18:24:04 +0300 | [diff] [blame] | 600 | if ((da | pa) & ~IOLARGE_MASK) { |
| 601 | dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", |
| 602 | __func__, da, pa, IOLARGE_SIZE); |
| 603 | return -EINVAL; |
| 604 | } |
| 605 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 606 | if (IS_ERR(iopte)) |
| 607 | return PTR_ERR(iopte); |
| 608 | |
| 609 | for (i = 0; i < 16; i++) |
| 610 | *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 611 | flush_iopte_range(obj->dev, pt_dma, offset, 16); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 612 | return 0; |
| 613 | } |
| 614 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 615 | static int |
| 616 | iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 617 | { |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 618 | int (*fn)(struct omap_iommu *, u32, u32, u32); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 619 | u32 prot; |
| 620 | int err; |
| 621 | |
| 622 | if (!obj || !e) |
| 623 | return -EINVAL; |
| 624 | |
| 625 | switch (e->pgsz) { |
| 626 | case MMU_CAM_PGSZ_16M: |
| 627 | fn = iopgd_alloc_super; |
| 628 | break; |
| 629 | case MMU_CAM_PGSZ_1M: |
| 630 | fn = iopgd_alloc_section; |
| 631 | break; |
| 632 | case MMU_CAM_PGSZ_64K: |
| 633 | fn = iopte_alloc_large; |
| 634 | break; |
| 635 | case MMU_CAM_PGSZ_4K: |
| 636 | fn = iopte_alloc_page; |
| 637 | break; |
| 638 | default: |
| 639 | fn = NULL; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 640 | break; |
| 641 | } |
| 642 | |
Suman Anna | 7c1ab60 | 2016-04-04 17:46:19 -0500 | [diff] [blame] | 643 | if (WARN_ON(!fn)) |
| 644 | return -EINVAL; |
| 645 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 646 | prot = get_iopte_attr(e); |
| 647 | |
| 648 | spin_lock(&obj->page_table_lock); |
| 649 | err = fn(obj, e->da, e->pa, prot); |
| 650 | spin_unlock(&obj->page_table_lock); |
| 651 | |
| 652 | return err; |
| 653 | } |
| 654 | |
| 655 | /** |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 656 | * omap_iopgtable_store_entry - Make an iommu pte entry |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 657 | * @obj: target iommu |
| 658 | * @e: an iommu tlb entry info |
| 659 | **/ |
Suman Anna | 4899a56 | 2014-10-22 17:22:32 -0500 | [diff] [blame] | 660 | static int |
| 661 | omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 662 | { |
| 663 | int err; |
| 664 | |
| 665 | flush_iotlb_page(obj, e->da); |
| 666 | err = iopgtable_store_entry_core(obj, e); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 667 | if (!err) |
Ohad Ben-Cohen | 5da14a4 | 2011-08-16 15:19:10 +0300 | [diff] [blame] | 668 | prefetch_iotlb_entry(obj, e); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 669 | return err; |
| 670 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 671 | |
| 672 | /** |
| 673 | * iopgtable_lookup_entry - Lookup an iommu pte entry |
| 674 | * @obj: target iommu |
| 675 | * @da: iommu device virtual address |
| 676 | * @ppgd: iommu pgd entry pointer to be returned |
| 677 | * @ppte: iommu pte entry pointer to be returned |
| 678 | **/ |
Ohad Ben-Cohen | e1f2381 | 2011-08-16 14:58:14 +0300 | [diff] [blame] | 679 | static void |
| 680 | iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 681 | { |
| 682 | u32 *iopgd, *iopte = NULL; |
| 683 | |
| 684 | iopgd = iopgd_offset(obj, da); |
| 685 | if (!*iopgd) |
| 686 | goto out; |
| 687 | |
Hiroshi DOYU | a1a5445 | 2010-05-13 09:45:35 +0300 | [diff] [blame] | 688 | if (iopgd_is_table(*iopgd)) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 689 | iopte = iopte_offset(iopgd, da); |
| 690 | out: |
| 691 | *ppgd = iopgd; |
| 692 | *ppte = iopte; |
| 693 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 694 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 695 | static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 696 | { |
| 697 | size_t bytes; |
| 698 | u32 *iopgd = iopgd_offset(obj, da); |
| 699 | int nent = 1; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 700 | dma_addr_t pt_dma; |
| 701 | unsigned long pd_offset = iopgd_index(da) * sizeof(da); |
| 702 | unsigned long pt_offset = iopte_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 703 | |
| 704 | if (!*iopgd) |
| 705 | return 0; |
| 706 | |
Hiroshi DOYU | a1a5445 | 2010-05-13 09:45:35 +0300 | [diff] [blame] | 707 | if (iopgd_is_table(*iopgd)) { |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 708 | int i; |
| 709 | u32 *iopte = iopte_offset(iopgd, da); |
| 710 | |
| 711 | bytes = IOPTE_SIZE; |
| 712 | if (*iopte & IOPTE_LARGE) { |
| 713 | nent *= 16; |
| 714 | /* rewind to the 1st entry */ |
Hiroshi DOYU | c127c7d | 2010-02-15 10:03:32 -0800 | [diff] [blame] | 715 | iopte = iopte_offset(iopgd, (da & IOLARGE_MASK)); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 716 | } |
| 717 | bytes *= nent; |
| 718 | memset(iopte, 0, nent * sizeof(*iopte)); |
Ralf Goebel | 04c532a | 2018-08-06 17:00:36 +0200 | [diff] [blame] | 719 | pt_dma = iopgd_page_paddr(iopgd); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 720 | flush_iopte_range(obj->dev, pt_dma, pt_offset, nent); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 721 | |
| 722 | /* |
| 723 | * do table walk to check if this table is necessary or not |
| 724 | */ |
| 725 | iopte = iopte_offset(iopgd, 0); |
| 726 | for (i = 0; i < PTRS_PER_IOPTE; i++) |
| 727 | if (iopte[i]) |
| 728 | goto out; |
| 729 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 730 | iopte_free(obj, iopte, true); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 731 | nent = 1; /* for the next L1 entry */ |
| 732 | } else { |
| 733 | bytes = IOPGD_SIZE; |
Hiroshi DOYU | dcc730d | 2009-10-22 14:46:32 -0700 | [diff] [blame] | 734 | if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) { |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 735 | nent *= 16; |
| 736 | /* rewind to the 1st entry */ |
Hiroshi DOYU | 8d33ea5 | 2010-02-15 10:03:32 -0800 | [diff] [blame] | 737 | iopgd = iopgd_offset(obj, (da & IOSUPER_MASK)); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 738 | } |
| 739 | bytes *= nent; |
| 740 | } |
| 741 | memset(iopgd, 0, nent * sizeof(*iopgd)); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 742 | flush_iopte_range(obj->dev, obj->pd_dma, pd_offset, nent); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 743 | out: |
| 744 | return bytes; |
| 745 | } |
| 746 | |
| 747 | /** |
| 748 | * iopgtable_clear_entry - Remove an iommu pte entry |
| 749 | * @obj: target iommu |
| 750 | * @da: iommu device virtual address |
| 751 | **/ |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 752 | static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 753 | { |
| 754 | size_t bytes; |
| 755 | |
| 756 | spin_lock(&obj->page_table_lock); |
| 757 | |
| 758 | bytes = iopgtable_clear_entry_core(obj, da); |
| 759 | flush_iotlb_page(obj, da); |
| 760 | |
| 761 | spin_unlock(&obj->page_table_lock); |
| 762 | |
| 763 | return bytes; |
| 764 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 765 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 766 | static void iopgtable_clear_entry_all(struct omap_iommu *obj) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 767 | { |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 768 | unsigned long offset; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 769 | int i; |
| 770 | |
| 771 | spin_lock(&obj->page_table_lock); |
| 772 | |
| 773 | for (i = 0; i < PTRS_PER_IOPGD; i++) { |
| 774 | u32 da; |
| 775 | u32 *iopgd; |
| 776 | |
| 777 | da = i << IOPGD_SHIFT; |
| 778 | iopgd = iopgd_offset(obj, da); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 779 | offset = iopgd_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 780 | |
| 781 | if (!*iopgd) |
| 782 | continue; |
| 783 | |
Hiroshi DOYU | a1a5445 | 2010-05-13 09:45:35 +0300 | [diff] [blame] | 784 | if (iopgd_is_table(*iopgd)) |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 785 | iopte_free(obj, iopte_offset(iopgd, 0), true); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 786 | |
| 787 | *iopgd = 0; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 788 | flush_iopte_range(obj->dev, obj->pd_dma, offset, 1); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 789 | } |
| 790 | |
| 791 | flush_iotlb_all(obj); |
| 792 | |
| 793 | spin_unlock(&obj->page_table_lock); |
| 794 | } |
| 795 | |
| 796 | /* |
| 797 | * Device IOMMU generic operations |
| 798 | */ |
| 799 | static irqreturn_t iommu_fault_handler(int irq, void *data) |
| 800 | { |
David Cohen | d594f1f | 2011-02-16 19:35:51 +0000 | [diff] [blame] | 801 | u32 da, errs; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 802 | u32 *iopgd, *iopte; |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 803 | struct omap_iommu *obj = data; |
Ohad Ben-Cohen | e7f10f0 | 2011-09-13 15:26:29 -0400 | [diff] [blame] | 804 | struct iommu_domain *domain = obj->domain; |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 805 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 806 | |
Suman Anna | 0d36428 | 2017-09-05 17:56:17 -0500 | [diff] [blame] | 807 | if (!omap_domain->dev) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 808 | return IRQ_NONE; |
| 809 | |
David Cohen | d594f1f | 2011-02-16 19:35:51 +0000 | [diff] [blame] | 810 | errs = iommu_report_fault(obj, &da); |
Laurent Pinchart | c56b2dd | 2011-05-10 16:56:46 +0200 | [diff] [blame] | 811 | if (errs == 0) |
| 812 | return IRQ_HANDLED; |
David Cohen | d594f1f | 2011-02-16 19:35:51 +0000 | [diff] [blame] | 813 | |
| 814 | /* Fault callback or TLB/PTE Dynamic loading */ |
Ohad Ben-Cohen | e7f10f0 | 2011-09-13 15:26:29 -0400 | [diff] [blame] | 815 | if (!report_iommu_fault(domain, obj->dev, da, 0)) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 816 | return IRQ_HANDLED; |
| 817 | |
Fernando Guzman Lugo | 159d3e3 | 2017-07-28 15:49:13 -0500 | [diff] [blame] | 818 | iommu_write_reg(obj, 0, MMU_IRQENABLE); |
Hiroshi DOYU | 37b2981 | 2010-05-24 02:01:52 +0000 | [diff] [blame] | 819 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 820 | iopgd = iopgd_offset(obj, da); |
| 821 | |
Hiroshi DOYU | a1a5445 | 2010-05-13 09:45:35 +0300 | [diff] [blame] | 822 | if (!iopgd_is_table(*iopgd)) { |
Suman Anna | b6c2e09 | 2013-05-30 18:10:59 -0500 | [diff] [blame] | 823 | dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n", |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 824 | obj->name, errs, da, iopgd, *iopgd); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 825 | return IRQ_NONE; |
| 826 | } |
| 827 | |
| 828 | iopte = iopte_offset(iopgd, da); |
| 829 | |
Suman Anna | b6c2e09 | 2013-05-30 18:10:59 -0500 | [diff] [blame] | 830 | dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n", |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 831 | obj->name, errs, da, iopgd, *iopgd, iopte, *iopte); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 832 | |
| 833 | return IRQ_NONE; |
| 834 | } |
| 835 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 836 | /** |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 837 | * omap_iommu_attach() - attach iommu device to an iommu domain |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 838 | * @obj: target omap iommu device |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 839 | * @iopgd: page table |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 840 | **/ |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 841 | static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 842 | { |
Suman Anna | 7ee08b9e | 2014-02-28 14:42:33 -0600 | [diff] [blame] | 843 | int err; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 844 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 845 | spin_lock(&obj->iommu_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 846 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 847 | obj->pd_dma = dma_map_single(obj->dev, iopgd, IOPGD_TABLE_SIZE, |
| 848 | DMA_TO_DEVICE); |
| 849 | if (dma_mapping_error(obj->dev, obj->pd_dma)) { |
| 850 | dev_err(obj->dev, "DMA map error for L1 table\n"); |
| 851 | err = -ENOMEM; |
| 852 | goto out_err; |
| 853 | } |
| 854 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 855 | obj->iopgd = iopgd; |
| 856 | err = iommu_enable(obj); |
| 857 | if (err) |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 858 | goto out_err; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 859 | flush_iotlb_all(obj); |
| 860 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 861 | spin_unlock(&obj->iommu_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 862 | |
| 863 | dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 864 | |
| 865 | return 0; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 866 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 867 | out_err: |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 868 | spin_unlock(&obj->iommu_lock); |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 869 | |
| 870 | return err; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 871 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 872 | |
| 873 | /** |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 874 | * omap_iommu_detach - release iommu device |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 875 | * @obj: target iommu |
| 876 | **/ |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 877 | static void omap_iommu_detach(struct omap_iommu *obj) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 878 | { |
Roel Kluin | acf9d46 | 2010-01-08 10:29:05 -0800 | [diff] [blame] | 879 | if (!obj || IS_ERR(obj)) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 880 | return; |
| 881 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 882 | spin_lock(&obj->iommu_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 883 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 884 | dma_unmap_single(obj->dev, obj->pd_dma, IOPGD_TABLE_SIZE, |
| 885 | DMA_TO_DEVICE); |
Suman Anna | 2088ecb | 2014-10-22 17:22:19 -0500 | [diff] [blame] | 886 | iommu_disable(obj); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 887 | obj->pd_dma = 0; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 888 | obj->iopgd = NULL; |
| 889 | |
| 890 | spin_unlock(&obj->iommu_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 891 | |
| 892 | dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); |
| 893 | } |
David Cohen | d594f1f | 2011-02-16 19:35:51 +0000 | [diff] [blame] | 894 | |
Suman Anna | db8918f | 2019-08-07 11:26:47 +0300 | [diff] [blame^] | 895 | /** |
| 896 | * omap_iommu_runtime_suspend - disable an iommu device |
| 897 | * @dev: iommu device |
| 898 | * |
| 899 | * This function performs all that is necessary to disable an |
| 900 | * IOMMU device, either during final detachment from a client |
| 901 | * device, or during system/runtime suspend of the device. This |
| 902 | * includes programming all the appropriate IOMMU registers, and |
| 903 | * managing the associated omap_hwmod's state and the device's |
| 904 | * reset line. |
| 905 | **/ |
| 906 | static int omap_iommu_runtime_suspend(struct device *dev) |
| 907 | { |
| 908 | struct platform_device *pdev = to_platform_device(dev); |
| 909 | struct iommu_platform_data *pdata = dev_get_platdata(dev); |
| 910 | struct omap_iommu *obj = to_iommu(dev); |
| 911 | int ret; |
| 912 | |
| 913 | omap2_iommu_disable(obj); |
| 914 | |
| 915 | if (pdata && pdata->device_idle) |
| 916 | pdata->device_idle(pdev); |
| 917 | |
| 918 | if (pdata && pdata->assert_reset) |
| 919 | pdata->assert_reset(pdev, pdata->reset_name); |
| 920 | |
| 921 | if (pdata && pdata->set_pwrdm_constraint) { |
| 922 | ret = pdata->set_pwrdm_constraint(pdev, false, &obj->pwrst); |
| 923 | if (ret) { |
| 924 | dev_warn(obj->dev, "pwrdm_constraint failed to be reset, status = %d\n", |
| 925 | ret); |
| 926 | } |
| 927 | } |
| 928 | |
| 929 | return 0; |
| 930 | } |
| 931 | |
| 932 | /** |
| 933 | * omap_iommu_runtime_resume - enable an iommu device |
| 934 | * @dev: iommu device |
| 935 | * |
| 936 | * This function performs all that is necessary to enable an |
| 937 | * IOMMU device, either during initial attachment to a client |
| 938 | * device, or during system/runtime resume of the device. This |
| 939 | * includes programming all the appropriate IOMMU registers, and |
| 940 | * managing the associated omap_hwmod's state and the device's |
| 941 | * reset line. |
| 942 | **/ |
| 943 | static int omap_iommu_runtime_resume(struct device *dev) |
| 944 | { |
| 945 | struct platform_device *pdev = to_platform_device(dev); |
| 946 | struct iommu_platform_data *pdata = dev_get_platdata(dev); |
| 947 | struct omap_iommu *obj = to_iommu(dev); |
| 948 | int ret = 0; |
| 949 | |
| 950 | if (pdata && pdata->set_pwrdm_constraint) { |
| 951 | ret = pdata->set_pwrdm_constraint(pdev, true, &obj->pwrst); |
| 952 | if (ret) { |
| 953 | dev_warn(obj->dev, "pwrdm_constraint failed to be set, status = %d\n", |
| 954 | ret); |
| 955 | } |
| 956 | } |
| 957 | |
| 958 | if (pdata && pdata->deassert_reset) { |
| 959 | ret = pdata->deassert_reset(pdev, pdata->reset_name); |
| 960 | if (ret) { |
| 961 | dev_err(dev, "deassert_reset failed: %d\n", ret); |
| 962 | return ret; |
| 963 | } |
| 964 | } |
| 965 | |
| 966 | if (pdata && pdata->device_enable) |
| 967 | pdata->device_enable(pdev); |
| 968 | |
| 969 | ret = omap2_iommu_enable(obj); |
| 970 | |
| 971 | return ret; |
| 972 | } |
| 973 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 974 | static bool omap_iommu_can_register(struct platform_device *pdev) |
| 975 | { |
| 976 | struct device_node *np = pdev->dev.of_node; |
| 977 | |
| 978 | if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu")) |
| 979 | return true; |
| 980 | |
| 981 | /* |
| 982 | * restrict IOMMU core registration only for processor-port MDMA MMUs |
| 983 | * on DRA7 DSPs |
| 984 | */ |
| 985 | if ((!strcmp(dev_name(&pdev->dev), "40d01000.mmu")) || |
| 986 | (!strcmp(dev_name(&pdev->dev), "41501000.mmu"))) |
| 987 | return true; |
| 988 | |
| 989 | return false; |
| 990 | } |
| 991 | |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 992 | static int omap_iommu_dra7_get_dsp_system_cfg(struct platform_device *pdev, |
| 993 | struct omap_iommu *obj) |
| 994 | { |
| 995 | struct device_node *np = pdev->dev.of_node; |
| 996 | int ret; |
| 997 | |
| 998 | if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu")) |
| 999 | return 0; |
| 1000 | |
| 1001 | if (!of_property_read_bool(np, "ti,syscon-mmuconfig")) { |
| 1002 | dev_err(&pdev->dev, "ti,syscon-mmuconfig property is missing\n"); |
| 1003 | return -EINVAL; |
| 1004 | } |
| 1005 | |
| 1006 | obj->syscfg = |
| 1007 | syscon_regmap_lookup_by_phandle(np, "ti,syscon-mmuconfig"); |
| 1008 | if (IS_ERR(obj->syscfg)) { |
| 1009 | /* can fail with -EPROBE_DEFER */ |
| 1010 | ret = PTR_ERR(obj->syscfg); |
| 1011 | return ret; |
| 1012 | } |
| 1013 | |
| 1014 | if (of_property_read_u32_index(np, "ti,syscon-mmuconfig", 1, |
| 1015 | &obj->id)) { |
| 1016 | dev_err(&pdev->dev, "couldn't get the IOMMU instance id within subsystem\n"); |
| 1017 | return -EINVAL; |
| 1018 | } |
| 1019 | |
| 1020 | if (obj->id != 0 && obj->id != 1) { |
| 1021 | dev_err(&pdev->dev, "invalid IOMMU instance id\n"); |
| 1022 | return -EINVAL; |
| 1023 | } |
| 1024 | |
| 1025 | return 0; |
| 1026 | } |
| 1027 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1028 | /* |
| 1029 | * OMAP Device MMU(IOMMU) detection |
| 1030 | */ |
Greg Kroah-Hartman | d34d651 | 2012-12-21 15:05:21 -0800 | [diff] [blame] | 1031 | static int omap_iommu_probe(struct platform_device *pdev) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1032 | { |
| 1033 | int err = -ENODEV; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1034 | int irq; |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 1035 | struct omap_iommu *obj; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1036 | struct resource *res; |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 1037 | struct device_node *of = pdev->dev.of_node; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1038 | |
Suman Anna | 49a57ef | 2017-04-12 00:21:27 -0500 | [diff] [blame] | 1039 | if (!of) { |
| 1040 | pr_err("%s: only DT-based devices are supported\n", __func__); |
| 1041 | return -ENODEV; |
| 1042 | } |
| 1043 | |
Suman Anna | f129b3d | 2014-02-28 14:42:32 -0600 | [diff] [blame] | 1044 | obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1045 | if (!obj) |
| 1046 | return -ENOMEM; |
| 1047 | |
Suman Anna | db8918f | 2019-08-07 11:26:47 +0300 | [diff] [blame^] | 1048 | /* |
| 1049 | * self-manage the ordering dependencies between omap_device_enable/idle |
| 1050 | * and omap_device_assert/deassert_hardreset API |
| 1051 | */ |
| 1052 | if (pdev->dev.pm_domain) { |
| 1053 | dev_dbg(&pdev->dev, "device pm_domain is being reset\n"); |
| 1054 | pdev->dev.pm_domain = NULL; |
| 1055 | } |
| 1056 | |
Suman Anna | 49a57ef | 2017-04-12 00:21:27 -0500 | [diff] [blame] | 1057 | obj->name = dev_name(&pdev->dev); |
| 1058 | obj->nr_tlb_entries = 32; |
| 1059 | err = of_property_read_u32(of, "ti,#tlb-entries", &obj->nr_tlb_entries); |
| 1060 | if (err && err != -EINVAL) |
| 1061 | return err; |
| 1062 | if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8) |
| 1063 | return -EINVAL; |
| 1064 | if (of_find_property(of, "ti,iommu-bus-err-back", NULL)) |
| 1065 | obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN; |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 1066 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1067 | obj->dev = &pdev->dev; |
| 1068 | obj->ctx = (void *)obj + sizeof(*obj); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1069 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1070 | spin_lock_init(&obj->iommu_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1071 | spin_lock_init(&obj->page_table_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1072 | |
| 1073 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
Suman Anna | f129b3d | 2014-02-28 14:42:32 -0600 | [diff] [blame] | 1074 | obj->regbase = devm_ioremap_resource(obj->dev, res); |
| 1075 | if (IS_ERR(obj->regbase)) |
| 1076 | return PTR_ERR(obj->regbase); |
Aaro Koskinen | da4a0f7 | 2011-03-14 12:28:32 +0000 | [diff] [blame] | 1077 | |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 1078 | err = omap_iommu_dra7_get_dsp_system_cfg(pdev, obj); |
| 1079 | if (err) |
| 1080 | return err; |
| 1081 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1082 | irq = platform_get_irq(pdev, 0); |
Suman Anna | f129b3d | 2014-02-28 14:42:32 -0600 | [diff] [blame] | 1083 | if (irq < 0) |
| 1084 | return -ENODEV; |
| 1085 | |
| 1086 | err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED, |
| 1087 | dev_name(obj->dev), obj); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1088 | if (err < 0) |
Suman Anna | f129b3d | 2014-02-28 14:42:32 -0600 | [diff] [blame] | 1089 | return err; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1090 | platform_set_drvdata(pdev, obj); |
| 1091 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1092 | if (omap_iommu_can_register(pdev)) { |
| 1093 | obj->group = iommu_group_alloc(); |
| 1094 | if (IS_ERR(obj->group)) |
| 1095 | return PTR_ERR(obj->group); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1096 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1097 | err = iommu_device_sysfs_add(&obj->iommu, obj->dev, NULL, |
| 1098 | obj->name); |
| 1099 | if (err) |
| 1100 | goto out_group; |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1101 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1102 | iommu_device_set_ops(&obj->iommu, &omap_iommu_ops); |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1103 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1104 | err = iommu_device_register(&obj->iommu); |
| 1105 | if (err) |
| 1106 | goto out_sysfs; |
| 1107 | } |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1108 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 1109 | pm_runtime_irq_safe(obj->dev); |
| 1110 | pm_runtime_enable(obj->dev); |
| 1111 | |
Suman Anna | 61c7535 | 2014-10-22 17:22:30 -0500 | [diff] [blame] | 1112 | omap_iommu_debugfs_add(obj); |
| 1113 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1114 | dev_info(&pdev->dev, "%s registered\n", obj->name); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1115 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1116 | return 0; |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1117 | |
| 1118 | out_sysfs: |
| 1119 | iommu_device_sysfs_remove(&obj->iommu); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1120 | out_group: |
| 1121 | iommu_group_put(obj->group); |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1122 | return err; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1123 | } |
| 1124 | |
Greg Kroah-Hartman | d34d651 | 2012-12-21 15:05:21 -0800 | [diff] [blame] | 1125 | static int omap_iommu_remove(struct platform_device *pdev) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1126 | { |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 1127 | struct omap_iommu *obj = platform_get_drvdata(pdev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1128 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1129 | if (obj->group) { |
| 1130 | iommu_group_put(obj->group); |
| 1131 | obj->group = NULL; |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1132 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1133 | iommu_device_sysfs_remove(&obj->iommu); |
| 1134 | iommu_device_unregister(&obj->iommu); |
| 1135 | } |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1136 | |
Suman Anna | 61c7535 | 2014-10-22 17:22:30 -0500 | [diff] [blame] | 1137 | omap_iommu_debugfs_remove(obj); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1138 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 1139 | pm_runtime_disable(obj->dev); |
| 1140 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1141 | dev_info(&pdev->dev, "%s removed\n", obj->name); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1142 | return 0; |
| 1143 | } |
| 1144 | |
Suman Anna | db8918f | 2019-08-07 11:26:47 +0300 | [diff] [blame^] | 1145 | static const struct dev_pm_ops omap_iommu_pm_ops = { |
| 1146 | SET_RUNTIME_PM_OPS(omap_iommu_runtime_suspend, |
| 1147 | omap_iommu_runtime_resume, NULL) |
| 1148 | }; |
| 1149 | |
Kiran Padwal | d943b0f | 2014-09-11 19:07:36 +0530 | [diff] [blame] | 1150 | static const struct of_device_id omap_iommu_of_match[] = { |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 1151 | { .compatible = "ti,omap2-iommu" }, |
| 1152 | { .compatible = "ti,omap4-iommu" }, |
| 1153 | { .compatible = "ti,dra7-iommu" }, |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 1154 | { .compatible = "ti,dra7-dsp-iommu" }, |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 1155 | {}, |
| 1156 | }; |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 1157 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1158 | static struct platform_driver omap_iommu_driver = { |
| 1159 | .probe = omap_iommu_probe, |
Greg Kroah-Hartman | d34d651 | 2012-12-21 15:05:21 -0800 | [diff] [blame] | 1160 | .remove = omap_iommu_remove, |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1161 | .driver = { |
| 1162 | .name = "omap-iommu", |
Suman Anna | db8918f | 2019-08-07 11:26:47 +0300 | [diff] [blame^] | 1163 | .pm = &omap_iommu_pm_ops, |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 1164 | .of_match_table = of_match_ptr(omap_iommu_of_match), |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1165 | }, |
| 1166 | }; |
| 1167 | |
Laurent Pinchart | 286f600 | 2014-03-08 00:44:38 +0100 | [diff] [blame] | 1168 | static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz) |
Tony Lindgren | ed1c7de | 2012-11-02 12:24:06 -0700 | [diff] [blame] | 1169 | { |
| 1170 | memset(e, 0, sizeof(*e)); |
| 1171 | |
| 1172 | e->da = da; |
| 1173 | e->pa = pa; |
Suman Anna | d760e3e | 2014-03-17 20:31:32 -0500 | [diff] [blame] | 1174 | e->valid = MMU_CAM_V; |
Laurent Pinchart | 286f600 | 2014-03-08 00:44:38 +0100 | [diff] [blame] | 1175 | e->pgsz = pgsz; |
| 1176 | e->endian = MMU_RAM_ENDIAN_LITTLE; |
| 1177 | e->elsz = MMU_RAM_ELSZ_8; |
| 1178 | e->mixed = 0; |
Tony Lindgren | ed1c7de | 2012-11-02 12:24:06 -0700 | [diff] [blame] | 1179 | |
| 1180 | return iopgsz_to_bytes(e->pgsz); |
| 1181 | } |
| 1182 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1183 | static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1184 | phys_addr_t pa, size_t bytes, int prot) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1185 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1186 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1187 | struct device *dev = omap_domain->dev; |
| 1188 | struct omap_iommu_device *iommu; |
| 1189 | struct omap_iommu *oiommu; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1190 | struct iotlb_entry e; |
| 1191 | int omap_pgsz; |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1192 | u32 ret = -EINVAL; |
| 1193 | int i; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1194 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1195 | omap_pgsz = bytes_to_iopgsz(bytes); |
| 1196 | if (omap_pgsz < 0) { |
| 1197 | dev_err(dev, "invalid size to map: %d\n", bytes); |
| 1198 | return -EINVAL; |
| 1199 | } |
| 1200 | |
Joerg Roedel | 1d7f449 | 2015-01-22 14:42:06 +0100 | [diff] [blame] | 1201 | dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%x\n", da, &pa, bytes); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1202 | |
Laurent Pinchart | 286f600 | 2014-03-08 00:44:38 +0100 | [diff] [blame] | 1203 | iotlb_init_entry(&e, da, pa, omap_pgsz); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1204 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1205 | iommu = omap_domain->iommus; |
| 1206 | for (i = 0; i < omap_domain->num_iommus; i++, iommu++) { |
| 1207 | oiommu = iommu->iommu_dev; |
| 1208 | ret = omap_iopgtable_store_entry(oiommu, &e); |
| 1209 | if (ret) { |
| 1210 | dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", |
| 1211 | ret); |
| 1212 | break; |
| 1213 | } |
| 1214 | } |
| 1215 | |
| 1216 | if (ret) { |
| 1217 | while (i--) { |
| 1218 | iommu--; |
| 1219 | oiommu = iommu->iommu_dev; |
| 1220 | iopgtable_clear_entry(oiommu, da); |
| 1221 | } |
| 1222 | } |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1223 | |
Ohad Ben-Cohen | b4550d4 | 2011-09-02 13:32:31 -0400 | [diff] [blame] | 1224 | return ret; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1225 | } |
| 1226 | |
Ohad Ben-Cohen | 5009065 | 2011-11-10 11:32:25 +0200 | [diff] [blame] | 1227 | static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1228 | size_t size) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1229 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1230 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1231 | struct device *dev = omap_domain->dev; |
| 1232 | struct omap_iommu_device *iommu; |
| 1233 | struct omap_iommu *oiommu; |
| 1234 | bool error = false; |
| 1235 | size_t bytes = 0; |
| 1236 | int i; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1237 | |
Ohad Ben-Cohen | 5009065 | 2011-11-10 11:32:25 +0200 | [diff] [blame] | 1238 | dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1239 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1240 | iommu = omap_domain->iommus; |
| 1241 | for (i = 0; i < omap_domain->num_iommus; i++, iommu++) { |
| 1242 | oiommu = iommu->iommu_dev; |
| 1243 | bytes = iopgtable_clear_entry(oiommu, da); |
| 1244 | if (!bytes) |
| 1245 | error = true; |
| 1246 | } |
| 1247 | |
| 1248 | /* |
| 1249 | * simplify return - we are only checking if any of the iommus |
| 1250 | * reported an error, but not if all of them are unmapping the |
| 1251 | * same number of entries. This should not occur due to the |
| 1252 | * mirror programming. |
| 1253 | */ |
| 1254 | return error ? 0 : bytes; |
| 1255 | } |
| 1256 | |
| 1257 | static int omap_iommu_count(struct device *dev) |
| 1258 | { |
| 1259 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
| 1260 | int count = 0; |
| 1261 | |
| 1262 | while (arch_data->iommu_dev) { |
| 1263 | count++; |
| 1264 | arch_data++; |
| 1265 | } |
| 1266 | |
| 1267 | return count; |
| 1268 | } |
| 1269 | |
| 1270 | /* caller should call cleanup if this function fails */ |
| 1271 | static int omap_iommu_attach_init(struct device *dev, |
| 1272 | struct omap_iommu_domain *odomain) |
| 1273 | { |
| 1274 | struct omap_iommu_device *iommu; |
| 1275 | int i; |
| 1276 | |
| 1277 | odomain->num_iommus = omap_iommu_count(dev); |
| 1278 | if (!odomain->num_iommus) |
| 1279 | return -EINVAL; |
| 1280 | |
| 1281 | odomain->iommus = kcalloc(odomain->num_iommus, sizeof(*iommu), |
| 1282 | GFP_ATOMIC); |
| 1283 | if (!odomain->iommus) |
| 1284 | return -ENOMEM; |
| 1285 | |
| 1286 | iommu = odomain->iommus; |
| 1287 | for (i = 0; i < odomain->num_iommus; i++, iommu++) { |
| 1288 | iommu->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_ATOMIC); |
| 1289 | if (!iommu->pgtable) |
| 1290 | return -ENOMEM; |
| 1291 | |
| 1292 | /* |
| 1293 | * should never fail, but please keep this around to ensure |
| 1294 | * we keep the hardware happy |
| 1295 | */ |
| 1296 | if (WARN_ON(!IS_ALIGNED((long)iommu->pgtable, |
| 1297 | IOPGD_TABLE_SIZE))) |
| 1298 | return -EINVAL; |
| 1299 | } |
| 1300 | |
| 1301 | return 0; |
| 1302 | } |
| 1303 | |
| 1304 | static void omap_iommu_detach_fini(struct omap_iommu_domain *odomain) |
| 1305 | { |
| 1306 | int i; |
| 1307 | struct omap_iommu_device *iommu = odomain->iommus; |
| 1308 | |
| 1309 | for (i = 0; iommu && i < odomain->num_iommus; i++, iommu++) |
| 1310 | kfree(iommu->pgtable); |
| 1311 | |
| 1312 | kfree(odomain->iommus); |
| 1313 | odomain->num_iommus = 0; |
| 1314 | odomain->iommus = NULL; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1315 | } |
| 1316 | |
| 1317 | static int |
| 1318 | omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) |
| 1319 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1320 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 1321 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1322 | struct omap_iommu_device *iommu; |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 1323 | struct omap_iommu *oiommu; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1324 | int ret = 0; |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1325 | int i; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1326 | |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 1327 | if (!arch_data || !arch_data->iommu_dev) { |
Suman Anna | e3f595b | 2014-09-04 17:27:29 -0500 | [diff] [blame] | 1328 | dev_err(dev, "device doesn't have an associated iommu\n"); |
| 1329 | return -EINVAL; |
| 1330 | } |
| 1331 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1332 | spin_lock(&omap_domain->lock); |
| 1333 | |
Suman Anna | 0d36428 | 2017-09-05 17:56:17 -0500 | [diff] [blame] | 1334 | /* only a single client device can be attached to a domain */ |
| 1335 | if (omap_domain->dev) { |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1336 | dev_err(dev, "iommu domain is already attached\n"); |
| 1337 | ret = -EBUSY; |
| 1338 | goto out; |
| 1339 | } |
| 1340 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1341 | ret = omap_iommu_attach_init(dev, omap_domain); |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 1342 | if (ret) { |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1343 | dev_err(dev, "failed to allocate required iommu data %d\n", |
| 1344 | ret); |
| 1345 | goto init_fail; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1346 | } |
| 1347 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1348 | iommu = omap_domain->iommus; |
| 1349 | for (i = 0; i < omap_domain->num_iommus; i++, iommu++, arch_data++) { |
| 1350 | /* configure and enable the omap iommu */ |
| 1351 | oiommu = arch_data->iommu_dev; |
| 1352 | ret = omap_iommu_attach(oiommu, iommu->pgtable); |
| 1353 | if (ret) { |
| 1354 | dev_err(dev, "can't get omap iommu: %d\n", ret); |
| 1355 | goto attach_fail; |
| 1356 | } |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1357 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1358 | oiommu->domain = domain; |
| 1359 | iommu->iommu_dev = oiommu; |
| 1360 | } |
| 1361 | |
| 1362 | omap_domain->dev = dev; |
| 1363 | |
| 1364 | goto out; |
| 1365 | |
| 1366 | attach_fail: |
| 1367 | while (i--) { |
| 1368 | iommu--; |
| 1369 | arch_data--; |
| 1370 | oiommu = iommu->iommu_dev; |
| 1371 | omap_iommu_detach(oiommu); |
| 1372 | iommu->iommu_dev = NULL; |
| 1373 | oiommu->domain = NULL; |
| 1374 | } |
| 1375 | init_fail: |
| 1376 | omap_iommu_detach_fini(omap_domain); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1377 | out: |
| 1378 | spin_unlock(&omap_domain->lock); |
| 1379 | return ret; |
| 1380 | } |
| 1381 | |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1382 | static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1383 | struct device *dev) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1384 | { |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1385 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
| 1386 | struct omap_iommu_device *iommu = omap_domain->iommus; |
| 1387 | struct omap_iommu *oiommu; |
| 1388 | int i; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1389 | |
Suman Anna | 0d36428 | 2017-09-05 17:56:17 -0500 | [diff] [blame] | 1390 | if (!omap_domain->dev) { |
| 1391 | dev_err(dev, "domain has no attached device\n"); |
| 1392 | return; |
| 1393 | } |
| 1394 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1395 | /* only a single device is supported per domain for now */ |
Suman Anna | 0d36428 | 2017-09-05 17:56:17 -0500 | [diff] [blame] | 1396 | if (omap_domain->dev != dev) { |
| 1397 | dev_err(dev, "invalid attached device\n"); |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1398 | return; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1399 | } |
| 1400 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1401 | /* |
| 1402 | * cleanup in the reverse order of attachment - this addresses |
| 1403 | * any h/w dependencies between multiple instances, if any |
| 1404 | */ |
| 1405 | iommu += (omap_domain->num_iommus - 1); |
| 1406 | arch_data += (omap_domain->num_iommus - 1); |
| 1407 | for (i = 0; i < omap_domain->num_iommus; i++, iommu--, arch_data--) { |
| 1408 | oiommu = iommu->iommu_dev; |
| 1409 | iopgtable_clear_entry_all(oiommu); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1410 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1411 | omap_iommu_detach(oiommu); |
| 1412 | iommu->iommu_dev = NULL; |
| 1413 | oiommu->domain = NULL; |
| 1414 | } |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1415 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1416 | omap_iommu_detach_fini(omap_domain); |
| 1417 | |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1418 | omap_domain->dev = NULL; |
| 1419 | } |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1420 | |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1421 | static void omap_iommu_detach_dev(struct iommu_domain *domain, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1422 | struct device *dev) |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1423 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1424 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1425 | |
| 1426 | spin_lock(&omap_domain->lock); |
| 1427 | _omap_iommu_detach_dev(omap_domain, dev); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1428 | spin_unlock(&omap_domain->lock); |
| 1429 | } |
| 1430 | |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1431 | static struct iommu_domain *omap_iommu_domain_alloc(unsigned type) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1432 | { |
| 1433 | struct omap_iommu_domain *omap_domain; |
| 1434 | |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1435 | if (type != IOMMU_DOMAIN_UNMANAGED) |
| 1436 | return NULL; |
| 1437 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1438 | omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL); |
Suman Anna | 99ee98d | 2015-07-20 17:33:29 -0500 | [diff] [blame] | 1439 | if (!omap_domain) |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1440 | return NULL; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1441 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1442 | spin_lock_init(&omap_domain->lock); |
| 1443 | |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1444 | omap_domain->domain.geometry.aperture_start = 0; |
| 1445 | omap_domain->domain.geometry.aperture_end = (1ULL << 32) - 1; |
| 1446 | omap_domain->domain.geometry.force_aperture = true; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1447 | |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1448 | return &omap_domain->domain; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1449 | } |
| 1450 | |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1451 | static void omap_iommu_domain_free(struct iommu_domain *domain) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1452 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1453 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1454 | |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1455 | /* |
| 1456 | * An iommu device is still attached |
| 1457 | * (currently, only one device can be attached) ? |
| 1458 | */ |
Suman Anna | 0d36428 | 2017-09-05 17:56:17 -0500 | [diff] [blame] | 1459 | if (omap_domain->dev) |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1460 | _omap_iommu_detach_dev(omap_domain, omap_domain->dev); |
| 1461 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1462 | kfree(omap_domain); |
| 1463 | } |
| 1464 | |
| 1465 | static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1466 | dma_addr_t da) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1467 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1468 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1469 | struct omap_iommu_device *iommu = omap_domain->iommus; |
| 1470 | struct omap_iommu *oiommu = iommu->iommu_dev; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1471 | struct device *dev = oiommu->dev; |
| 1472 | u32 *pgd, *pte; |
| 1473 | phys_addr_t ret = 0; |
| 1474 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1475 | /* |
| 1476 | * all the iommus within the domain will have identical programming, |
| 1477 | * so perform the lookup using just the first iommu |
| 1478 | */ |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1479 | iopgtable_lookup_entry(oiommu, da, &pgd, &pte); |
| 1480 | |
| 1481 | if (pte) { |
| 1482 | if (iopte_is_small(*pte)) |
| 1483 | ret = omap_iommu_translate(*pte, da, IOPTE_MASK); |
| 1484 | else if (iopte_is_large(*pte)) |
| 1485 | ret = omap_iommu_translate(*pte, da, IOLARGE_MASK); |
| 1486 | else |
Suman Anna | 2abfcfb | 2013-05-30 18:10:38 -0500 | [diff] [blame] | 1487 | dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1488 | (unsigned long long)da); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1489 | } else { |
| 1490 | if (iopgd_is_section(*pgd)) |
| 1491 | ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK); |
| 1492 | else if (iopgd_is_super(*pgd)) |
| 1493 | ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK); |
| 1494 | else |
Suman Anna | 2abfcfb | 2013-05-30 18:10:38 -0500 | [diff] [blame] | 1495 | dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1496 | (unsigned long long)da); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1497 | } |
| 1498 | |
| 1499 | return ret; |
| 1500 | } |
| 1501 | |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1502 | static int omap_iommu_add_device(struct device *dev) |
| 1503 | { |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1504 | struct omap_iommu_arch_data *arch_data, *tmp; |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 1505 | struct omap_iommu *oiommu; |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1506 | struct iommu_group *group; |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1507 | struct device_node *np; |
Suman Anna | 7d68277 | 2014-09-04 17:27:30 -0500 | [diff] [blame] | 1508 | struct platform_device *pdev; |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1509 | int num_iommus, i; |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1510 | int ret; |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1511 | |
| 1512 | /* |
| 1513 | * Allocate the archdata iommu structure for DT-based devices. |
| 1514 | * |
| 1515 | * TODO: Simplify this when removing non-DT support completely from the |
| 1516 | * IOMMU users. |
| 1517 | */ |
| 1518 | if (!dev->of_node) |
| 1519 | return 0; |
| 1520 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1521 | /* |
| 1522 | * retrieve the count of IOMMU nodes using phandle size as element size |
| 1523 | * since #iommu-cells = 0 for OMAP |
| 1524 | */ |
| 1525 | num_iommus = of_property_count_elems_of_size(dev->of_node, "iommus", |
| 1526 | sizeof(phandle)); |
| 1527 | if (num_iommus < 0) |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1528 | return 0; |
| 1529 | |
Kees Cook | 6396bb2 | 2018-06-12 14:03:40 -0700 | [diff] [blame] | 1530 | arch_data = kcalloc(num_iommus + 1, sizeof(*arch_data), GFP_KERNEL); |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1531 | if (!arch_data) |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1532 | return -ENOMEM; |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1533 | |
| 1534 | for (i = 0, tmp = arch_data; i < num_iommus; i++, tmp++) { |
| 1535 | np = of_parse_phandle(dev->of_node, "iommus", i); |
| 1536 | if (!np) { |
| 1537 | kfree(arch_data); |
| 1538 | return -EINVAL; |
| 1539 | } |
| 1540 | |
| 1541 | pdev = of_find_device_by_node(np); |
| 1542 | if (WARN_ON(!pdev)) { |
| 1543 | of_node_put(np); |
| 1544 | kfree(arch_data); |
| 1545 | return -EINVAL; |
| 1546 | } |
| 1547 | |
| 1548 | oiommu = platform_get_drvdata(pdev); |
| 1549 | if (!oiommu) { |
| 1550 | of_node_put(np); |
| 1551 | kfree(arch_data); |
| 1552 | return -EINVAL; |
| 1553 | } |
| 1554 | |
| 1555 | tmp->iommu_dev = oiommu; |
| 1556 | |
| 1557 | of_node_put(np); |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1558 | } |
| 1559 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1560 | /* |
| 1561 | * use the first IOMMU alone for the sysfs device linking. |
| 1562 | * TODO: Evaluate if a single iommu_group needs to be |
| 1563 | * maintained for both IOMMUs |
| 1564 | */ |
| 1565 | oiommu = arch_data->iommu_dev; |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1566 | ret = iommu_device_link(&oiommu->iommu, dev); |
| 1567 | if (ret) { |
| 1568 | kfree(arch_data); |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1569 | return ret; |
| 1570 | } |
| 1571 | |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1572 | dev->archdata.iommu = arch_data; |
| 1573 | |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1574 | /* |
| 1575 | * IOMMU group initialization calls into omap_iommu_device_group, which |
| 1576 | * needs a valid dev->archdata.iommu pointer |
| 1577 | */ |
| 1578 | group = iommu_group_get_for_dev(dev); |
| 1579 | if (IS_ERR(group)) { |
| 1580 | iommu_device_unlink(&oiommu->iommu, dev); |
| 1581 | dev->archdata.iommu = NULL; |
| 1582 | kfree(arch_data); |
| 1583 | return PTR_ERR(group); |
| 1584 | } |
| 1585 | iommu_group_put(group); |
| 1586 | |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1587 | return 0; |
| 1588 | } |
| 1589 | |
| 1590 | static void omap_iommu_remove_device(struct device *dev) |
| 1591 | { |
| 1592 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
| 1593 | |
| 1594 | if (!dev->of_node || !arch_data) |
| 1595 | return; |
| 1596 | |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1597 | iommu_device_unlink(&arch_data->iommu_dev->iommu, dev); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1598 | iommu_group_remove_device(dev); |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1599 | |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 1600 | dev->archdata.iommu = NULL; |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1601 | kfree(arch_data); |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1602 | |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1603 | } |
| 1604 | |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1605 | static struct iommu_group *omap_iommu_device_group(struct device *dev) |
| 1606 | { |
| 1607 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
Joerg Roedel | 8faf5e5 | 2017-06-28 12:50:16 +0200 | [diff] [blame] | 1608 | struct iommu_group *group = ERR_PTR(-EINVAL); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1609 | |
| 1610 | if (arch_data->iommu_dev) |
Jeffy Chen | b6d57f1 | 2018-03-01 19:22:08 +0800 | [diff] [blame] | 1611 | group = iommu_group_ref_get(arch_data->iommu_dev->group); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1612 | |
| 1613 | return group; |
| 1614 | } |
| 1615 | |
Thierry Reding | b22f643 | 2014-06-27 09:03:12 +0200 | [diff] [blame] | 1616 | static const struct iommu_ops omap_iommu_ops = { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1617 | .domain_alloc = omap_iommu_domain_alloc, |
| 1618 | .domain_free = omap_iommu_domain_free, |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1619 | .attach_dev = omap_iommu_attach_dev, |
| 1620 | .detach_dev = omap_iommu_detach_dev, |
| 1621 | .map = omap_iommu_map, |
| 1622 | .unmap = omap_iommu_unmap, |
| 1623 | .iova_to_phys = omap_iommu_iova_to_phys, |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1624 | .add_device = omap_iommu_add_device, |
| 1625 | .remove_device = omap_iommu_remove_device, |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1626 | .device_group = omap_iommu_device_group, |
Ohad Ben-Cohen | 66bc8cf | 2011-11-10 11:32:27 +0200 | [diff] [blame] | 1627 | .pgsize_bitmap = OMAP_IOMMU_PGSIZES, |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1628 | }; |
| 1629 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1630 | static int __init omap_iommu_init(void) |
| 1631 | { |
| 1632 | struct kmem_cache *p; |
| 1633 | const unsigned long flags = SLAB_HWCACHE_ALIGN; |
| 1634 | size_t align = 1 << 10; /* L2 pagetable alignement */ |
Thierry Reding | f938aab | 2015-02-06 11:44:06 +0100 | [diff] [blame] | 1635 | struct device_node *np; |
Suman Anna | abaa7e5 | 2017-04-12 00:21:26 -0500 | [diff] [blame] | 1636 | int ret; |
Thierry Reding | f938aab | 2015-02-06 11:44:06 +0100 | [diff] [blame] | 1637 | |
| 1638 | np = of_find_matching_node(NULL, omap_iommu_of_match); |
| 1639 | if (!np) |
| 1640 | return 0; |
| 1641 | |
| 1642 | of_node_put(np); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1643 | |
| 1644 | p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags, |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 1645 | NULL); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1646 | if (!p) |
| 1647 | return -ENOMEM; |
| 1648 | iopte_cachep = p; |
| 1649 | |
Suman Anna | 61c7535 | 2014-10-22 17:22:30 -0500 | [diff] [blame] | 1650 | omap_iommu_debugfs_init(); |
| 1651 | |
Suman Anna | abaa7e5 | 2017-04-12 00:21:26 -0500 | [diff] [blame] | 1652 | ret = platform_driver_register(&omap_iommu_driver); |
| 1653 | if (ret) { |
| 1654 | pr_err("%s: failed to register driver\n", __func__); |
| 1655 | goto fail_driver; |
| 1656 | } |
| 1657 | |
| 1658 | ret = bus_set_iommu(&platform_bus_type, &omap_iommu_ops); |
| 1659 | if (ret) |
| 1660 | goto fail_bus; |
| 1661 | |
| 1662 | return 0; |
| 1663 | |
| 1664 | fail_bus: |
| 1665 | platform_driver_unregister(&omap_iommu_driver); |
| 1666 | fail_driver: |
| 1667 | kmem_cache_destroy(iopte_cachep); |
| 1668 | return ret; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1669 | } |
Ohad Ben-Cohen | 435792d | 2012-02-26 12:14:14 +0200 | [diff] [blame] | 1670 | subsys_initcall(omap_iommu_init); |
Suman Anna | 0cdbf72 | 2015-07-20 17:33:24 -0500 | [diff] [blame] | 1671 | /* must be ready before omap3isp is probed */ |