iommu/io-pgtable: Indicate granule for TLB maintenance
IOMMU hardware with range-based TLB maintenance commands can work
happily with the iova and size arguments passed via the tlb_add_flush
callback, but for IOMMUs which require separate commands per entry in
the range, it is not straightforward to infer the necessary granularity
when it comes to issuing the actual commands.
Add an additional argument indicating the granularity for the benefit
of drivers needing to know, and update the ARM LPAE code appropriately
(for non-leaf invalidations we currently just assume the worst-case
page granularity rather than walking the table to check).
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 366a354..7a5c772 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -58,8 +58,10 @@
((((d)->levels - ((l) - ARM_LPAE_START_LVL(d) + 1)) \
* (d)->bits_per_level) + (d)->pg_shift)
+#define ARM_LPAE_GRANULE(d) (1UL << (d)->pg_shift)
+
#define ARM_LPAE_PAGES_PER_PGD(d) \
- DIV_ROUND_UP((d)->pgd_size, 1UL << (d)->pg_shift)
+ DIV_ROUND_UP((d)->pgd_size, ARM_LPAE_GRANULE(d))
/*
* Calculate the index at level l used to map virtual address a using the
@@ -169,7 +171,7 @@
/* IOPTE accessors */
#define iopte_deref(pte,d) \
(__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \
- & ~((1ULL << (d)->pg_shift) - 1)))
+ & ~(ARM_LPAE_GRANULE(d) - 1ULL)))
#define iopte_type(pte,l) \
(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
@@ -326,7 +328,7 @@
/* Grab a pointer to the next level */
pte = *ptep;
if (!pte) {
- cptep = __arm_lpae_alloc_pages(1UL << data->pg_shift,
+ cptep = __arm_lpae_alloc_pages(ARM_LPAE_GRANULE(data),
GFP_ATOMIC, cfg);
if (!cptep)
return -ENOMEM;
@@ -412,7 +414,7 @@
if (lvl == ARM_LPAE_START_LVL(data))
table_size = data->pgd_size;
else
- table_size = 1UL << data->pg_shift;
+ table_size = ARM_LPAE_GRANULE(data);
start = ptep;
end = (void *)ptep + table_size;
@@ -473,7 +475,7 @@
__arm_lpae_set_pte(ptep, table, cfg);
iova &= ~(blk_size - 1);
- cfg->tlb->tlb_add_flush(iova, blk_size, true, data->iop.cookie);
+ cfg->tlb->tlb_add_flush(iova, blk_size, blk_size, true, data->iop.cookie);
return size;
}
@@ -501,12 +503,13 @@
if (!iopte_leaf(pte, lvl)) {
/* Also flush any partial walks */
- tlb->tlb_add_flush(iova, size, false, cookie);
+ tlb->tlb_add_flush(iova, size, ARM_LPAE_GRANULE(data),
+ false, cookie);
tlb->tlb_sync(cookie);
ptep = iopte_deref(pte, data);
__arm_lpae_free_pgtable(data, lvl + 1, ptep);
} else {
- tlb->tlb_add_flush(iova, size, true, cookie);
+ tlb->tlb_add_flush(iova, size, size, true, cookie);
}
return size;
@@ -572,7 +575,7 @@
return 0;
found_translation:
- iova &= ((1 << data->pg_shift) - 1);
+ iova &= (ARM_LPAE_GRANULE(data) - 1);
return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova;
}
@@ -670,7 +673,7 @@
(ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) |
(ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
- switch (1 << data->pg_shift) {
+ switch (ARM_LPAE_GRANULE(data)) {
case SZ_4K:
reg |= ARM_LPAE_TCR_TG0_4K;
break;
@@ -771,7 +774,7 @@
sl = ARM_LPAE_START_LVL(data);
- switch (1 << data->pg_shift) {
+ switch (ARM_LPAE_GRANULE(data)) {
case SZ_4K:
reg |= ARM_LPAE_TCR_TG0_4K;
sl++; /* SL0 format is different for 4K granule size */
@@ -891,8 +894,8 @@
WARN_ON(cookie != cfg_cookie);
}
-static void dummy_tlb_add_flush(unsigned long iova, size_t size, bool leaf,
- void *cookie)
+static void dummy_tlb_add_flush(unsigned long iova, size_t size,
+ size_t granule, bool leaf, void *cookie)
{
WARN_ON(cookie != cfg_cookie);
WARN_ON(!(size & cfg_cookie->pgsize_bitmap));