blob: c18da36c75b199990d90375dda6e711c16d1104b [file] [log] [blame]
Gregory CLEMENT009f1312012-08-02 11:16:29 +03001/*
Thomas Petazzonie12f12a2014-11-13 10:39:00 +01002 * Coherency fabric (Aurora) support for Armada 370, 375, 38x and XP
3 * platforms.
Gregory CLEMENT009f1312012-08-02 11:16:29 +03004 *
5 * Copyright (C) 2012 Marvell
6 *
7 * Yehuda Yitschak <yehuday@marvell.com>
8 * Gregory Clement <gregory.clement@free-electrons.com>
9 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
10 *
11 * This file is licensed under the terms of the GNU General Public
12 * License version 2. This program is licensed "as is" without any
13 * warranty of any kind, whether express or implied.
14 *
Thomas Petazzonie12f12a2014-11-13 10:39:00 +010015 * The Armada 370, 375, 38x and XP SOCs have a coherency fabric which is
Gregory CLEMENT009f1312012-08-02 11:16:29 +030016 * responsible for ensuring hardware coherency between all CPUs and between
17 * CPUs and I/O masters. This file initializes the coherency fabric and
18 * supplies basic routines for configuring and controlling hardware coherency
19 */
20
Thomas Petazzoni5ab5afd2014-04-14 15:47:05 +020021#define pr_fmt(fmt) "mvebu-coherency: " fmt
22
Gregory CLEMENT009f1312012-08-02 11:16:29 +030023#include <linux/kernel.h>
24#include <linux/init.h>
25#include <linux/of_address.h>
26#include <linux/io.h>
27#include <linux/smp.h>
Gregory CLEMENTe60304f2012-10-12 19:20:36 +020028#include <linux/dma-mapping.h>
29#include <linux/platform_device.h>
Thomas Petazzoni5ab5afd2014-04-14 15:47:05 +020030#include <linux/slab.h>
31#include <linux/mbus.h>
Thomas Petazzonib0063aa2014-05-13 18:04:30 +020032#include <linux/pci.h>
Gregory CLEMENT009f1312012-08-02 11:16:29 +030033#include <asm/smp_plat.h>
Thomas Petazzoni580ff0e2013-06-06 12:24:28 +020034#include <asm/cacheflush.h>
Thomas Petazzoni497a9232014-05-15 16:59:34 +020035#include <asm/mach/map.h>
Gregory CLEMENT009f1312012-08-02 11:16:29 +030036#include "armada-370-xp.h"
Jisheng Zhangb12634e2013-11-07 17:02:38 +080037#include "coherency.h"
Thomas Petazzoni39438562014-05-05 17:05:26 +020038#include "mvebu-soc-id.h"
Gregory CLEMENT009f1312012-08-02 11:16:29 +030039
Paul Gortmaker8bd26e32013-06-17 15:43:14 -040040unsigned long coherency_phys_base;
Gregory CLEMENTccd6a132014-04-14 17:10:05 +020041void __iomem *coherency_base;
Gregory CLEMENTe60304f2012-10-12 19:20:36 +020042static void __iomem *coherency_cpu_base;
Gregory CLEMENT009f1312012-08-02 11:16:29 +030043
44/* Coherency fabric registers */
Gregory CLEMENTe60304f2012-10-12 19:20:36 +020045#define IO_SYNC_BARRIER_CTL_OFFSET 0x0
46
Thomas Petazzoni924d38f2014-04-14 15:46:59 +020047enum {
Thomas Petazzoni501f9282014-04-14 15:47:00 +020048 COHERENCY_FABRIC_TYPE_NONE,
Thomas Petazzoni924d38f2014-04-14 15:46:59 +020049 COHERENCY_FABRIC_TYPE_ARMADA_370_XP,
Thomas Petazzoni77fa4b92014-04-14 15:47:04 +020050 COHERENCY_FABRIC_TYPE_ARMADA_375,
Thomas Petazzonid0de9322014-04-14 15:47:06 +020051 COHERENCY_FABRIC_TYPE_ARMADA_380,
Thomas Petazzoni924d38f2014-04-14 15:46:59 +020052};
53
Gregory CLEMENT009f1312012-08-02 11:16:29 +030054static struct of_device_id of_coherency_table[] = {
Thomas Petazzoni924d38f2014-04-14 15:46:59 +020055 {.compatible = "marvell,coherency-fabric",
56 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_370_XP },
Thomas Petazzoni77fa4b92014-04-14 15:47:04 +020057 {.compatible = "marvell,armada-375-coherency-fabric",
58 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_375 },
Thomas Petazzonid0de9322014-04-14 15:47:06 +020059 {.compatible = "marvell,armada-380-coherency-fabric",
60 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_380 },
Gregory CLEMENT009f1312012-08-02 11:16:29 +030061 { /* end of list */ },
62};
63
Gregory CLEMENT2e8a5942014-04-14 17:10:08 +020064/* Functions defined in coherency_ll.S */
65int ll_enable_coherency(void);
66void ll_add_cpu_to_smp_group(void);
Gregory CLEMENT009f1312012-08-02 11:16:29 +030067
Gregory CLEMENT952f4ca2014-04-14 17:10:07 +020068int set_cpu_coherent(void)
Gregory CLEMENT009f1312012-08-02 11:16:29 +030069{
70 if (!coherency_base) {
Gregory CLEMENTb41375f2014-04-14 17:10:06 +020071 pr_warn("Can't make current CPU cache coherent.\n");
Gregory CLEMENT009f1312012-08-02 11:16:29 +030072 pr_warn("Coherency fabric is not initialized\n");
73 return 1;
74 }
75
Gregory CLEMENT2e8a5942014-04-14 17:10:08 +020076 ll_add_cpu_to_smp_group();
77 return ll_enable_coherency();
Gregory CLEMENT009f1312012-08-02 11:16:29 +030078}
79
Gregory CLEMENTe60304f2012-10-12 19:20:36 +020080static inline void mvebu_hwcc_sync_io_barrier(void)
81{
82 writel(0x1, coherency_cpu_base + IO_SYNC_BARRIER_CTL_OFFSET);
83 while (readl(coherency_cpu_base + IO_SYNC_BARRIER_CTL_OFFSET) & 0x1);
84}
85
86static dma_addr_t mvebu_hwcc_dma_map_page(struct device *dev, struct page *page,
87 unsigned long offset, size_t size,
88 enum dma_data_direction dir,
89 struct dma_attrs *attrs)
90{
91 if (dir != DMA_TO_DEVICE)
92 mvebu_hwcc_sync_io_barrier();
93 return pfn_to_dma(dev, page_to_pfn(page)) + offset;
94}
95
96
97static void mvebu_hwcc_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
98 size_t size, enum dma_data_direction dir,
99 struct dma_attrs *attrs)
100{
101 if (dir != DMA_TO_DEVICE)
102 mvebu_hwcc_sync_io_barrier();
103}
104
105static void mvebu_hwcc_dma_sync(struct device *dev, dma_addr_t dma_handle,
106 size_t size, enum dma_data_direction dir)
107{
108 if (dir != DMA_TO_DEVICE)
109 mvebu_hwcc_sync_io_barrier();
110}
111
112static struct dma_map_ops mvebu_hwcc_dma_ops = {
113 .alloc = arm_dma_alloc,
114 .free = arm_dma_free,
115 .mmap = arm_dma_mmap,
116 .map_page = mvebu_hwcc_dma_map_page,
117 .unmap_page = mvebu_hwcc_dma_unmap_page,
118 .get_sgtable = arm_dma_get_sgtable,
119 .map_sg = arm_dma_map_sg,
120 .unmap_sg = arm_dma_unmap_sg,
121 .sync_single_for_cpu = mvebu_hwcc_dma_sync,
122 .sync_single_for_device = mvebu_hwcc_dma_sync,
123 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
124 .sync_sg_for_device = arm_dma_sync_sg_for_device,
125 .set_dma_mask = arm_dma_set_mask,
126};
127
Thomas Petazzonib0063aa2014-05-13 18:04:30 +0200128static int mvebu_hwcc_notifier(struct notifier_block *nb,
129 unsigned long event, void *__dev)
Gregory CLEMENTe60304f2012-10-12 19:20:36 +0200130{
131 struct device *dev = __dev;
132
133 if (event != BUS_NOTIFY_ADD_DEVICE)
134 return NOTIFY_DONE;
135 set_dma_ops(dev, &mvebu_hwcc_dma_ops);
136
137 return NOTIFY_OK;
138}
139
Thomas Petazzonib0063aa2014-05-13 18:04:30 +0200140static struct notifier_block mvebu_hwcc_nb = {
141 .notifier_call = mvebu_hwcc_notifier,
Gregory CLEMENTe60304f2012-10-12 19:20:36 +0200142};
143
Ezequiel Garciaa728b972014-07-08 10:37:37 -0300144static struct notifier_block mvebu_hwcc_pci_nb = {
145 .notifier_call = mvebu_hwcc_notifier,
146};
147
Thomas Petazzoni924d38f2014-04-14 15:46:59 +0200148static void __init armada_370_coherency_init(struct device_node *np)
149{
150 struct resource res;
151
152 of_address_to_resource(np, 0, &res);
153 coherency_phys_base = res.start;
154 /*
155 * Ensure secondary CPUs will see the updated value,
156 * which they read before they join the coherency
157 * fabric, and therefore before they are coherent with
158 * the boot CPU cache.
159 */
160 sync_cache_w(&coherency_phys_base);
161 coherency_base = of_iomap(np, 0);
162 coherency_cpu_base = of_iomap(np, 1);
Gregory CLEMENT952f4ca2014-04-14 17:10:07 +0200163 set_cpu_coherent();
Thomas Petazzoni924d38f2014-04-14 15:46:59 +0200164}
165
Thomas Petazzoni497a9232014-05-15 16:59:34 +0200166/*
167 * This ioremap hook is used on Armada 375/38x to ensure that PCIe
168 * memory areas are mapped as MT_UNCACHED instead of MT_DEVICE. This
169 * is needed as a workaround for a deadlock issue between the PCIe
170 * interface and the cache controller.
171 */
172static void __iomem *
173armada_pcie_wa_ioremap_caller(phys_addr_t phys_addr, size_t size,
174 unsigned int mtype, void *caller)
175{
176 struct resource pcie_mem;
177
178 mvebu_mbus_get_pcie_mem_aperture(&pcie_mem);
179
180 if (pcie_mem.start <= phys_addr && (phys_addr + size) <= pcie_mem.end)
181 mtype = MT_UNCACHED;
182
183 return __arm_ioremap_caller(phys_addr, size, mtype, caller);
184}
185
Thomas Petazzonid0de9322014-04-14 15:47:06 +0200186static void __init armada_375_380_coherency_init(struct device_node *np)
Thomas Petazzoni77fa4b92014-04-14 15:47:04 +0200187{
Thomas Petazzoni497a9232014-05-15 16:59:34 +0200188 struct device_node *cache_dn;
189
Thomas Petazzoni77fa4b92014-04-14 15:47:04 +0200190 coherency_cpu_base = of_iomap(np, 0);
Thomas Petazzoni497a9232014-05-15 16:59:34 +0200191 arch_ioremap_caller = armada_pcie_wa_ioremap_caller;
192
193 /*
194 * Add the PL310 property "arm,io-coherent". This makes sure the
195 * outer sync operation is not used, which allows to
196 * workaround the system erratum that causes deadlocks when
197 * doing PCIe in an SMP situation on Armada 375 and Armada
198 * 38x.
199 */
200 for_each_compatible_node(cache_dn, NULL, "arm,pl310-cache") {
201 struct property *p;
202
203 p = kzalloc(sizeof(*p), GFP_KERNEL);
204 p->name = kstrdup("arm,io-coherent", GFP_KERNEL);
205 of_add_property(cache_dn, p);
206 }
Thomas Petazzoni77fa4b92014-04-14 15:47:04 +0200207}
208
Thomas Petazzoni501f9282014-04-14 15:47:00 +0200209static int coherency_type(void)
Gregory CLEMENT009f1312012-08-02 11:16:29 +0300210{
211 struct device_node *np;
Thomas Petazzoni5fbba082014-04-14 15:47:02 +0200212 const struct of_device_id *match;
Thomas Petazzonie5535542014-11-13 10:38:57 +0100213 int type;
214
215 /*
216 * The coherency fabric is needed:
217 * - For coherency between processors on Armada XP, so only
218 * when SMP is enabled.
219 * - For coherency between the processor and I/O devices, but
220 * this coherency requires many pre-requisites (write
221 * allocate cache policy, shareable pages, SMP bit set) that
222 * are only meant in SMP situations.
223 *
224 * Note that this means that on Armada 370, there is currently
225 * no way to use hardware I/O coherency, because even when
226 * CONFIG_SMP is enabled, is_smp() returns false due to the
227 * Armada 370 being a single-core processor. To lift this
228 * limitation, we would have to find a way to make the cache
229 * policy set to write-allocate (on all Armada SoCs), and to
230 * set the shareable attribute in page tables (on all Armada
231 * SoCs except the Armada 370). Unfortunately, such decisions
232 * are taken very early in the kernel boot process, at a point
233 * where we don't know yet on which SoC we are running.
234
235 */
236 if (!is_smp())
237 return COHERENCY_FABRIC_TYPE_NONE;
Gregory CLEMENT009f1312012-08-02 11:16:29 +0300238
Thomas Petazzoni5fbba082014-04-14 15:47:02 +0200239 np = of_find_matching_node_and_match(NULL, of_coherency_table, &match);
Thomas Petazzonie5535542014-11-13 10:38:57 +0100240 if (!np)
241 return COHERENCY_FABRIC_TYPE_NONE;
Thomas Petazzoni924d38f2014-04-14 15:46:59 +0200242
Thomas Petazzonie5535542014-11-13 10:38:57 +0100243 type = (int) match->data;
Thomas Petazzoni924d38f2014-04-14 15:46:59 +0200244
Thomas Petazzonie5535542014-11-13 10:38:57 +0100245 of_node_put(np);
Thomas Petazzoni77fa4b92014-04-14 15:47:04 +0200246
Thomas Petazzonie5535542014-11-13 10:38:57 +0100247 return type;
Thomas Petazzoni501f9282014-04-14 15:47:00 +0200248}
249
250int coherency_available(void)
251{
252 return coherency_type() != COHERENCY_FABRIC_TYPE_NONE;
253}
254
255int __init coherency_init(void)
256{
257 int type = coherency_type();
258 struct device_node *np;
259
260 np = of_find_matching_node(NULL, of_coherency_table);
261
262 if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP)
263 armada_370_coherency_init(np);
Thomas Petazzonid0de9322014-04-14 15:47:06 +0200264 else if (type == COHERENCY_FABRIC_TYPE_ARMADA_375 ||
265 type == COHERENCY_FABRIC_TYPE_ARMADA_380)
266 armada_375_380_coherency_init(np);
Thomas Petazzoni501f9282014-04-14 15:47:00 +0200267
Thomas Petazzoni2eb04ae2014-10-27 16:32:35 +0100268 of_node_put(np);
269
Gregory CLEMENT009f1312012-08-02 11:16:29 +0300270 return 0;
271}
Thomas Petazzoni865e0522013-06-05 09:04:55 +0200272
273static int __init coherency_late_init(void)
274{
Thomas Petazzonief01c6c2014-11-13 10:38:59 +0100275 if (coherency_available())
276 bus_register_notifier(&platform_bus_type,
277 &mvebu_hwcc_nb);
Thomas Petazzoni865e0522013-06-05 09:04:55 +0200278 return 0;
279}
280
281postcore_initcall(coherency_late_init);
Thomas Petazzonib0063aa2014-05-13 18:04:30 +0200282
Thomas Petazzoni8828ccc2014-05-20 17:13:03 +0200283#if IS_ENABLED(CONFIG_PCI)
Thomas Petazzonib0063aa2014-05-13 18:04:30 +0200284static int __init coherency_pci_init(void)
285{
286 if (coherency_available())
287 bus_register_notifier(&pci_bus_type,
Ezequiel Garciaa728b972014-07-08 10:37:37 -0300288 &mvebu_hwcc_pci_nb);
Thomas Petazzonib0063aa2014-05-13 18:04:30 +0200289 return 0;
290}
291
292arch_initcall(coherency_pci_init);
Thomas Petazzoni8828ccc2014-05-20 17:13:03 +0200293#endif