blob: 627883bc6d5f29f4a4510373ddb8e552398cb5ed [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
Justin P. Mattock79add622011-04-04 14:15:29 -07006 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
9 */
Ralf Baechlea754f702007-11-03 01:01:37 +000010#include <linux/hardirq.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/init.h>
Ralf Baechledb813fe2007-09-27 18:26:43 +010012#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/kernel.h>
Ralf Baechle641e97f2007-10-11 23:46:05 +010014#include <linux/linkage.h>
Ralf Baechleff522052013-09-17 12:44:31 +020015#include <linux/preempt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/sched.h>
Ralf Baechle631330f2009-06-19 14:05:26 +010017#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/mm.h>
Chris Dearman35133692007-09-19 00:58:24 +010019#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/bitops.h>
21
22#include <asm/bcache.h>
23#include <asm/bootinfo.h>
Ralf Baechleec74e362005-07-13 11:48:45 +000024#include <asm/cache.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <asm/cacheops.h>
26#include <asm/cpu.h>
27#include <asm/cpu-features.h>
Ralf Baechle69f24d12013-09-17 10:25:47 +020028#include <asm/cpu-type.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <asm/io.h>
30#include <asm/page.h>
31#include <asm/pgtable.h>
32#include <asm/r4kcache.h>
Ralf Baechlee001e522007-07-28 12:45:47 +010033#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include <asm/mmu_context.h>
35#include <asm/war.h>
Thiemo Seuferba5187d2005-04-25 16:36:23 +000036#include <asm/cacheflush.h> /* for run_uncached() */
David Daney9cd9669b2012-05-15 00:04:49 -070037#include <asm/traps.h>
Steven J. Hillb6d92b42013-03-25 13:47:29 -050038#include <asm/dma-coherence.h>
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010039
40/*
41 * Special Variant of smp_call_function for use by cache functions:
42 *
43 * o No return value
44 * o collapses to normal function call on UP kernels
45 * o collapses to normal function call on systems with a single shared
46 * primary cache.
Ralf Baechlec8c5f3f2010-10-29 19:08:25 +010047 * o doesn't disable interrupts on the local CPU
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010048 */
Ralf Baechle48a26e62010-10-29 19:08:25 +010049static inline void r4k_on_each_cpu(void (*func) (void *info), void *info)
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010050{
51 preempt_disable();
52
53#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC)
Ralf Baechle48a26e62010-10-29 19:08:25 +010054 smp_call_function(func, info, 1);
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010055#endif
56 func(info);
57 preempt_enable();
58}
59
Ralf Baechle39b8d522008-04-28 17:14:26 +010060#if defined(CONFIG_MIPS_CMP)
61#define cpu_has_safe_index_cacheops 0
62#else
63#define cpu_has_safe_index_cacheops 1
64#endif
65
Ralf Baechleec74e362005-07-13 11:48:45 +000066/*
67 * Must die.
68 */
69static unsigned long icache_size __read_mostly;
70static unsigned long dcache_size __read_mostly;
71static unsigned long scache_size __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070072
73/*
74 * Dummy cache handling routines for machines without boardcaches
75 */
Chris Dearman73f40352006-06-20 18:06:52 +010076static void cache_noop(void) {}
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
78static struct bcache_ops no_sc_ops = {
Chris Dearman73f40352006-06-20 18:06:52 +010079 .bc_enable = (void *)cache_noop,
80 .bc_disable = (void *)cache_noop,
81 .bc_wback_inv = (void *)cache_noop,
82 .bc_inv = (void *)cache_noop
Linus Torvalds1da177e2005-04-16 15:20:36 -070083};
84
85struct bcache_ops *bcops = &no_sc_ops;
86
Thiemo Seufer330cfe02005-09-01 18:33:58 +000087#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
88#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
Linus Torvalds1da177e2005-04-16 15:20:36 -070089
90#define R4600_HIT_CACHEOP_WAR_IMPL \
91do { \
92 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \
93 *(volatile unsigned long *)CKSEG1; \
94 if (R4600_V1_HIT_CACHEOP_WAR) \
95 __asm__ __volatile__("nop;nop;nop;nop"); \
96} while (0)
97
98static void (*r4k_blast_dcache_page)(unsigned long addr);
99
100static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
101{
102 R4600_HIT_CACHEOP_WAR_IMPL;
103 blast_dcache32_page(addr);
104}
105
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700106static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
107{
108 R4600_HIT_CACHEOP_WAR_IMPL;
109 blast_dcache64_page(addr);
110}
111
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000112static void r4k_blast_dcache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113{
114 unsigned long dc_lsize = cpu_dcache_line_size();
115
Chris Dearman73f40352006-06-20 18:06:52 +0100116 if (dc_lsize == 0)
117 r4k_blast_dcache_page = (void *)cache_noop;
118 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119 r4k_blast_dcache_page = blast_dcache16_page;
120 else if (dc_lsize == 32)
121 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700122 else if (dc_lsize == 64)
123 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124}
125
126static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
127
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000128static void r4k_blast_dcache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129{
130 unsigned long dc_lsize = cpu_dcache_line_size();
131
Chris Dearman73f40352006-06-20 18:06:52 +0100132 if (dc_lsize == 0)
133 r4k_blast_dcache_page_indexed = (void *)cache_noop;
134 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
136 else if (dc_lsize == 32)
137 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700138 else if (dc_lsize == 64)
139 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140}
141
Sanjay Lalf2e36562012-11-21 18:34:10 -0800142void (* r4k_blast_dcache)(void);
143EXPORT_SYMBOL(r4k_blast_dcache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000145static void r4k_blast_dcache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146{
147 unsigned long dc_lsize = cpu_dcache_line_size();
148
Chris Dearman73f40352006-06-20 18:06:52 +0100149 if (dc_lsize == 0)
150 r4k_blast_dcache = (void *)cache_noop;
151 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 r4k_blast_dcache = blast_dcache16;
153 else if (dc_lsize == 32)
154 r4k_blast_dcache = blast_dcache32;
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700155 else if (dc_lsize == 64)
156 r4k_blast_dcache = blast_dcache64;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157}
158
159/* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
160#define JUMP_TO_ALIGN(order) \
161 __asm__ __volatile__( \
162 "b\t1f\n\t" \
163 ".align\t" #order "\n\t" \
164 "1:\n\t" \
165 )
166#define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
Ralf Baechle70342282013-01-22 12:59:30 +0100167#define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
169static inline void blast_r4600_v1_icache32(void)
170{
171 unsigned long flags;
172
173 local_irq_save(flags);
174 blast_icache32();
175 local_irq_restore(flags);
176}
177
178static inline void tx49_blast_icache32(void)
179{
180 unsigned long start = INDEX_BASE;
181 unsigned long end = start + current_cpu_data.icache.waysize;
182 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
183 unsigned long ws_end = current_cpu_data.icache.ways <<
Ralf Baechle70342282013-01-22 12:59:30 +0100184 current_cpu_data.icache.waybit;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 unsigned long ws, addr;
186
187 CACHE32_UNROLL32_ALIGN2;
188 /* I'm in even chunk. blast odd chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700189 for (ws = 0; ws < ws_end; ws += ws_inc)
190 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100191 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192 CACHE32_UNROLL32_ALIGN;
193 /* I'm in odd chunk. blast even chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700194 for (ws = 0; ws < ws_end; ws += ws_inc)
195 for (addr = start; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100196 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197}
198
199static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
200{
201 unsigned long flags;
202
203 local_irq_save(flags);
204 blast_icache32_page_indexed(page);
205 local_irq_restore(flags);
206}
207
208static inline void tx49_blast_icache32_page_indexed(unsigned long page)
209{
Atsushi Nemoto67a3f6de2006-04-04 17:34:14 +0900210 unsigned long indexmask = current_cpu_data.icache.waysize - 1;
211 unsigned long start = INDEX_BASE + (page & indexmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 unsigned long end = start + PAGE_SIZE;
213 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
214 unsigned long ws_end = current_cpu_data.icache.ways <<
Ralf Baechle70342282013-01-22 12:59:30 +0100215 current_cpu_data.icache.waybit;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 unsigned long ws, addr;
217
218 CACHE32_UNROLL32_ALIGN2;
219 /* I'm in even chunk. blast odd chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700220 for (ws = 0; ws < ws_end; ws += ws_inc)
221 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100222 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 CACHE32_UNROLL32_ALIGN;
224 /* I'm in odd chunk. blast even chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700225 for (ws = 0; ws < ws_end; ws += ws_inc)
226 for (addr = start; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100227 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228}
229
230static void (* r4k_blast_icache_page)(unsigned long addr);
231
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000232static void r4k_blast_icache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233{
234 unsigned long ic_lsize = cpu_icache_line_size();
235
Chris Dearman73f40352006-06-20 18:06:52 +0100236 if (ic_lsize == 0)
237 r4k_blast_icache_page = (void *)cache_noop;
238 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 r4k_blast_icache_page = blast_icache16_page;
240 else if (ic_lsize == 32)
241 r4k_blast_icache_page = blast_icache32_page;
242 else if (ic_lsize == 64)
243 r4k_blast_icache_page = blast_icache64_page;
244}
245
246
247static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
248
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000249static void r4k_blast_icache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250{
251 unsigned long ic_lsize = cpu_icache_line_size();
252
Chris Dearman73f40352006-06-20 18:06:52 +0100253 if (ic_lsize == 0)
254 r4k_blast_icache_page_indexed = (void *)cache_noop;
255 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
257 else if (ic_lsize == 32) {
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000258 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 r4k_blast_icache_page_indexed =
260 blast_icache32_r4600_v1_page_indexed;
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000261 else if (TX49XX_ICACHE_INDEX_INV_WAR)
262 r4k_blast_icache_page_indexed =
263 tx49_blast_icache32_page_indexed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 else
265 r4k_blast_icache_page_indexed =
266 blast_icache32_page_indexed;
267 } else if (ic_lsize == 64)
268 r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
269}
270
Sanjay Lalf2e36562012-11-21 18:34:10 -0800271void (* r4k_blast_icache)(void);
272EXPORT_SYMBOL(r4k_blast_icache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000274static void r4k_blast_icache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275{
276 unsigned long ic_lsize = cpu_icache_line_size();
277
Chris Dearman73f40352006-06-20 18:06:52 +0100278 if (ic_lsize == 0)
279 r4k_blast_icache = (void *)cache_noop;
280 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281 r4k_blast_icache = blast_icache16;
282 else if (ic_lsize == 32) {
283 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
284 r4k_blast_icache = blast_r4600_v1_icache32;
285 else if (TX49XX_ICACHE_INDEX_INV_WAR)
286 r4k_blast_icache = tx49_blast_icache32;
287 else
288 r4k_blast_icache = blast_icache32;
289 } else if (ic_lsize == 64)
290 r4k_blast_icache = blast_icache64;
291}
292
293static void (* r4k_blast_scache_page)(unsigned long addr);
294
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000295static void r4k_blast_scache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296{
297 unsigned long sc_lsize = cpu_scache_line_size();
298
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000299 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100300 r4k_blast_scache_page = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000301 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 r4k_blast_scache_page = blast_scache16_page;
303 else if (sc_lsize == 32)
304 r4k_blast_scache_page = blast_scache32_page;
305 else if (sc_lsize == 64)
306 r4k_blast_scache_page = blast_scache64_page;
307 else if (sc_lsize == 128)
308 r4k_blast_scache_page = blast_scache128_page;
309}
310
311static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
312
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000313static void r4k_blast_scache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314{
315 unsigned long sc_lsize = cpu_scache_line_size();
316
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000317 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100318 r4k_blast_scache_page_indexed = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000319 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320 r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
321 else if (sc_lsize == 32)
322 r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
323 else if (sc_lsize == 64)
324 r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
325 else if (sc_lsize == 128)
326 r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
327}
328
329static void (* r4k_blast_scache)(void);
330
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000331static void r4k_blast_scache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332{
333 unsigned long sc_lsize = cpu_scache_line_size();
334
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000335 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100336 r4k_blast_scache = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000337 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 r4k_blast_scache = blast_scache16;
339 else if (sc_lsize == 32)
340 r4k_blast_scache = blast_scache32;
341 else if (sc_lsize == 64)
342 r4k_blast_scache = blast_scache64;
343 else if (sc_lsize == 128)
344 r4k_blast_scache = blast_scache128;
345}
346
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347static inline void local_r4k___flush_cache_all(void * args)
348{
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800349#if defined(CONFIG_CPU_LOONGSON2)
350 r4k_blast_scache();
351 return;
352#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 r4k_blast_dcache();
354 r4k_blast_icache();
355
Ralf Baechle10cc3522007-10-11 23:46:15 +0100356 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 case CPU_R4000SC:
358 case CPU_R4000MC:
359 case CPU_R4400SC:
360 case CPU_R4400MC:
361 case CPU_R10000:
362 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -0400363 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 r4k_blast_scache();
365 }
366}
367
368static void r4k___flush_cache_all(void)
369{
Ralf Baechle48a26e62010-10-29 19:08:25 +0100370 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371}
372
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100373static inline int has_valid_asid(const struct mm_struct *mm)
374{
375#if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC)
376 int i;
377
378 for_each_online_cpu(i)
379 if (cpu_context(i, mm))
380 return 1;
381
382 return 0;
383#else
384 return cpu_context(smp_processor_id(), mm);
385#endif
386}
387
Ralf Baechle9c5a3d72008-04-05 15:13:23 +0100388static void r4k__flush_cache_vmap(void)
389{
390 r4k_blast_dcache();
391}
392
393static void r4k__flush_cache_vunmap(void)
394{
395 r4k_blast_dcache();
396}
397
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398static inline void local_r4k_flush_cache_range(void * args)
399{
400 struct vm_area_struct *vma = args;
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000401 int exec = vma->vm_flags & VM_EXEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100403 if (!(has_valid_asid(vma->vm_mm)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 return;
405
Atsushi Nemoto0550d9d2006-08-22 21:15:47 +0900406 r4k_blast_dcache();
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000407 if (exec)
408 r4k_blast_icache();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409}
410
411static void r4k_flush_cache_range(struct vm_area_struct *vma,
412 unsigned long start, unsigned long end)
413{
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000414 int exec = vma->vm_flags & VM_EXEC;
Atsushi Nemoto0550d9d2006-08-22 21:15:47 +0900415
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000416 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
Ralf Baechle48a26e62010-10-29 19:08:25 +0100417 r4k_on_each_cpu(local_r4k_flush_cache_range, vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418}
419
420static inline void local_r4k_flush_cache_mm(void * args)
421{
422 struct mm_struct *mm = args;
423
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100424 if (!has_valid_asid(mm))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 return;
426
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 /*
428 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we
429 * only flush the primary caches but R10000 and R12000 behave sane ...
Ralf Baechle617667b2006-11-30 01:14:48 +0000430 * R4000SC and R4400SC indexed S-cache ops also invalidate primary
431 * caches, so we can bail out early.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 */
Ralf Baechle10cc3522007-10-11 23:46:15 +0100433 if (current_cpu_type() == CPU_R4000SC ||
434 current_cpu_type() == CPU_R4000MC ||
435 current_cpu_type() == CPU_R4400SC ||
436 current_cpu_type() == CPU_R4400MC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 r4k_blast_scache();
Ralf Baechle617667b2006-11-30 01:14:48 +0000438 return;
439 }
440
441 r4k_blast_dcache();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442}
443
444static void r4k_flush_cache_mm(struct mm_struct *mm)
445{
446 if (!cpu_has_dc_aliases)
447 return;
448
Ralf Baechle48a26e62010-10-29 19:08:25 +0100449 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450}
451
452struct flush_cache_page_args {
453 struct vm_area_struct *vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100454 unsigned long addr;
Atsushi Nemotode628932006-03-13 18:23:03 +0900455 unsigned long pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456};
457
458static inline void local_r4k_flush_cache_page(void *args)
459{
460 struct flush_cache_page_args *fcp_args = args;
461 struct vm_area_struct *vma = fcp_args->vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100462 unsigned long addr = fcp_args->addr;
Ralf Baechledb813fe2007-09-27 18:26:43 +0100463 struct page *page = pfn_to_page(fcp_args->pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 int exec = vma->vm_flags & VM_EXEC;
465 struct mm_struct *mm = vma->vm_mm;
Ralf Baechlec9c50232008-06-14 22:22:08 +0100466 int map_coherent = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 pgd_t *pgdp;
Ralf Baechlec6e8b582005-02-10 12:19:59 +0000468 pud_t *pudp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 pmd_t *pmdp;
470 pte_t *ptep;
Ralf Baechledb813fe2007-09-27 18:26:43 +0100471 void *vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472
Ralf Baechle79acf832005-02-10 13:54:37 +0000473 /*
474 * If ownes no valid ASID yet, cannot possibly have gotten
475 * this page into the cache.
476 */
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100477 if (!has_valid_asid(mm))
Ralf Baechle79acf832005-02-10 13:54:37 +0000478 return;
479
Ralf Baechle6ec25802005-10-12 00:02:34 +0100480 addr &= PAGE_MASK;
481 pgdp = pgd_offset(mm, addr);
482 pudp = pud_offset(pgdp, addr);
483 pmdp = pmd_offset(pudp, addr);
484 ptep = pte_offset(pmdp, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485
486 /*
487 * If the page isn't marked valid, the page cannot possibly be
488 * in the cache.
489 */
Ralf Baechle526af352008-01-29 10:14:55 +0000490 if (!(pte_present(*ptep)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491 return;
492
Ralf Baechledb813fe2007-09-27 18:26:43 +0100493 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
494 vaddr = NULL;
495 else {
496 /*
497 * Use kmap_coherent or kmap_atomic to do flushes for
498 * another ASID than the current one.
499 */
Ralf Baechlec9c50232008-06-14 22:22:08 +0100500 map_coherent = (cpu_has_dc_aliases &&
501 page_mapped(page) && !Page_dcache_dirty(page));
502 if (map_coherent)
Ralf Baechledb813fe2007-09-27 18:26:43 +0100503 vaddr = kmap_coherent(page, addr);
504 else
Cong Wang9c020482011-11-25 23:14:15 +0800505 vaddr = kmap_atomic(page);
Ralf Baechledb813fe2007-09-27 18:26:43 +0100506 addr = (unsigned long)vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 }
508
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
Ralf Baechledb813fe2007-09-27 18:26:43 +0100510 r4k_blast_dcache_page(addr);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100511 if (exec && !cpu_icache_snoops_remote_store)
512 r4k_blast_scache_page(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 }
514 if (exec) {
Ralf Baechledb813fe2007-09-27 18:26:43 +0100515 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 int cpu = smp_processor_id();
517
Thiemo Seufer26a51b22005-02-19 13:32:02 +0000518 if (cpu_context(cpu, mm) != 0)
519 drop_mmu_context(mm, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 } else
Ralf Baechledb813fe2007-09-27 18:26:43 +0100521 r4k_blast_icache_page(addr);
522 }
523
524 if (vaddr) {
Ralf Baechlec9c50232008-06-14 22:22:08 +0100525 if (map_coherent)
Ralf Baechledb813fe2007-09-27 18:26:43 +0100526 kunmap_coherent();
527 else
Cong Wang9c020482011-11-25 23:14:15 +0800528 kunmap_atomic(vaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529 }
530}
531
Ralf Baechle6ec25802005-10-12 00:02:34 +0100532static void r4k_flush_cache_page(struct vm_area_struct *vma,
533 unsigned long addr, unsigned long pfn)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534{
535 struct flush_cache_page_args args;
536
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 args.vma = vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100538 args.addr = addr;
Atsushi Nemotode628932006-03-13 18:23:03 +0900539 args.pfn = pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540
Ralf Baechle48a26e62010-10-29 19:08:25 +0100541 r4k_on_each_cpu(local_r4k_flush_cache_page, &args);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542}
543
544static inline void local_r4k_flush_data_cache_page(void * addr)
545{
546 r4k_blast_dcache_page((unsigned long) addr);
547}
548
549static void r4k_flush_data_cache_page(unsigned long addr)
550{
Ralf Baechlea754f702007-11-03 01:01:37 +0000551 if (in_atomic())
552 local_r4k_flush_data_cache_page((void *)addr);
553 else
Ralf Baechle48a26e62010-10-29 19:08:25 +0100554 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555}
556
557struct flush_icache_range_args {
Atsushi Nemotod4264f12006-01-29 02:27:51 +0900558 unsigned long start;
559 unsigned long end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560};
561
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +0200562static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 if (!cpu_has_ic_fills_f_dc) {
Chris Dearman73f40352006-06-20 18:06:52 +0100565 if (end - start >= dcache_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 r4k_blast_dcache();
567 } else {
Thiemo Seufer10a3dab2005-09-09 20:26:54 +0000568 R4600_HIT_CACHEOP_WAR_IMPL;
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900569 protected_blast_dcache_range(start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 }
572
573 if (end - start > icache_size)
574 r4k_blast_icache();
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900575 else
576 protected_blast_icache_range(start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577}
578
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +0200579static inline void local_r4k_flush_icache_range_ipi(void *args)
580{
581 struct flush_icache_range_args *fir_args = args;
582 unsigned long start = fir_args->start;
583 unsigned long end = fir_args->end;
584
585 local_r4k_flush_icache_range(start, end);
586}
587
Atsushi Nemotod4264f12006-01-29 02:27:51 +0900588static void r4k_flush_icache_range(unsigned long start, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589{
590 struct flush_icache_range_args args;
591
592 args.start = start;
593 args.end = end;
594
Ralf Baechle48a26e62010-10-29 19:08:25 +0100595 r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args);
Ralf Baechlecc61c1f2005-07-12 18:35:38 +0000596 instruction_hazard();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597}
598
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599#ifdef CONFIG_DMA_NONCOHERENT
600
601static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
602{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603 /* Catch bad driver code */
604 BUG_ON(size == 0);
605
Ralf Baechleff522052013-09-17 12:44:31 +0200606 preempt_disable();
Ralf Baechlefc5d2d22006-07-06 13:04:01 +0100607 if (cpu_has_inclusive_pcaches) {
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900608 if (size >= scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 r4k_blast_scache();
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900610 else
611 blast_scache_range(addr, addr + size);
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700612 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 return;
614 }
615
616 /*
617 * Either no secondary cache or the available caches don't have the
618 * subset property so we have to flush the primary caches
619 * explicitly
620 */
Ralf Baechle39b8d522008-04-28 17:14:26 +0100621 if (cpu_has_safe_index_cacheops && size >= dcache_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 r4k_blast_dcache();
623 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 R4600_HIT_CACHEOP_WAR_IMPL;
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900625 blast_dcache_range(addr, addr + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626 }
Ralf Baechleff522052013-09-17 12:44:31 +0200627 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628
629 bc_wback_inv(addr, size);
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700630 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631}
632
633static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
634{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 /* Catch bad driver code */
636 BUG_ON(size == 0);
637
Ralf Baechleff522052013-09-17 12:44:31 +0200638 preempt_disable();
Ralf Baechlefc5d2d22006-07-06 13:04:01 +0100639 if (cpu_has_inclusive_pcaches) {
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900640 if (size >= scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641 r4k_blast_scache();
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000642 else {
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000643 /*
644 * There is no clearly documented alignment requirement
645 * for the cache instruction on MIPS processors and
646 * some processors, among them the RM5200 and RM7000
647 * QED processors will throw an address error for cache
Ralf Baechle70342282013-01-22 12:59:30 +0100648 * hit ops with insufficient alignment. Solved by
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000649 * aligning the address to cache line size.
650 */
Thomas Bogendoerfere9c33572007-11-26 23:40:01 +0100651 blast_inv_scache_range(addr, addr + size);
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000652 }
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700653 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 return;
655 }
656
Ralf Baechle39b8d522008-04-28 17:14:26 +0100657 if (cpu_has_safe_index_cacheops && size >= dcache_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658 r4k_blast_dcache();
659 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660 R4600_HIT_CACHEOP_WAR_IMPL;
Thomas Bogendoerfere9c33572007-11-26 23:40:01 +0100661 blast_inv_dcache_range(addr, addr + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 }
Ralf Baechleff522052013-09-17 12:44:31 +0200663 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664
665 bc_inv(addr, size);
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700666 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667}
668#endif /* CONFIG_DMA_NONCOHERENT */
669
670/*
671 * While we're protected against bad userland addresses we don't care
672 * very much about what happens in that case. Usually a segmentation
673 * fault will dump the process later on anyway ...
674 */
675static void local_r4k_flush_cache_sigtramp(void * arg)
676{
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000677 unsigned long ic_lsize = cpu_icache_line_size();
678 unsigned long dc_lsize = cpu_dcache_line_size();
679 unsigned long sc_lsize = cpu_scache_line_size();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 unsigned long addr = (unsigned long) arg;
681
682 R4600_HIT_CACHEOP_WAR_IMPL;
Chris Dearman73f40352006-06-20 18:06:52 +0100683 if (dc_lsize)
684 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000685 if (!cpu_icache_snoops_remote_store && scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 protected_writeback_scache_line(addr & ~(sc_lsize - 1));
Chris Dearman73f40352006-06-20 18:06:52 +0100687 if (ic_lsize)
688 protected_flush_icache_line(addr & ~(ic_lsize - 1));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 if (MIPS4K_ICACHE_REFILL_WAR) {
690 __asm__ __volatile__ (
691 ".set push\n\t"
692 ".set noat\n\t"
693 ".set mips3\n\t"
Ralf Baechle875d43e2005-09-03 15:56:16 -0700694#ifdef CONFIG_32BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695 "la $at,1f\n\t"
696#endif
Ralf Baechle875d43e2005-09-03 15:56:16 -0700697#ifdef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 "dla $at,1f\n\t"
699#endif
700 "cache %0,($at)\n\t"
701 "nop; nop; nop\n"
702 "1:\n\t"
703 ".set pop"
704 :
705 : "i" (Hit_Invalidate_I));
706 }
707 if (MIPS_CACHE_SYNC_WAR)
708 __asm__ __volatile__ ("sync");
709}
710
711static void r4k_flush_cache_sigtramp(unsigned long addr)
712{
Ralf Baechle48a26e62010-10-29 19:08:25 +0100713 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714}
715
716static void r4k_flush_icache_all(void)
717{
718 if (cpu_has_vtag_icache)
719 r4k_blast_icache();
720}
721
Ralf Baechled9cdc9012011-06-17 16:20:28 +0100722struct flush_kernel_vmap_range_args {
723 unsigned long vaddr;
724 int size;
725};
726
727static inline void local_r4k_flush_kernel_vmap_range(void *args)
728{
729 struct flush_kernel_vmap_range_args *vmra = args;
730 unsigned long vaddr = vmra->vaddr;
731 int size = vmra->size;
732
733 /*
734 * Aliases only affect the primary caches so don't bother with
735 * S-caches or T-caches.
736 */
737 if (cpu_has_safe_index_cacheops && size >= dcache_size)
738 r4k_blast_dcache();
739 else {
740 R4600_HIT_CACHEOP_WAR_IMPL;
741 blast_dcache_range(vaddr, vaddr + size);
742 }
743}
744
745static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size)
746{
747 struct flush_kernel_vmap_range_args args;
748
749 args.vaddr = (unsigned long) vaddr;
750 args.size = size;
751
752 r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args);
753}
754
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755static inline void rm7k_erratum31(void)
756{
757 const unsigned long ic_lsize = 32;
758 unsigned long addr;
759
760 /* RM7000 erratum #31. The icache is screwed at startup. */
761 write_c0_taglo(0);
762 write_c0_taghi(0);
763
764 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
765 __asm__ __volatile__ (
Thiemo Seuferd8748a32005-09-02 09:56:12 +0000766 ".set push\n\t"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 ".set noreorder\n\t"
768 ".set mips3\n\t"
769 "cache\t%1, 0(%0)\n\t"
770 "cache\t%1, 0x1000(%0)\n\t"
771 "cache\t%1, 0x2000(%0)\n\t"
772 "cache\t%1, 0x3000(%0)\n\t"
773 "cache\t%2, 0(%0)\n\t"
774 "cache\t%2, 0x1000(%0)\n\t"
775 "cache\t%2, 0x2000(%0)\n\t"
776 "cache\t%2, 0x3000(%0)\n\t"
777 "cache\t%1, 0(%0)\n\t"
778 "cache\t%1, 0x1000(%0)\n\t"
779 "cache\t%1, 0x2000(%0)\n\t"
780 "cache\t%1, 0x3000(%0)\n\t"
Thiemo Seuferd8748a32005-09-02 09:56:12 +0000781 ".set pop\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782 :
783 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
784 }
785}
786
Steven J. Hill006a8512012-06-26 04:11:03 +0000787static inline void alias_74k_erratum(struct cpuinfo_mips *c)
788{
Maciej W. Rozycki9213ad72013-09-18 19:08:15 +0100789 unsigned int imp = c->processor_id & PRID_IMP_MASK;
790 unsigned int rev = c->processor_id & PRID_REV_MASK;
791
Steven J. Hill006a8512012-06-26 04:11:03 +0000792 /*
793 * Early versions of the 74K do not update the cache tags on a
794 * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG
795 * aliases. In this case it is better to treat the cache as always
796 * having aliases.
797 */
Maciej W. Rozycki9213ad72013-09-18 19:08:15 +0100798 switch (imp) {
799 case PRID_IMP_74K:
800 if (rev <= PRID_REV_ENCODE_332(2, 4, 0))
801 c->dcache.flags |= MIPS_CACHE_VTAG;
802 if (rev == PRID_REV_ENCODE_332(2, 4, 0))
803 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
804 break;
805 case PRID_IMP_1074K:
806 if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) {
807 c->dcache.flags |= MIPS_CACHE_VTAG;
808 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
809 }
810 break;
811 default:
812 BUG();
Steven J. Hill006a8512012-06-26 04:11:03 +0000813 }
814}
815
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000816static char *way_string[] = { NULL, "direct mapped", "2-way",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
818};
819
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000820static void probe_pcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821{
822 struct cpuinfo_mips *c = &current_cpu_data;
823 unsigned int config = read_c0_config();
824 unsigned int prid = read_c0_prid();
825 unsigned long config1;
826 unsigned int lsize;
827
Ralf Baechle69f24d12013-09-17 10:25:47 +0200828 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829 case CPU_R4600: /* QED style two way caches? */
830 case CPU_R4700:
831 case CPU_R5000:
832 case CPU_NEVADA:
833 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
834 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
835 c->icache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900836 c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837
838 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
839 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
840 c->dcache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900841 c->dcache.waybit= __ffs(dcache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842
843 c->options |= MIPS_CPU_CACHE_CDEX_P;
844 break;
845
846 case CPU_R5432:
847 case CPU_R5500:
848 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
849 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
850 c->icache.ways = 2;
851 c->icache.waybit= 0;
852
853 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
854 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
855 c->dcache.ways = 2;
856 c->dcache.waybit = 0;
857
Shinya Kuribayashi58648102009-03-18 09:04:01 +0900858 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859 break;
860
861 case CPU_TX49XX:
862 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
863 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
864 c->icache.ways = 4;
865 c->icache.waybit= 0;
866
867 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
868 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
869 c->dcache.ways = 4;
870 c->dcache.waybit = 0;
871
872 c->options |= MIPS_CPU_CACHE_CDEX_P;
Atsushi Nemotode862b42006-03-17 12:59:22 +0900873 c->options |= MIPS_CPU_PREFETCH;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 break;
875
876 case CPU_R4000PC:
877 case CPU_R4000SC:
878 case CPU_R4000MC:
879 case CPU_R4400PC:
880 case CPU_R4400SC:
881 case CPU_R4400MC:
882 case CPU_R4300:
883 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
884 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
885 c->icache.ways = 1;
Ralf Baechle70342282013-01-22 12:59:30 +0100886 c->icache.waybit = 0; /* doesn't matter */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887
888 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
889 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
890 c->dcache.ways = 1;
891 c->dcache.waybit = 0; /* does not matter */
892
893 c->options |= MIPS_CPU_CACHE_CDEX_P;
894 break;
895
896 case CPU_R10000:
897 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -0400898 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
900 c->icache.linesz = 64;
901 c->icache.ways = 2;
902 c->icache.waybit = 0;
903
904 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
905 c->dcache.linesz = 32;
906 c->dcache.ways = 2;
907 c->dcache.waybit = 0;
908
909 c->options |= MIPS_CPU_PREFETCH;
910 break;
911
912 case CPU_VR4133:
Yoichi Yuasa2874fe52006-07-08 00:42:12 +0900913 write_c0_config(config & ~VR41_CONF_P4K);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 case CPU_VR4131:
915 /* Workaround for cache instruction bug of VR4131 */
916 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
917 c->processor_id == 0x0c82U) {
Yoichi Yuasa4e8ab362006-07-04 22:59:41 +0900918 config |= 0x00400000U;
919 if (c->processor_id == 0x0c80U)
920 config |= VR41_CONF_BP;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921 write_c0_config(config);
Yoichi Yuasa1058ecd2006-07-08 00:42:01 +0900922 } else
923 c->options |= MIPS_CPU_CACHE_CDEX_P;
924
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
926 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
927 c->icache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900928 c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929
930 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
931 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
932 c->dcache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900933 c->dcache.waybit = __ffs(dcache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934 break;
935
936 case CPU_VR41XX:
937 case CPU_VR4111:
938 case CPU_VR4121:
939 case CPU_VR4122:
940 case CPU_VR4181:
941 case CPU_VR4181A:
942 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
943 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
944 c->icache.ways = 1;
Ralf Baechle70342282013-01-22 12:59:30 +0100945 c->icache.waybit = 0; /* doesn't matter */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946
947 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
948 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
949 c->dcache.ways = 1;
950 c->dcache.waybit = 0; /* does not matter */
951
952 c->options |= MIPS_CPU_CACHE_CDEX_P;
953 break;
954
955 case CPU_RM7000:
956 rm7k_erratum31();
957
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
959 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
960 c->icache.ways = 4;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900961 c->icache.waybit = __ffs(icache_size / c->icache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962
963 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
964 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
965 c->dcache.ways = 4;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900966 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 c->options |= MIPS_CPU_CACHE_CDEX_P;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 c->options |= MIPS_CPU_PREFETCH;
970 break;
971
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800972 case CPU_LOONGSON2:
973 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
974 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
975 if (prid & 0x3)
976 c->icache.ways = 4;
977 else
978 c->icache.ways = 2;
979 c->icache.waybit = 0;
980
981 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
982 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
983 if (prid & 0x3)
984 c->dcache.ways = 4;
985 else
986 c->dcache.ways = 2;
987 c->dcache.waybit = 0;
988 break;
989
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990 default:
991 if (!(config & MIPS_CONF_M))
992 panic("Don't know how to probe P-caches on this cpu.");
993
994 /*
995 * So we seem to be a MIPS32 or MIPS64 CPU
996 * So let's probe the I-cache ...
997 */
998 config1 = read_c0_config1();
999
1000 if ((lsize = ((config1 >> 19) & 7)))
1001 c->icache.linesz = 2 << lsize;
1002 else
1003 c->icache.linesz = lsize;
Douglas Leungdc34b052012-07-19 09:11:13 +02001004 c->icache.sets = 32 << (((config1 >> 22) + 1) & 7);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 c->icache.ways = 1 + ((config1 >> 16) & 7);
1006
1007 icache_size = c->icache.sets *
Ralf Baechle70342282013-01-22 12:59:30 +01001008 c->icache.ways *
1009 c->icache.linesz;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +09001010 c->icache.waybit = __ffs(icache_size/c->icache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011
1012 if (config & 0x8) /* VI bit */
1013 c->icache.flags |= MIPS_CACHE_VTAG;
1014
1015 /*
1016 * Now probe the MIPS32 / MIPS64 data cache.
1017 */
1018 c->dcache.flags = 0;
1019
1020 if ((lsize = ((config1 >> 10) & 7)))
1021 c->dcache.linesz = 2 << lsize;
1022 else
1023 c->dcache.linesz= lsize;
Douglas Leungdc34b052012-07-19 09:11:13 +02001024 c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025 c->dcache.ways = 1 + ((config1 >> 7) & 7);
1026
1027 dcache_size = c->dcache.sets *
Ralf Baechle70342282013-01-22 12:59:30 +01001028 c->dcache.ways *
1029 c->dcache.linesz;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +09001030 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031
1032 c->options |= MIPS_CPU_PREFETCH;
1033 break;
1034 }
1035
1036 /*
1037 * Processor configuration sanity check for the R4000SC erratum
Ralf Baechle70342282013-01-22 12:59:30 +01001038 * #5. With page sizes larger than 32kB there is no possibility
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 * to get a VCE exception anymore so we don't care about this
1040 * misconfiguration. The case is rather theoretical anyway;
1041 * presumably no vendor is shipping his hardware in the "bad"
1042 * configuration.
1043 */
Maciej W. Rozycki8ff374b2013-09-17 16:58:10 +01001044 if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 &&
1045 (prid & PRID_REV_MASK) < PRID_REV_R4400 &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046 !(config & CONF_SC) && c->icache.linesz != 16 &&
1047 PAGE_SIZE <= 0x8000)
1048 panic("Improper R4000SC processor configuration detected");
1049
1050 /* compute a couple of other cache variables */
1051 c->icache.waysize = icache_size / c->icache.ways;
1052 c->dcache.waysize = dcache_size / c->dcache.ways;
1053
Chris Dearman73f40352006-06-20 18:06:52 +01001054 c->icache.sets = c->icache.linesz ?
1055 icache_size / (c->icache.linesz * c->icache.ways) : 0;
1056 c->dcache.sets = c->dcache.linesz ?
1057 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058
1059 /*
1060 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB
1061 * 2-way virtually indexed so normally would suffer from aliases. So
1062 * normally they'd suffer from aliases but magic in the hardware deals
1063 * with that for us so we don't need to take care ourselves.
1064 */
Ralf Baechle69f24d12013-09-17 10:25:47 +02001065 switch (current_cpu_type()) {
Ralf Baechlea95970f2005-02-07 21:41:32 +00001066 case CPU_20KC:
Ralf Baechle505403b2005-02-07 21:53:39 +00001067 case CPU_25KF:
Ralf Baechle641e97f2007-10-11 23:46:05 +01001068 case CPU_SB1:
1069 case CPU_SB1A:
Jayachandran Cefa0f812011-05-07 01:36:21 +05301070 case CPU_XLR:
Atsushi Nemotode628932006-03-13 18:23:03 +09001071 c->dcache.flags |= MIPS_CACHE_PINDEX;
Ralf Baechle641e97f2007-10-11 23:46:05 +01001072 break;
1073
Ralf Baechled1e344e2005-02-04 15:51:26 +00001074 case CPU_R10000:
1075 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -04001076 case CPU_R14000:
Ralf Baechled1e344e2005-02-04 15:51:26 +00001077 break;
Ralf Baechle641e97f2007-10-11 23:46:05 +01001078
Steven J. Hill113c62d2012-07-06 23:56:00 +02001079 case CPU_M14KC:
Steven J. Hillf8fa4812012-12-07 03:51:35 +00001080 case CPU_M14KEC:
Ralf Baechled1e344e2005-02-04 15:51:26 +00001081 case CPU_24K:
Nigel Stephens98a41de2006-04-27 15:50:32 +01001082 case CPU_34K:
Ralf Baechle2e78ae32006-06-23 18:48:21 +01001083 case CPU_74K:
Ralf Baechle39b8d522008-04-28 17:14:26 +01001084 case CPU_1004K:
Ralf Baechle69f24d12013-09-17 10:25:47 +02001085 if (current_cpu_type() == CPU_74K)
Steven J. Hill006a8512012-06-26 04:11:03 +00001086 alias_74k_erratum(c);
Ralf Baechlebeab3752006-06-19 21:56:25 +01001087 if ((read_c0_config7() & (1 << 16))) {
1088 /* effectively physically indexed dcache,
1089 thus no virtual aliases. */
1090 c->dcache.flags |= MIPS_CACHE_PINDEX;
1091 break;
1092 }
Ralf Baechled1e344e2005-02-04 15:51:26 +00001093 default:
Ralf Baechlebeab3752006-06-19 21:56:25 +01001094 if (c->dcache.waysize > PAGE_SIZE)
1095 c->dcache.flags |= MIPS_CACHE_ALIASES;
Ralf Baechled1e344e2005-02-04 15:51:26 +00001096 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001097
Ralf Baechle69f24d12013-09-17 10:25:47 +02001098 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 case CPU_20KC:
1100 /*
1101 * Some older 20Kc chips doesn't have the 'VI' bit in
1102 * the config register.
1103 */
1104 c->icache.flags |= MIPS_CACHE_VTAG;
1105 break;
1106
Manuel Lauss270717a2009-03-25 17:49:28 +01001107 case CPU_ALCHEMY:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1109 break;
1110 }
1111
Ralf Baechle70342282013-01-22 12:59:30 +01001112#ifdef CONFIG_CPU_LOONGSON2
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001113 /*
1114 * LOONGSON2 has 4 way icache, but when using indexed cache op,
1115 * one op will act on all 4 ways
1116 */
1117 c->icache.ways = 1;
1118#endif
1119
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
1121 icache_size >> 10,
Ralf Baechle7fc73162009-04-01 16:11:53 +02001122 c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123 way_string[c->icache.ways], c->icache.linesz);
1124
Ralf Baechle64bfca52007-10-15 16:35:45 +01001125 printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
1126 dcache_size >> 10, way_string[c->dcache.ways],
1127 (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT",
1128 (c->dcache.flags & MIPS_CACHE_ALIASES) ?
1129 "cache aliases" : "no aliases",
1130 c->dcache.linesz);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131}
1132
1133/*
1134 * If you even _breathe_ on this function, look at the gcc output and make sure
1135 * it does not pop things on and off the stack for the cache sizing loop that
1136 * executes in KSEG1 space or else you will crash and burn badly. You have
1137 * been warned.
1138 */
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001139static int probe_scache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001141 unsigned long flags, addr, begin, end, pow2;
1142 unsigned int config = read_c0_config();
1143 struct cpuinfo_mips *c = &current_cpu_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144
1145 if (config & CONF_SC)
1146 return 0;
1147
Ralf Baechlee001e522007-07-28 12:45:47 +01001148 begin = (unsigned long) &_stext;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001149 begin &= ~((4 * 1024 * 1024) - 1);
1150 end = begin + (4 * 1024 * 1024);
1151
1152 /*
1153 * This is such a bitch, you'd think they would make it easy to do
1154 * this. Away you daemons of stupidity!
1155 */
1156 local_irq_save(flags);
1157
1158 /* Fill each size-multiple cache line with a valid tag. */
1159 pow2 = (64 * 1024);
1160 for (addr = begin; addr < end; addr = (begin + pow2)) {
1161 unsigned long *p = (unsigned long *) addr;
1162 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
1163 pow2 <<= 1;
1164 }
1165
1166 /* Load first line with zero (therefore invalid) tag. */
1167 write_c0_taglo(0);
1168 write_c0_taghi(0);
1169 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
1170 cache_op(Index_Store_Tag_I, begin);
1171 cache_op(Index_Store_Tag_D, begin);
1172 cache_op(Index_Store_Tag_SD, begin);
1173
1174 /* Now search for the wrap around point. */
1175 pow2 = (128 * 1024);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1177 cache_op(Index_Load_Tag_SD, addr);
1178 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
1179 if (!read_c0_taglo())
1180 break;
1181 pow2 <<= 1;
1182 }
1183 local_irq_restore(flags);
1184 addr -= begin;
1185
1186 scache_size = addr;
1187 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1188 c->scache.ways = 1;
1189 c->dcache.waybit = 0; /* does not matter */
1190
1191 return 1;
1192}
1193
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001194#if defined(CONFIG_CPU_LOONGSON2)
1195static void __init loongson2_sc_init(void)
1196{
1197 struct cpuinfo_mips *c = &current_cpu_data;
1198
1199 scache_size = 512*1024;
1200 c->scache.linesz = 32;
1201 c->scache.ways = 4;
1202 c->scache.waybit = 0;
1203 c->scache.waysize = scache_size / (c->scache.ways);
1204 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1205 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1206 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1207
1208 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1209}
1210#endif
1211
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212extern int r5k_sc_init(void);
1213extern int rm7k_sc_init(void);
Chris Dearman9318c512006-06-20 17:15:20 +01001214extern int mips_sc_init(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001216static void setup_scache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217{
1218 struct cpuinfo_mips *c = &current_cpu_data;
1219 unsigned int config = read_c0_config();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 int sc_present = 0;
1221
1222 /*
1223 * Do the probing thing on R4000SC and R4400SC processors. Other
1224 * processors don't have a S-cache that would be relevant to the
Joe Perches603e82e2008-02-03 16:54:53 +02001225 * Linux memory management.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226 */
Ralf Baechle69f24d12013-09-17 10:25:47 +02001227 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 case CPU_R4000SC:
1229 case CPU_R4000MC:
1230 case CPU_R4400SC:
1231 case CPU_R4400MC:
Thiemo Seuferba5187d2005-04-25 16:36:23 +00001232 sc_present = run_uncached(probe_scache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233 if (sc_present)
1234 c->options |= MIPS_CPU_CACHE_CDEX_S;
1235 break;
1236
1237 case CPU_R10000:
1238 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -04001239 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1241 c->scache.linesz = 64 << ((config >> 13) & 1);
1242 c->scache.ways = 2;
1243 c->scache.waybit= 0;
1244 sc_present = 1;
1245 break;
1246
1247 case CPU_R5000:
1248 case CPU_NEVADA:
1249#ifdef CONFIG_R5000_CPU_SCACHE
1250 r5k_sc_init();
1251#endif
Ralf Baechle70342282013-01-22 12:59:30 +01001252 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253
1254 case CPU_RM7000:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255#ifdef CONFIG_RM7000_CPU_SCACHE
1256 rm7k_sc_init();
1257#endif
1258 return;
1259
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001260#if defined(CONFIG_CPU_LOONGSON2)
1261 case CPU_LOONGSON2:
1262 loongson2_sc_init();
1263 return;
1264#endif
Jayachandran Ca3d4fb22011-11-16 00:21:20 +00001265 case CPU_XLP:
1266 /* don't need to worry about L2, fully coherent */
1267 return;
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001268
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269 default:
Deng-Cheng Zhuadb37892013-04-01 18:14:28 +00001270 if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 |
1271 MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M64R2)) {
Chris Dearman9318c512006-06-20 17:15:20 +01001272#ifdef CONFIG_MIPS_CPU_SCACHE
1273 if (mips_sc_init ()) {
1274 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz;
1275 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n",
1276 scache_size >> 10,
1277 way_string[c->scache.ways], c->scache.linesz);
1278 }
1279#else
1280 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1281 panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1282#endif
1283 return;
1284 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285 sc_present = 0;
1286 }
1287
1288 if (!sc_present)
1289 return;
1290
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 /* compute a couple of other cache variables */
1292 c->scache.waysize = scache_size / c->scache.ways;
1293
1294 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1295
1296 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1297 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1298
Ralf Baechlefc5d2d22006-07-06 13:04:01 +01001299 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300}
1301
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001302void au1x00_fixup_config_od(void)
1303{
1304 /*
1305 * c0_config.od (bit 19) was write only (and read as 0)
1306 * on the early revisions of Alchemy SOCs. It disables the bus
1307 * transaction overlapping and needs to be set to fix various errata.
1308 */
1309 switch (read_c0_prid()) {
1310 case 0x00030100: /* Au1000 DA */
1311 case 0x00030201: /* Au1000 HA */
1312 case 0x00030202: /* Au1000 HB */
1313 case 0x01030200: /* Au1500 AB */
1314 /*
1315 * Au1100 errata actually keeps silence about this bit, so we set it
1316 * just in case for those revisions that require it to be set according
Manuel Lauss270717a2009-03-25 17:49:28 +01001317 * to the (now gone) cpu table.
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001318 */
1319 case 0x02030200: /* Au1100 AB */
1320 case 0x02030201: /* Au1100 BA */
1321 case 0x02030202: /* Au1100 BC */
1322 set_c0_config(1 << 19);
1323 break;
1324 }
1325}
1326
Ralf Baechle89052bd2008-06-12 17:26:02 +01001327/* CP0 hazard avoidance. */
1328#define NXP_BARRIER() \
1329 __asm__ __volatile__( \
1330 ".set noreorder\n\t" \
1331 "nop; nop; nop; nop; nop; nop;\n\t" \
1332 ".set reorder\n\t")
1333
1334static void nxp_pr4450_fixup_config(void)
1335{
1336 unsigned long config0;
1337
1338 config0 = read_c0_config();
1339
1340 /* clear all three cache coherency fields */
1341 config0 &= ~(0x7 | (7 << 25) | (7 << 28));
1342 config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) |
1343 ((_page_cachable_default >> _CACHE_SHIFT) << 25) |
1344 ((_page_cachable_default >> _CACHE_SHIFT) << 28));
1345 write_c0_config(config0);
1346 NXP_BARRIER();
1347}
1348
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001349static int cca = -1;
Chris Dearman35133692007-09-19 00:58:24 +01001350
1351static int __init cca_setup(char *str)
1352{
1353 get_option(&str, &cca);
1354
Shane McDonaldb5b64f22012-06-14 02:26:40 +00001355 return 0;
Chris Dearman35133692007-09-19 00:58:24 +01001356}
1357
Shane McDonaldb5b64f22012-06-14 02:26:40 +00001358early_param("cca", cca_setup);
Chris Dearman35133692007-09-19 00:58:24 +01001359
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001360static void coherency_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361{
Chris Dearman35133692007-09-19 00:58:24 +01001362 if (cca < 0 || cca > 7)
1363 cca = read_c0_config() & CONF_CM_CMASK;
1364 _page_cachable_default = cca << _CACHE_SHIFT;
1365
1366 pr_debug("Using cache attribute %d\n", cca);
1367 change_c0_config(CONF_CM_CMASK, cca);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001368
1369 /*
1370 * c0_status.cu=0 specifies that updates by the sc instruction use
1371 * the coherency mode specified by the TLB; 1 means cachable
1372 * coherent update on write will be used. Not all processors have
1373 * this bit and; some wire it to zero, others like Toshiba had the
1374 * silly idea of putting something else there ...
1375 */
Ralf Baechle10cc3522007-10-11 23:46:15 +01001376 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001377 case CPU_R4000PC:
1378 case CPU_R4000SC:
1379 case CPU_R4000MC:
1380 case CPU_R4400PC:
1381 case CPU_R4400SC:
1382 case CPU_R4400MC:
1383 clear_c0_config(CONF_CU);
1384 break;
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001385 /*
Ralf Baechledf586d52006-08-01 23:42:30 +01001386 * We need to catch the early Alchemy SOCs with
Manuel Lauss270717a2009-03-25 17:49:28 +01001387 * the write-only co_config.od bit and set it back to one on:
1388 * Au1000 rev DA, HA, HB; Au1100 AB, BA, BC, Au1500 AB
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001389 */
Manuel Lauss270717a2009-03-25 17:49:28 +01001390 case CPU_ALCHEMY:
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001391 au1x00_fixup_config_od();
1392 break;
Ralf Baechle89052bd2008-06-12 17:26:02 +01001393
1394 case PRID_IMP_PR4450:
1395 nxp_pr4450_fixup_config();
1396 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397 }
1398}
1399
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001400static void r4k_cache_error_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401{
Ralf Baechle641e97f2007-10-11 23:46:05 +01001402 extern char __weak except_vec2_generic;
1403 extern char __weak except_vec2_sb1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001404
Ralf Baechle69f24d12013-09-17 10:25:47 +02001405 switch (current_cpu_type()) {
Ralf Baechle641e97f2007-10-11 23:46:05 +01001406 case CPU_SB1:
1407 case CPU_SB1A:
1408 set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
1409 break;
1410
1411 default:
1412 set_uncached_handler(0x100, &except_vec2_generic, 0x80);
1413 break;
1414 }
David Daney9cd9669b2012-05-15 00:04:49 -07001415}
1416
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001417void r4k_cache_init(void)
David Daney9cd9669b2012-05-15 00:04:49 -07001418{
1419 extern void build_clear_page(void);
1420 extern void build_copy_page(void);
1421 struct cpuinfo_mips *c = &current_cpu_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422
1423 probe_pcache();
1424 setup_scache();
1425
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426 r4k_blast_dcache_page_setup();
1427 r4k_blast_dcache_page_indexed_setup();
1428 r4k_blast_dcache_setup();
1429 r4k_blast_icache_page_setup();
1430 r4k_blast_icache_page_indexed_setup();
1431 r4k_blast_icache_setup();
1432 r4k_blast_scache_page_setup();
1433 r4k_blast_scache_page_indexed_setup();
1434 r4k_blast_scache_setup();
1435
1436 /*
1437 * Some MIPS32 and MIPS64 processors have physically indexed caches.
1438 * This code supports virtually indexed processors and will be
1439 * unnecessarily inefficient on physically indexed processors.
1440 */
Chris Dearman73f40352006-06-20 18:06:52 +01001441 if (c->dcache.linesz)
1442 shm_align_mask = max_t( unsigned long,
1443 c->dcache.sets * c->dcache.linesz - 1,
1444 PAGE_SIZE - 1);
1445 else
1446 shm_align_mask = PAGE_SIZE-1;
Ralf Baechle9c5a3d72008-04-05 15:13:23 +01001447
1448 __flush_cache_vmap = r4k__flush_cache_vmap;
1449 __flush_cache_vunmap = r4k__flush_cache_vunmap;
1450
Ralf Baechledb813fe2007-09-27 18:26:43 +01001451 flush_cache_all = cache_noop;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452 __flush_cache_all = r4k___flush_cache_all;
1453 flush_cache_mm = r4k_flush_cache_mm;
1454 flush_cache_page = r4k_flush_cache_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001455 flush_cache_range = r4k_flush_cache_range;
1456
Ralf Baechled9cdc9012011-06-17 16:20:28 +01001457 __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range;
1458
Linus Torvalds1da177e2005-04-16 15:20:36 -07001459 flush_cache_sigtramp = r4k_flush_cache_sigtramp;
1460 flush_icache_all = r4k_flush_icache_all;
Ralf Baechle7e3bfc72006-04-05 20:42:04 +01001461 local_flush_data_cache_page = local_r4k_flush_data_cache_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462 flush_data_cache_page = r4k_flush_data_cache_page;
1463 flush_icache_range = r4k_flush_icache_range;
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +02001464 local_flush_icache_range = local_r4k_flush_icache_range;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465
Ralf Baechle39b8d522008-04-28 17:14:26 +01001466#if defined(CONFIG_DMA_NONCOHERENT)
1467 if (coherentio) {
1468 _dma_cache_wback_inv = (void *)cache_noop;
1469 _dma_cache_wback = (void *)cache_noop;
1470 _dma_cache_inv = (void *)cache_noop;
1471 } else {
1472 _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
1473 _dma_cache_wback = r4k_dma_cache_wback_inv;
1474 _dma_cache_inv = r4k_dma_cache_inv;
1475 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001476#endif
1477
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478 build_clear_page();
1479 build_copy_page();
Steven J. Hillb6d92b42013-03-25 13:47:29 -05001480
1481 /*
1482 * We want to run CMP kernels on core with and without coherent
1483 * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether
1484 * or not to flush caches.
1485 */
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00001486 local_r4k___flush_cache_all(NULL);
Steven J. Hillb6d92b42013-03-25 13:47:29 -05001487
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00001488 coherency_setup();
David Daney9cd9669b2012-05-15 00:04:49 -07001489 board_cache_error_setup = r4k_cache_error_setup;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490}