blob: 0138cb2e456d566636fd24624937f9f1f8b5649e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
9 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/init.h>
11#include <linux/kernel.h>
12#include <linux/sched.h>
13#include <linux/mm.h>
14#include <linux/bitops.h>
15
16#include <asm/bcache.h>
17#include <asm/bootinfo.h>
Ralf Baechleec74e362005-07-13 11:48:45 +000018#include <asm/cache.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <asm/cacheops.h>
20#include <asm/cpu.h>
21#include <asm/cpu-features.h>
22#include <asm/io.h>
23#include <asm/page.h>
24#include <asm/pgtable.h>
25#include <asm/r4kcache.h>
26#include <asm/system.h>
27#include <asm/mmu_context.h>
28#include <asm/war.h>
Thiemo Seuferba5187d2005-04-25 16:36:23 +000029#include <asm/cacheflush.h> /* for run_uncached() */
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010031
32/*
33 * Special Variant of smp_call_function for use by cache functions:
34 *
35 * o No return value
36 * o collapses to normal function call on UP kernels
37 * o collapses to normal function call on systems with a single shared
38 * primary cache.
39 */
40static inline void r4k_on_each_cpu(void (*func) (void *info), void *info,
41 int retry, int wait)
42{
43 preempt_disable();
44
45#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC)
46 smp_call_function(func, info, retry, wait);
47#endif
48 func(info);
49 preempt_enable();
50}
51
Ralf Baechleec74e362005-07-13 11:48:45 +000052/*
53 * Must die.
54 */
55static unsigned long icache_size __read_mostly;
56static unsigned long dcache_size __read_mostly;
57static unsigned long scache_size __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070058
59/*
60 * Dummy cache handling routines for machines without boardcaches
61 */
Chris Dearman73f40352006-06-20 18:06:52 +010062static void cache_noop(void) {}
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
64static struct bcache_ops no_sc_ops = {
Chris Dearman73f40352006-06-20 18:06:52 +010065 .bc_enable = (void *)cache_noop,
66 .bc_disable = (void *)cache_noop,
67 .bc_wback_inv = (void *)cache_noop,
68 .bc_inv = (void *)cache_noop
Linus Torvalds1da177e2005-04-16 15:20:36 -070069};
70
71struct bcache_ops *bcops = &no_sc_ops;
72
Thiemo Seufer330cfe02005-09-01 18:33:58 +000073#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
74#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
Linus Torvalds1da177e2005-04-16 15:20:36 -070075
76#define R4600_HIT_CACHEOP_WAR_IMPL \
77do { \
78 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \
79 *(volatile unsigned long *)CKSEG1; \
80 if (R4600_V1_HIT_CACHEOP_WAR) \
81 __asm__ __volatile__("nop;nop;nop;nop"); \
82} while (0)
83
84static void (*r4k_blast_dcache_page)(unsigned long addr);
85
86static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
87{
88 R4600_HIT_CACHEOP_WAR_IMPL;
89 blast_dcache32_page(addr);
90}
91
Ralf Baechlea00f6312006-08-01 23:39:42 +010092static void __init r4k_blast_dcache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -070093{
94 unsigned long dc_lsize = cpu_dcache_line_size();
95
Chris Dearman73f40352006-06-20 18:06:52 +010096 if (dc_lsize == 0)
97 r4k_blast_dcache_page = (void *)cache_noop;
98 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 r4k_blast_dcache_page = blast_dcache16_page;
100 else if (dc_lsize == 32)
101 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
102}
103
104static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
105
Ralf Baechlea00f6312006-08-01 23:39:42 +0100106static void __init r4k_blast_dcache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107{
108 unsigned long dc_lsize = cpu_dcache_line_size();
109
Chris Dearman73f40352006-06-20 18:06:52 +0100110 if (dc_lsize == 0)
111 r4k_blast_dcache_page_indexed = (void *)cache_noop;
112 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
114 else if (dc_lsize == 32)
115 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
116}
117
118static void (* r4k_blast_dcache)(void);
119
Ralf Baechlea00f6312006-08-01 23:39:42 +0100120static void __init r4k_blast_dcache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121{
122 unsigned long dc_lsize = cpu_dcache_line_size();
123
Chris Dearman73f40352006-06-20 18:06:52 +0100124 if (dc_lsize == 0)
125 r4k_blast_dcache = (void *)cache_noop;
126 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127 r4k_blast_dcache = blast_dcache16;
128 else if (dc_lsize == 32)
129 r4k_blast_dcache = blast_dcache32;
130}
131
132/* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
133#define JUMP_TO_ALIGN(order) \
134 __asm__ __volatile__( \
135 "b\t1f\n\t" \
136 ".align\t" #order "\n\t" \
137 "1:\n\t" \
138 )
139#define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
140#define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
141
142static inline void blast_r4600_v1_icache32(void)
143{
144 unsigned long flags;
145
146 local_irq_save(flags);
147 blast_icache32();
148 local_irq_restore(flags);
149}
150
151static inline void tx49_blast_icache32(void)
152{
153 unsigned long start = INDEX_BASE;
154 unsigned long end = start + current_cpu_data.icache.waysize;
155 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
156 unsigned long ws_end = current_cpu_data.icache.ways <<
157 current_cpu_data.icache.waybit;
158 unsigned long ws, addr;
159
160 CACHE32_UNROLL32_ALIGN2;
161 /* I'm in even chunk. blast odd chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700162 for (ws = 0; ws < ws_end; ws += ws_inc)
163 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 cache32_unroll32(addr|ws,Index_Invalidate_I);
165 CACHE32_UNROLL32_ALIGN;
166 /* I'm in odd chunk. blast even chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700167 for (ws = 0; ws < ws_end; ws += ws_inc)
168 for (addr = start; addr < end; addr += 0x400 * 2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 cache32_unroll32(addr|ws,Index_Invalidate_I);
170}
171
172static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
173{
174 unsigned long flags;
175
176 local_irq_save(flags);
177 blast_icache32_page_indexed(page);
178 local_irq_restore(flags);
179}
180
181static inline void tx49_blast_icache32_page_indexed(unsigned long page)
182{
Atsushi Nemoto67a3f6de2006-04-04 17:34:14 +0900183 unsigned long indexmask = current_cpu_data.icache.waysize - 1;
184 unsigned long start = INDEX_BASE + (page & indexmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 unsigned long end = start + PAGE_SIZE;
186 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
187 unsigned long ws_end = current_cpu_data.icache.ways <<
188 current_cpu_data.icache.waybit;
189 unsigned long ws, addr;
190
191 CACHE32_UNROLL32_ALIGN2;
192 /* I'm in even chunk. blast odd chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700193 for (ws = 0; ws < ws_end; ws += ws_inc)
194 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 cache32_unroll32(addr|ws,Index_Invalidate_I);
196 CACHE32_UNROLL32_ALIGN;
197 /* I'm in odd chunk. blast even chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700198 for (ws = 0; ws < ws_end; ws += ws_inc)
199 for (addr = start; addr < end; addr += 0x400 * 2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 cache32_unroll32(addr|ws,Index_Invalidate_I);
201}
202
203static void (* r4k_blast_icache_page)(unsigned long addr);
204
Ralf Baechlea00f6312006-08-01 23:39:42 +0100205static void __init r4k_blast_icache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206{
207 unsigned long ic_lsize = cpu_icache_line_size();
208
Chris Dearman73f40352006-06-20 18:06:52 +0100209 if (ic_lsize == 0)
210 r4k_blast_icache_page = (void *)cache_noop;
211 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 r4k_blast_icache_page = blast_icache16_page;
213 else if (ic_lsize == 32)
214 r4k_blast_icache_page = blast_icache32_page;
215 else if (ic_lsize == 64)
216 r4k_blast_icache_page = blast_icache64_page;
217}
218
219
220static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
221
Ralf Baechlea00f6312006-08-01 23:39:42 +0100222static void __init r4k_blast_icache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223{
224 unsigned long ic_lsize = cpu_icache_line_size();
225
Chris Dearman73f40352006-06-20 18:06:52 +0100226 if (ic_lsize == 0)
227 r4k_blast_icache_page_indexed = (void *)cache_noop;
228 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
230 else if (ic_lsize == 32) {
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000231 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 r4k_blast_icache_page_indexed =
233 blast_icache32_r4600_v1_page_indexed;
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000234 else if (TX49XX_ICACHE_INDEX_INV_WAR)
235 r4k_blast_icache_page_indexed =
236 tx49_blast_icache32_page_indexed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237 else
238 r4k_blast_icache_page_indexed =
239 blast_icache32_page_indexed;
240 } else if (ic_lsize == 64)
241 r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
242}
243
244static void (* r4k_blast_icache)(void);
245
Ralf Baechlea00f6312006-08-01 23:39:42 +0100246static void __init r4k_blast_icache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247{
248 unsigned long ic_lsize = cpu_icache_line_size();
249
Chris Dearman73f40352006-06-20 18:06:52 +0100250 if (ic_lsize == 0)
251 r4k_blast_icache = (void *)cache_noop;
252 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253 r4k_blast_icache = blast_icache16;
254 else if (ic_lsize == 32) {
255 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
256 r4k_blast_icache = blast_r4600_v1_icache32;
257 else if (TX49XX_ICACHE_INDEX_INV_WAR)
258 r4k_blast_icache = tx49_blast_icache32;
259 else
260 r4k_blast_icache = blast_icache32;
261 } else if (ic_lsize == 64)
262 r4k_blast_icache = blast_icache64;
263}
264
265static void (* r4k_blast_scache_page)(unsigned long addr);
266
Ralf Baechlea00f6312006-08-01 23:39:42 +0100267static void __init r4k_blast_scache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268{
269 unsigned long sc_lsize = cpu_scache_line_size();
270
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000271 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100272 r4k_blast_scache_page = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000273 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 r4k_blast_scache_page = blast_scache16_page;
275 else if (sc_lsize == 32)
276 r4k_blast_scache_page = blast_scache32_page;
277 else if (sc_lsize == 64)
278 r4k_blast_scache_page = blast_scache64_page;
279 else if (sc_lsize == 128)
280 r4k_blast_scache_page = blast_scache128_page;
281}
282
283static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
284
Ralf Baechlea00f6312006-08-01 23:39:42 +0100285static void __init r4k_blast_scache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286{
287 unsigned long sc_lsize = cpu_scache_line_size();
288
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000289 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100290 r4k_blast_scache_page_indexed = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000291 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
293 else if (sc_lsize == 32)
294 r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
295 else if (sc_lsize == 64)
296 r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
297 else if (sc_lsize == 128)
298 r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
299}
300
301static void (* r4k_blast_scache)(void);
302
Ralf Baechlea00f6312006-08-01 23:39:42 +0100303static void __init r4k_blast_scache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304{
305 unsigned long sc_lsize = cpu_scache_line_size();
306
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000307 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100308 r4k_blast_scache = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000309 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 r4k_blast_scache = blast_scache16;
311 else if (sc_lsize == 32)
312 r4k_blast_scache = blast_scache32;
313 else if (sc_lsize == 64)
314 r4k_blast_scache = blast_scache64;
315 else if (sc_lsize == 128)
316 r4k_blast_scache = blast_scache128;
317}
318
319/*
320 * This is former mm's flush_cache_all() which really should be
321 * flush_cache_vunmap these days ...
322 */
323static inline void local_r4k_flush_cache_all(void * args)
324{
325 r4k_blast_dcache();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326}
327
328static void r4k_flush_cache_all(void)
329{
330 if (!cpu_has_dc_aliases)
331 return;
332
Ralf Baechle7f3f1d02006-05-12 13:20:06 +0100333 r4k_on_each_cpu(local_r4k_flush_cache_all, NULL, 1, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334}
335
336static inline void local_r4k___flush_cache_all(void * args)
337{
338 r4k_blast_dcache();
339 r4k_blast_icache();
340
341 switch (current_cpu_data.cputype) {
342 case CPU_R4000SC:
343 case CPU_R4000MC:
344 case CPU_R4400SC:
345 case CPU_R4400MC:
346 case CPU_R10000:
347 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -0400348 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 r4k_blast_scache();
350 }
351}
352
353static void r4k___flush_cache_all(void)
354{
Ralf Baechle7f3f1d02006-05-12 13:20:06 +0100355 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356}
357
358static inline void local_r4k_flush_cache_range(void * args)
359{
360 struct vm_area_struct *vma = args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361
362 if (!(cpu_context(smp_processor_id(), vma->vm_mm)))
363 return;
364
Atsushi Nemoto0550d9d2006-08-22 21:15:47 +0900365 r4k_blast_dcache();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366}
367
368static void r4k_flush_cache_range(struct vm_area_struct *vma,
369 unsigned long start, unsigned long end)
370{
Atsushi Nemoto0550d9d2006-08-22 21:15:47 +0900371 if (!cpu_has_dc_aliases)
372 return;
373
Ralf Baechle7f3f1d02006-05-12 13:20:06 +0100374 r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375}
376
377static inline void local_r4k_flush_cache_mm(void * args)
378{
379 struct mm_struct *mm = args;
380
381 if (!cpu_context(smp_processor_id(), mm))
382 return;
383
384 r4k_blast_dcache();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385
386 /*
387 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we
388 * only flush the primary caches but R10000 and R12000 behave sane ...
389 */
390 if (current_cpu_data.cputype == CPU_R4000SC ||
391 current_cpu_data.cputype == CPU_R4000MC ||
392 current_cpu_data.cputype == CPU_R4400SC ||
393 current_cpu_data.cputype == CPU_R4400MC)
394 r4k_blast_scache();
395}
396
397static void r4k_flush_cache_mm(struct mm_struct *mm)
398{
399 if (!cpu_has_dc_aliases)
400 return;
401
Ralf Baechle7f3f1d02006-05-12 13:20:06 +0100402 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403}
404
405struct flush_cache_page_args {
406 struct vm_area_struct *vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100407 unsigned long addr;
Atsushi Nemotode628932006-03-13 18:23:03 +0900408 unsigned long pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409};
410
411static inline void local_r4k_flush_cache_page(void *args)
412{
413 struct flush_cache_page_args *fcp_args = args;
414 struct vm_area_struct *vma = fcp_args->vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100415 unsigned long addr = fcp_args->addr;
Atsushi Nemotode628932006-03-13 18:23:03 +0900416 unsigned long paddr = fcp_args->pfn << PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 int exec = vma->vm_flags & VM_EXEC;
418 struct mm_struct *mm = vma->vm_mm;
419 pgd_t *pgdp;
Ralf Baechlec6e8b582005-02-10 12:19:59 +0000420 pud_t *pudp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 pmd_t *pmdp;
422 pte_t *ptep;
423
Ralf Baechle79acf832005-02-10 13:54:37 +0000424 /*
425 * If ownes no valid ASID yet, cannot possibly have gotten
426 * this page into the cache.
427 */
Thiemo Seufer26a51b22005-02-19 13:32:02 +0000428 if (cpu_context(smp_processor_id(), mm) == 0)
Ralf Baechle79acf832005-02-10 13:54:37 +0000429 return;
430
Ralf Baechle6ec25802005-10-12 00:02:34 +0100431 addr &= PAGE_MASK;
432 pgdp = pgd_offset(mm, addr);
433 pudp = pud_offset(pgdp, addr);
434 pmdp = pmd_offset(pudp, addr);
435 ptep = pte_offset(pmdp, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436
437 /*
438 * If the page isn't marked valid, the page cannot possibly be
439 * in the cache.
440 */
441 if (!(pte_val(*ptep) & _PAGE_PRESENT))
442 return;
443
444 /*
445 * Doing flushes for another ASID than the current one is
446 * too difficult since stupid R4k caches do a TLB translation
447 * for every cache flush operation. So we do indexed flushes
448 * in that case, which doesn't overly flush the cache too much.
449 */
450 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
451 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
Ralf Baechle6ec25802005-10-12 00:02:34 +0100452 r4k_blast_dcache_page(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 if (exec && !cpu_icache_snoops_remote_store)
Ralf Baechle6ec25802005-10-12 00:02:34 +0100454 r4k_blast_scache_page(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455 }
456 if (exec)
Ralf Baechle6ec25802005-10-12 00:02:34 +0100457 r4k_blast_icache_page(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458
459 return;
460 }
461
462 /*
463 * Do indexed flush, too much work to get the (possible) TLB refills
464 * to work correctly.
465 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
Atsushi Nemotode628932006-03-13 18:23:03 +0900467 r4k_blast_dcache_page_indexed(cpu_has_pindexed_dcache ?
468 paddr : addr);
469 if (exec && !cpu_icache_snoops_remote_store) {
470 r4k_blast_scache_page_indexed(paddr);
471 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 }
473 if (exec) {
Atsushi Nemotof6502792006-08-25 17:55:31 +0900474 if (cpu_has_vtag_icache && mm == current->active_mm) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 int cpu = smp_processor_id();
476
Thiemo Seufer26a51b22005-02-19 13:32:02 +0000477 if (cpu_context(cpu, mm) != 0)
478 drop_mmu_context(mm, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479 } else
Ralf Baechle6ec25802005-10-12 00:02:34 +0100480 r4k_blast_icache_page_indexed(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481 }
482}
483
Ralf Baechle6ec25802005-10-12 00:02:34 +0100484static void r4k_flush_cache_page(struct vm_area_struct *vma,
485 unsigned long addr, unsigned long pfn)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486{
487 struct flush_cache_page_args args;
488
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489 args.vma = vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100490 args.addr = addr;
Atsushi Nemotode628932006-03-13 18:23:03 +0900491 args.pfn = pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492
Ralf Baechle7f3f1d02006-05-12 13:20:06 +0100493 r4k_on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494}
495
496static inline void local_r4k_flush_data_cache_page(void * addr)
497{
498 r4k_blast_dcache_page((unsigned long) addr);
499}
500
501static void r4k_flush_data_cache_page(unsigned long addr)
502{
Ralf Baechle7f3f1d02006-05-12 13:20:06 +0100503 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 1, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504}
505
506struct flush_icache_range_args {
Atsushi Nemotod4264f12006-01-29 02:27:51 +0900507 unsigned long start;
508 unsigned long end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509};
510
511static inline void local_r4k_flush_icache_range(void *args)
512{
513 struct flush_icache_range_args *fir_args = args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 unsigned long start = fir_args->start;
515 unsigned long end = fir_args->end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516
517 if (!cpu_has_ic_fills_f_dc) {
Chris Dearman73f40352006-06-20 18:06:52 +0100518 if (end - start >= dcache_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 r4k_blast_dcache();
520 } else {
Thiemo Seufer10a3dab2005-09-09 20:26:54 +0000521 R4600_HIT_CACHEOP_WAR_IMPL;
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900522 protected_blast_dcache_range(start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 }
524
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000525 if (!cpu_icache_snoops_remote_store && scache_size) {
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900526 if (end - start > scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 r4k_blast_scache();
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900528 else
529 protected_blast_scache_range(start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 }
531 }
532
533 if (end - start > icache_size)
534 r4k_blast_icache();
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900535 else
536 protected_blast_icache_range(start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537}
538
Atsushi Nemotod4264f12006-01-29 02:27:51 +0900539static void r4k_flush_icache_range(unsigned long start, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540{
541 struct flush_icache_range_args args;
542
543 args.start = start;
544 args.end = end;
545
Ralf Baechle7f3f1d02006-05-12 13:20:06 +0100546 r4k_on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1);
Ralf Baechlecc61c1f2005-07-12 18:35:38 +0000547 instruction_hazard();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548}
549
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550#ifdef CONFIG_DMA_NONCOHERENT
551
552static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
553{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 /* Catch bad driver code */
555 BUG_ON(size == 0);
556
Ralf Baechlefc5d2d22006-07-06 13:04:01 +0100557 if (cpu_has_inclusive_pcaches) {
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900558 if (size >= scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 r4k_blast_scache();
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900560 else
561 blast_scache_range(addr, addr + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 return;
563 }
564
565 /*
566 * Either no secondary cache or the available caches don't have the
567 * subset property so we have to flush the primary caches
568 * explicitly
569 */
570 if (size >= dcache_size) {
571 r4k_blast_dcache();
572 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573 R4600_HIT_CACHEOP_WAR_IMPL;
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900574 blast_dcache_range(addr, addr + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 }
576
577 bc_wback_inv(addr, size);
578}
579
580static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
581{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 /* Catch bad driver code */
583 BUG_ON(size == 0);
584
Ralf Baechlefc5d2d22006-07-06 13:04:01 +0100585 if (cpu_has_inclusive_pcaches) {
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900586 if (size >= scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 r4k_blast_scache();
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900588 else
589 blast_scache_range(addr, addr + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 return;
591 }
592
593 if (size >= dcache_size) {
594 r4k_blast_dcache();
595 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 R4600_HIT_CACHEOP_WAR_IMPL;
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900597 blast_dcache_range(addr, addr + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 }
599
600 bc_inv(addr, size);
601}
602#endif /* CONFIG_DMA_NONCOHERENT */
603
604/*
605 * While we're protected against bad userland addresses we don't care
606 * very much about what happens in that case. Usually a segmentation
607 * fault will dump the process later on anyway ...
608 */
609static void local_r4k_flush_cache_sigtramp(void * arg)
610{
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000611 unsigned long ic_lsize = cpu_icache_line_size();
612 unsigned long dc_lsize = cpu_dcache_line_size();
613 unsigned long sc_lsize = cpu_scache_line_size();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614 unsigned long addr = (unsigned long) arg;
615
616 R4600_HIT_CACHEOP_WAR_IMPL;
Chris Dearman73f40352006-06-20 18:06:52 +0100617 if (dc_lsize)
618 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000619 if (!cpu_icache_snoops_remote_store && scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 protected_writeback_scache_line(addr & ~(sc_lsize - 1));
Chris Dearman73f40352006-06-20 18:06:52 +0100621 if (ic_lsize)
622 protected_flush_icache_line(addr & ~(ic_lsize - 1));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 if (MIPS4K_ICACHE_REFILL_WAR) {
624 __asm__ __volatile__ (
625 ".set push\n\t"
626 ".set noat\n\t"
627 ".set mips3\n\t"
Ralf Baechle875d43e2005-09-03 15:56:16 -0700628#ifdef CONFIG_32BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 "la $at,1f\n\t"
630#endif
Ralf Baechle875d43e2005-09-03 15:56:16 -0700631#ifdef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 "dla $at,1f\n\t"
633#endif
634 "cache %0,($at)\n\t"
635 "nop; nop; nop\n"
636 "1:\n\t"
637 ".set pop"
638 :
639 : "i" (Hit_Invalidate_I));
640 }
641 if (MIPS_CACHE_SYNC_WAR)
642 __asm__ __volatile__ ("sync");
643}
644
645static void r4k_flush_cache_sigtramp(unsigned long addr)
646{
Ralf Baechle7f3f1d02006-05-12 13:20:06 +0100647 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648}
649
650static void r4k_flush_icache_all(void)
651{
652 if (cpu_has_vtag_icache)
653 r4k_blast_icache();
654}
655
656static inline void rm7k_erratum31(void)
657{
658 const unsigned long ic_lsize = 32;
659 unsigned long addr;
660
661 /* RM7000 erratum #31. The icache is screwed at startup. */
662 write_c0_taglo(0);
663 write_c0_taghi(0);
664
665 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
666 __asm__ __volatile__ (
Thiemo Seuferd8748a32005-09-02 09:56:12 +0000667 ".set push\n\t"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 ".set noreorder\n\t"
669 ".set mips3\n\t"
670 "cache\t%1, 0(%0)\n\t"
671 "cache\t%1, 0x1000(%0)\n\t"
672 "cache\t%1, 0x2000(%0)\n\t"
673 "cache\t%1, 0x3000(%0)\n\t"
674 "cache\t%2, 0(%0)\n\t"
675 "cache\t%2, 0x1000(%0)\n\t"
676 "cache\t%2, 0x2000(%0)\n\t"
677 "cache\t%2, 0x3000(%0)\n\t"
678 "cache\t%1, 0(%0)\n\t"
679 "cache\t%1, 0x1000(%0)\n\t"
680 "cache\t%1, 0x2000(%0)\n\t"
681 "cache\t%1, 0x3000(%0)\n\t"
Thiemo Seuferd8748a32005-09-02 09:56:12 +0000682 ".set pop\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 :
684 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
685 }
686}
687
688static char *way_string[] __initdata = { NULL, "direct mapped", "2-way",
689 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
690};
691
692static void __init probe_pcache(void)
693{
694 struct cpuinfo_mips *c = &current_cpu_data;
695 unsigned int config = read_c0_config();
696 unsigned int prid = read_c0_prid();
697 unsigned long config1;
698 unsigned int lsize;
699
700 switch (c->cputype) {
701 case CPU_R4600: /* QED style two way caches? */
702 case CPU_R4700:
703 case CPU_R5000:
704 case CPU_NEVADA:
705 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
706 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
707 c->icache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900708 c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709
710 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
711 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
712 c->dcache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900713 c->dcache.waybit= __ffs(dcache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714
715 c->options |= MIPS_CPU_CACHE_CDEX_P;
716 break;
717
718 case CPU_R5432:
719 case CPU_R5500:
720 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
721 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
722 c->icache.ways = 2;
723 c->icache.waybit= 0;
724
725 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
726 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
727 c->dcache.ways = 2;
728 c->dcache.waybit = 0;
729
730 c->options |= MIPS_CPU_CACHE_CDEX_P;
731 break;
732
733 case CPU_TX49XX:
734 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
735 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
736 c->icache.ways = 4;
737 c->icache.waybit= 0;
738
739 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
740 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
741 c->dcache.ways = 4;
742 c->dcache.waybit = 0;
743
744 c->options |= MIPS_CPU_CACHE_CDEX_P;
Atsushi Nemotode862b42006-03-17 12:59:22 +0900745 c->options |= MIPS_CPU_PREFETCH;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 break;
747
748 case CPU_R4000PC:
749 case CPU_R4000SC:
750 case CPU_R4000MC:
751 case CPU_R4400PC:
752 case CPU_R4400SC:
753 case CPU_R4400MC:
754 case CPU_R4300:
755 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
756 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
757 c->icache.ways = 1;
758 c->icache.waybit = 0; /* doesn't matter */
759
760 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
761 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
762 c->dcache.ways = 1;
763 c->dcache.waybit = 0; /* does not matter */
764
765 c->options |= MIPS_CPU_CACHE_CDEX_P;
766 break;
767
768 case CPU_R10000:
769 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -0400770 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
772 c->icache.linesz = 64;
773 c->icache.ways = 2;
774 c->icache.waybit = 0;
775
776 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
777 c->dcache.linesz = 32;
778 c->dcache.ways = 2;
779 c->dcache.waybit = 0;
780
781 c->options |= MIPS_CPU_PREFETCH;
782 break;
783
784 case CPU_VR4133:
Yoichi Yuasa2874fe52006-07-08 00:42:12 +0900785 write_c0_config(config & ~VR41_CONF_P4K);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 case CPU_VR4131:
787 /* Workaround for cache instruction bug of VR4131 */
788 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
789 c->processor_id == 0x0c82U) {
Yoichi Yuasa4e8ab362006-07-04 22:59:41 +0900790 config |= 0x00400000U;
791 if (c->processor_id == 0x0c80U)
792 config |= VR41_CONF_BP;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793 write_c0_config(config);
Yoichi Yuasa1058ecd2006-07-08 00:42:01 +0900794 } else
795 c->options |= MIPS_CPU_CACHE_CDEX_P;
796
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
798 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
799 c->icache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900800 c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801
802 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
803 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
804 c->dcache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900805 c->dcache.waybit = __ffs(dcache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806 break;
807
808 case CPU_VR41XX:
809 case CPU_VR4111:
810 case CPU_VR4121:
811 case CPU_VR4122:
812 case CPU_VR4181:
813 case CPU_VR4181A:
814 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
815 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
816 c->icache.ways = 1;
817 c->icache.waybit = 0; /* doesn't matter */
818
819 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
820 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
821 c->dcache.ways = 1;
822 c->dcache.waybit = 0; /* does not matter */
823
824 c->options |= MIPS_CPU_CACHE_CDEX_P;
825 break;
826
827 case CPU_RM7000:
828 rm7k_erratum31();
829
830 case CPU_RM9000:
831 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
832 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
833 c->icache.ways = 4;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900834 c->icache.waybit = __ffs(icache_size / c->icache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835
836 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
837 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
838 c->dcache.ways = 4;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900839 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840
841#if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR)
842 c->options |= MIPS_CPU_CACHE_CDEX_P;
843#endif
844 c->options |= MIPS_CPU_PREFETCH;
845 break;
846
847 default:
848 if (!(config & MIPS_CONF_M))
849 panic("Don't know how to probe P-caches on this cpu.");
850
851 /*
852 * So we seem to be a MIPS32 or MIPS64 CPU
853 * So let's probe the I-cache ...
854 */
855 config1 = read_c0_config1();
856
857 if ((lsize = ((config1 >> 19) & 7)))
858 c->icache.linesz = 2 << lsize;
859 else
860 c->icache.linesz = lsize;
861 c->icache.sets = 64 << ((config1 >> 22) & 7);
862 c->icache.ways = 1 + ((config1 >> 16) & 7);
863
864 icache_size = c->icache.sets *
865 c->icache.ways *
866 c->icache.linesz;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900867 c->icache.waybit = __ffs(icache_size/c->icache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868
869 if (config & 0x8) /* VI bit */
870 c->icache.flags |= MIPS_CACHE_VTAG;
871
872 /*
873 * Now probe the MIPS32 / MIPS64 data cache.
874 */
875 c->dcache.flags = 0;
876
877 if ((lsize = ((config1 >> 10) & 7)))
878 c->dcache.linesz = 2 << lsize;
879 else
880 c->dcache.linesz= lsize;
881 c->dcache.sets = 64 << ((config1 >> 13) & 7);
882 c->dcache.ways = 1 + ((config1 >> 7) & 7);
883
884 dcache_size = c->dcache.sets *
885 c->dcache.ways *
886 c->dcache.linesz;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900887 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888
889 c->options |= MIPS_CPU_PREFETCH;
890 break;
891 }
892
893 /*
894 * Processor configuration sanity check for the R4000SC erratum
895 * #5. With page sizes larger than 32kB there is no possibility
896 * to get a VCE exception anymore so we don't care about this
897 * misconfiguration. The case is rather theoretical anyway;
898 * presumably no vendor is shipping his hardware in the "bad"
899 * configuration.
900 */
901 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 &&
902 !(config & CONF_SC) && c->icache.linesz != 16 &&
903 PAGE_SIZE <= 0x8000)
904 panic("Improper R4000SC processor configuration detected");
905
906 /* compute a couple of other cache variables */
907 c->icache.waysize = icache_size / c->icache.ways;
908 c->dcache.waysize = dcache_size / c->dcache.ways;
909
Chris Dearman73f40352006-06-20 18:06:52 +0100910 c->icache.sets = c->icache.linesz ?
911 icache_size / (c->icache.linesz * c->icache.ways) : 0;
912 c->dcache.sets = c->dcache.linesz ?
913 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914
915 /*
916 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB
917 * 2-way virtually indexed so normally would suffer from aliases. So
918 * normally they'd suffer from aliases but magic in the hardware deals
919 * with that for us so we don't need to take care ourselves.
920 */
Ralf Baechled1e344e2005-02-04 15:51:26 +0000921 switch (c->cputype) {
Ralf Baechlea95970f2005-02-07 21:41:32 +0000922 case CPU_20KC:
Ralf Baechle505403b2005-02-07 21:53:39 +0000923 case CPU_25KF:
Atsushi Nemotode628932006-03-13 18:23:03 +0900924 c->dcache.flags |= MIPS_CACHE_PINDEX;
Ralf Baechled1e344e2005-02-04 15:51:26 +0000925 case CPU_R10000:
926 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -0400927 case CPU_R14000:
Ralf Baechlea95970f2005-02-07 21:41:32 +0000928 case CPU_SB1:
Ralf Baechled1e344e2005-02-04 15:51:26 +0000929 break;
930 case CPU_24K:
Nigel Stephens98a41de2006-04-27 15:50:32 +0100931 case CPU_34K:
Ralf Baechle2e78ae32006-06-23 18:48:21 +0100932 case CPU_74K:
Ralf Baechlebeab3752006-06-19 21:56:25 +0100933 if ((read_c0_config7() & (1 << 16))) {
934 /* effectively physically indexed dcache,
935 thus no virtual aliases. */
936 c->dcache.flags |= MIPS_CACHE_PINDEX;
937 break;
938 }
Ralf Baechled1e344e2005-02-04 15:51:26 +0000939 default:
Ralf Baechlebeab3752006-06-19 21:56:25 +0100940 if (c->dcache.waysize > PAGE_SIZE)
941 c->dcache.flags |= MIPS_CACHE_ALIASES;
Ralf Baechled1e344e2005-02-04 15:51:26 +0000942 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943
944 switch (c->cputype) {
945 case CPU_20KC:
946 /*
947 * Some older 20Kc chips doesn't have the 'VI' bit in
948 * the config register.
949 */
950 c->icache.flags |= MIPS_CACHE_VTAG;
951 break;
952
Pete Popove3ad1c22005-03-01 06:33:16 +0000953 case CPU_AU1000:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 case CPU_AU1500:
Pete Popove3ad1c22005-03-01 06:33:16 +0000955 case CPU_AU1100:
956 case CPU_AU1550:
957 case CPU_AU1200:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 c->icache.flags |= MIPS_CACHE_IC_F_DC;
959 break;
960 }
961
962 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
963 icache_size >> 10,
964 cpu_has_vtag_icache ? "virtually tagged" : "physically tagged",
965 way_string[c->icache.ways], c->icache.linesz);
966
967 printk("Primary data cache %ldkB, %s, linesize %d bytes.\n",
968 dcache_size >> 10, way_string[c->dcache.ways], c->dcache.linesz);
969}
970
971/*
972 * If you even _breathe_ on this function, look at the gcc output and make sure
973 * it does not pop things on and off the stack for the cache sizing loop that
974 * executes in KSEG1 space or else you will crash and burn badly. You have
975 * been warned.
976 */
977static int __init probe_scache(void)
978{
979 extern unsigned long stext;
980 unsigned long flags, addr, begin, end, pow2;
981 unsigned int config = read_c0_config();
982 struct cpuinfo_mips *c = &current_cpu_data;
983 int tmp;
984
985 if (config & CONF_SC)
986 return 0;
987
988 begin = (unsigned long) &stext;
989 begin &= ~((4 * 1024 * 1024) - 1);
990 end = begin + (4 * 1024 * 1024);
991
992 /*
993 * This is such a bitch, you'd think they would make it easy to do
994 * this. Away you daemons of stupidity!
995 */
996 local_irq_save(flags);
997
998 /* Fill each size-multiple cache line with a valid tag. */
999 pow2 = (64 * 1024);
1000 for (addr = begin; addr < end; addr = (begin + pow2)) {
1001 unsigned long *p = (unsigned long *) addr;
1002 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
1003 pow2 <<= 1;
1004 }
1005
1006 /* Load first line with zero (therefore invalid) tag. */
1007 write_c0_taglo(0);
1008 write_c0_taghi(0);
1009 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
1010 cache_op(Index_Store_Tag_I, begin);
1011 cache_op(Index_Store_Tag_D, begin);
1012 cache_op(Index_Store_Tag_SD, begin);
1013
1014 /* Now search for the wrap around point. */
1015 pow2 = (128 * 1024);
1016 tmp = 0;
1017 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1018 cache_op(Index_Load_Tag_SD, addr);
1019 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
1020 if (!read_c0_taglo())
1021 break;
1022 pow2 <<= 1;
1023 }
1024 local_irq_restore(flags);
1025 addr -= begin;
1026
1027 scache_size = addr;
1028 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1029 c->scache.ways = 1;
1030 c->dcache.waybit = 0; /* does not matter */
1031
1032 return 1;
1033}
1034
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035extern int r5k_sc_init(void);
1036extern int rm7k_sc_init(void);
Chris Dearman9318c512006-06-20 17:15:20 +01001037extern int mips_sc_init(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038
1039static void __init setup_scache(void)
1040{
1041 struct cpuinfo_mips *c = &current_cpu_data;
1042 unsigned int config = read_c0_config();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043 int sc_present = 0;
1044
1045 /*
1046 * Do the probing thing on R4000SC and R4400SC processors. Other
1047 * processors don't have a S-cache that would be relevant to the
1048 * Linux memory managment.
1049 */
1050 switch (c->cputype) {
1051 case CPU_R4000SC:
1052 case CPU_R4000MC:
1053 case CPU_R4400SC:
1054 case CPU_R4400MC:
Thiemo Seuferba5187d2005-04-25 16:36:23 +00001055 sc_present = run_uncached(probe_scache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056 if (sc_present)
1057 c->options |= MIPS_CPU_CACHE_CDEX_S;
1058 break;
1059
1060 case CPU_R10000:
1061 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -04001062 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1064 c->scache.linesz = 64 << ((config >> 13) & 1);
1065 c->scache.ways = 2;
1066 c->scache.waybit= 0;
1067 sc_present = 1;
1068 break;
1069
1070 case CPU_R5000:
1071 case CPU_NEVADA:
1072#ifdef CONFIG_R5000_CPU_SCACHE
1073 r5k_sc_init();
1074#endif
1075 return;
1076
1077 case CPU_RM7000:
1078 case CPU_RM9000:
1079#ifdef CONFIG_RM7000_CPU_SCACHE
1080 rm7k_sc_init();
1081#endif
1082 return;
1083
1084 default:
Chris Dearman9318c512006-06-20 17:15:20 +01001085 if (c->isa_level == MIPS_CPU_ISA_M32R1 ||
1086 c->isa_level == MIPS_CPU_ISA_M32R2 ||
1087 c->isa_level == MIPS_CPU_ISA_M64R1 ||
1088 c->isa_level == MIPS_CPU_ISA_M64R2) {
1089#ifdef CONFIG_MIPS_CPU_SCACHE
1090 if (mips_sc_init ()) {
1091 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz;
1092 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n",
1093 scache_size >> 10,
1094 way_string[c->scache.ways], c->scache.linesz);
1095 }
1096#else
1097 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1098 panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1099#endif
1100 return;
1101 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102 sc_present = 0;
1103 }
1104
1105 if (!sc_present)
1106 return;
1107
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 /* compute a couple of other cache variables */
1109 c->scache.waysize = scache_size / c->scache.ways;
1110
1111 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1112
1113 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1114 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1115
Ralf Baechlefc5d2d22006-07-06 13:04:01 +01001116 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117}
1118
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001119void au1x00_fixup_config_od(void)
1120{
1121 /*
1122 * c0_config.od (bit 19) was write only (and read as 0)
1123 * on the early revisions of Alchemy SOCs. It disables the bus
1124 * transaction overlapping and needs to be set to fix various errata.
1125 */
1126 switch (read_c0_prid()) {
1127 case 0x00030100: /* Au1000 DA */
1128 case 0x00030201: /* Au1000 HA */
1129 case 0x00030202: /* Au1000 HB */
1130 case 0x01030200: /* Au1500 AB */
1131 /*
1132 * Au1100 errata actually keeps silence about this bit, so we set it
1133 * just in case for those revisions that require it to be set according
1134 * to arch/mips/au1000/common/cputable.c
1135 */
1136 case 0x02030200: /* Au1100 AB */
1137 case 0x02030201: /* Au1100 BA */
1138 case 0x02030202: /* Au1100 BC */
1139 set_c0_config(1 << 19);
1140 break;
1141 }
1142}
1143
Ralf Baechlea00f6312006-08-01 23:39:42 +01001144static void __init coherency_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145{
1146 change_c0_config(CONF_CM_CMASK, CONF_CM_DEFAULT);
1147
1148 /*
1149 * c0_status.cu=0 specifies that updates by the sc instruction use
1150 * the coherency mode specified by the TLB; 1 means cachable
1151 * coherent update on write will be used. Not all processors have
1152 * this bit and; some wire it to zero, others like Toshiba had the
1153 * silly idea of putting something else there ...
1154 */
1155 switch (current_cpu_data.cputype) {
1156 case CPU_R4000PC:
1157 case CPU_R4000SC:
1158 case CPU_R4000MC:
1159 case CPU_R4400PC:
1160 case CPU_R4400SC:
1161 case CPU_R4400MC:
1162 clear_c0_config(CONF_CU);
1163 break;
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001164 /*
Ralf Baechledf586d52006-08-01 23:42:30 +01001165 * We need to catch the early Alchemy SOCs with
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001166 * the write-only co_config.od bit and set it back to one...
1167 */
1168 case CPU_AU1000: /* rev. DA, HA, HB */
1169 case CPU_AU1100: /* rev. AB, BA, BC ?? */
1170 case CPU_AU1500: /* rev. AB */
1171 au1x00_fixup_config_od();
1172 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173 }
1174}
1175
Ralf Baechle02cf2112005-10-01 13:06:32 +01001176void __init r4k_cache_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177{
1178 extern void build_clear_page(void);
1179 extern void build_copy_page(void);
1180 extern char except_vec2_generic;
1181 struct cpuinfo_mips *c = &current_cpu_data;
1182
1183 /* Default cache error handler for R4000 and R5000 family */
Ralf Baechlee01402b2005-07-14 15:57:16 +00001184 set_uncached_handler (0x100, &except_vec2_generic, 0x80);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001185
1186 probe_pcache();
1187 setup_scache();
1188
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189 r4k_blast_dcache_page_setup();
1190 r4k_blast_dcache_page_indexed_setup();
1191 r4k_blast_dcache_setup();
1192 r4k_blast_icache_page_setup();
1193 r4k_blast_icache_page_indexed_setup();
1194 r4k_blast_icache_setup();
1195 r4k_blast_scache_page_setup();
1196 r4k_blast_scache_page_indexed_setup();
1197 r4k_blast_scache_setup();
1198
1199 /*
1200 * Some MIPS32 and MIPS64 processors have physically indexed caches.
1201 * This code supports virtually indexed processors and will be
1202 * unnecessarily inefficient on physically indexed processors.
1203 */
Chris Dearman73f40352006-06-20 18:06:52 +01001204 if (c->dcache.linesz)
1205 shm_align_mask = max_t( unsigned long,
1206 c->dcache.sets * c->dcache.linesz - 1,
1207 PAGE_SIZE - 1);
1208 else
1209 shm_align_mask = PAGE_SIZE-1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210 flush_cache_all = r4k_flush_cache_all;
1211 __flush_cache_all = r4k___flush_cache_all;
1212 flush_cache_mm = r4k_flush_cache_mm;
1213 flush_cache_page = r4k_flush_cache_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214 flush_cache_range = r4k_flush_cache_range;
1215
1216 flush_cache_sigtramp = r4k_flush_cache_sigtramp;
1217 flush_icache_all = r4k_flush_icache_all;
Ralf Baechle7e3bfc72006-04-05 20:42:04 +01001218 local_flush_data_cache_page = local_r4k_flush_data_cache_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219 flush_data_cache_page = r4k_flush_data_cache_page;
1220 flush_icache_range = r4k_flush_icache_range;
1221
1222#ifdef CONFIG_DMA_NONCOHERENT
1223 _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
1224 _dma_cache_wback = r4k_dma_cache_wback_inv;
1225 _dma_cache_inv = r4k_dma_cache_inv;
1226#endif
1227
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 build_clear_page();
1229 build_copy_page();
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00001230 local_r4k___flush_cache_all(NULL);
1231 coherency_setup();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232}