blob: 24c1d3c30186c893c400eff0942b857207a833fa [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * Re-map IO memory to kernel address space so that we can access it.
3 * This is needed for high PCI addresses that aren't mapped in the
4 * 640k-1MB IO memory area on PC's
5 *
6 * (C) Copyright 1995 1996 Linus Torvalds
7 */
8
Thomas Gleixnere9332ca2008-01-30 13:34:05 +01009#include <linux/bootmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/init.h>
Haavard Skinnemoena148ecf2006-09-30 23:29:17 -070011#include <linux/io.h>
Thomas Gleixner3cbd09e2008-01-30 13:34:05 +010012#include <linux/module.h>
13#include <linux/slab.h>
14#include <linux/vmalloc.h>
Pekka Paalanend61fc442008-05-12 21:20:57 +020015#include <linux/mmiotrace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016
Thomas Gleixner3cbd09e2008-01-30 13:34:05 +010017#include <asm/cacheflush.h>
18#include <asm/e820.h>
19#include <asm/fixmap.h>
20#include <asm/pgtable.h>
21#include <asm/tlbflush.h>
Jeremy Fitzhardingef6df72e2008-01-30 13:34:11 +010022#include <asm/pgalloc.h>
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -070023#include <asm/pat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Thomas Gleixner240d3a72008-01-30 13:34:05 +010025#ifdef CONFIG_X86_64
26
27unsigned long __phys_addr(unsigned long x)
28{
29 if (x >= __START_KERNEL_map)
30 return x - __START_KERNEL_map + phys_base;
31 return x - PAGE_OFFSET;
32}
33EXPORT_SYMBOL(__phys_addr);
34
Thomas Gleixnere3100c82008-02-27 20:57:40 +010035static inline int phys_addr_valid(unsigned long addr)
36{
37 return addr < (1UL << boot_cpu_data.x86_phys_bits);
38}
39
40#else
41
42static inline int phys_addr_valid(unsigned long addr)
43{
44 return 1;
45}
46
Thomas Gleixner240d3a72008-01-30 13:34:05 +010047#endif
48
Thomas Gleixner5f5192b2008-01-30 13:34:06 +010049int page_is_ram(unsigned long pagenr)
50{
Ingo Molnar756a6c62008-03-25 08:31:17 +010051 resource_size_t addr, end;
Thomas Gleixner5f5192b2008-01-30 13:34:06 +010052 int i;
53
Arjan van de Vend8a9e6a2008-02-18 09:54:33 -080054 /*
55 * A special case is the first 4Kb of memory;
56 * This is a BIOS owned area, not kernel ram, but generally
57 * not listed as such in the E820 table.
58 */
59 if (pagenr == 0)
60 return 0;
61
Arjan van de Ven156fbc32008-02-18 09:58:45 -080062 /*
63 * Second special case: Some BIOSen report the PC BIOS
64 * area (640->1Mb) as ram even though it is not.
65 */
66 if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
67 pagenr < (BIOS_END >> PAGE_SHIFT))
68 return 0;
Arjan van de Vend8a9e6a2008-02-18 09:54:33 -080069
Thomas Gleixner5f5192b2008-01-30 13:34:06 +010070 for (i = 0; i < e820.nr_map; i++) {
71 /*
72 * Not usable memory:
73 */
74 if (e820.map[i].type != E820_RAM)
75 continue;
Thomas Gleixner5f5192b2008-01-30 13:34:06 +010076 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
77 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
Thomas Gleixner950f9d92008-01-30 13:34:06 +010078
Thomas Gleixner950f9d92008-01-30 13:34:06 +010079
Thomas Gleixner5f5192b2008-01-30 13:34:06 +010080 if ((pagenr >= addr) && (pagenr < end))
81 return 1;
82 }
83 return 0;
84}
85
Linus Torvalds1da177e2005-04-16 15:20:36 -070086/*
Thomas Gleixnere9332ca2008-01-30 13:34:05 +010087 * Fix up the linear direct mapping of the kernel to avoid cache attribute
88 * conflicts.
89 */
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -070090int ioremap_change_attr(unsigned long vaddr, unsigned long size,
91 unsigned long prot_val)
Thomas Gleixnere9332ca2008-01-30 13:34:05 +010092{
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +010093 unsigned long nrpages = size >> PAGE_SHIFT;
Harvey Harrison93809be2008-02-01 17:49:43 +010094 int err;
Thomas Gleixnere9332ca2008-01-30 13:34:05 +010095
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -070096 switch (prot_val) {
97 case _PAGE_CACHE_UC:
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +010098 default:
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -070099 err = _set_memory_uc(vaddr, nrpages);
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +0100100 break;
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700101 case _PAGE_CACHE_WC:
102 err = _set_memory_wc(vaddr, nrpages);
103 break;
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -0700104 case _PAGE_CACHE_WB:
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -0700105 err = _set_memory_wb(vaddr, nrpages);
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +0100106 break;
107 }
Thomas Gleixnere9332ca2008-01-30 13:34:05 +0100108
109 return err;
110}
111
112/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 * Remap an arbitrary physical address space into the kernel virtual
114 * address space. Needed when the kernel wants to access high addresses
115 * directly.
116 *
117 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
118 * have to convert them into an offset in a page-aligned mapping, but the
119 * caller shouldn't need to know that small detail.
120 */
Christoph Lameter23016962008-04-28 02:12:42 -0700121static void __iomem *__ioremap_caller(resource_size_t phys_addr,
122 unsigned long size, unsigned long prot_val, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123{
Ingo Molnar756a6c62008-03-25 08:31:17 +0100124 unsigned long pfn, offset, vaddr;
125 resource_size_t last_addr;
Pekka Paalanen87e547f2008-05-12 21:21:03 +0200126 const resource_size_t unaligned_phys_addr = phys_addr;
127 const unsigned long unaligned_size = size;
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100128 struct vm_struct *area;
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700129 unsigned long new_prot_val;
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +0100130 pgprot_t prot;
Venki Pallipadidee7cbb2008-03-24 14:39:55 -0700131 int retval;
Pekka Paalanend61fc442008-05-12 21:20:57 +0200132 void __iomem *ret_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
134 /* Don't allow wraparound or zero size */
135 last_addr = phys_addr + size - 1;
136 if (!size || last_addr < phys_addr)
137 return NULL;
138
Thomas Gleixnere3100c82008-02-27 20:57:40 +0100139 if (!phys_addr_valid(phys_addr)) {
venkatesh.pallipadi@intel.com6997ab42008-03-18 17:00:25 -0700140 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
Randy Dunlap4c8337a2008-04-10 15:09:50 -0700141 (unsigned long long)phys_addr);
Thomas Gleixnere3100c82008-02-27 20:57:40 +0100142 WARN_ON_ONCE(1);
143 return NULL;
144 }
145
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 /*
147 * Don't remap the low PCI/ISA area, it's always mapped..
148 */
Andreas Herrmannbcc643d2008-06-20 21:58:46 +0200149 if (is_ISA_range(phys_addr, last_addr))
Thomas Gleixner4b40fce2008-01-30 13:34:05 +0100150 return (__force void __iomem *)phys_to_virt(phys_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151
152 /*
153 * Don't allow anybody to remap normal RAM that we're using..
154 */
Andres Salomoncb8ab682008-04-30 11:30:24 -0400155 for (pfn = phys_addr >> PAGE_SHIFT;
156 (pfn << PAGE_SHIFT) < (last_addr & PAGE_MASK);
157 pfn++) {
Ingo Molnarbdd3cee2008-02-28 14:10:49 +0100158
Ingo Molnarba748d22008-03-03 09:37:41 +0100159 int is_ram = page_is_ram(pfn);
160
161 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
Thomas Gleixner266b9f82008-01-30 13:34:06 +0100162 return NULL;
Ingo Molnarba748d22008-03-03 09:37:41 +0100163 WARN_ON_ONCE(is_ram);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 }
165
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700166 /*
167 * Mappings have to be page-aligned
168 */
169 offset = phys_addr & ~PAGE_MASK;
170 phys_addr &= PAGE_MASK;
171 size = PAGE_ALIGN(last_addr+1) - phys_addr;
172
Venki Pallipadidee7cbb2008-03-24 14:39:55 -0700173 retval = reserve_memtype(phys_addr, phys_addr + size,
174 prot_val, &new_prot_val);
175 if (retval) {
Venki Pallipadib450e5e2008-03-25 16:51:26 -0700176 pr_debug("Warning: reserve_memtype returned %d\n", retval);
Venki Pallipadidee7cbb2008-03-24 14:39:55 -0700177 return NULL;
178 }
179
180 if (prot_val != new_prot_val) {
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700181 /*
182 * Do not fallback to certain memory types with certain
183 * requested type:
Suresh Siddhade33c442008-04-25 17:07:22 -0700184 * - request is uc-, return cannot be write-back
185 * - request is uc-, return cannot be write-combine
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700186 * - request is write-combine, return cannot be write-back
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700187 */
Suresh Siddhade33c442008-04-25 17:07:22 -0700188 if ((prot_val == _PAGE_CACHE_UC_MINUS &&
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700189 (new_prot_val == _PAGE_CACHE_WB ||
190 new_prot_val == _PAGE_CACHE_WC)) ||
191 (prot_val == _PAGE_CACHE_WC &&
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700192 new_prot_val == _PAGE_CACHE_WB)) {
Venki Pallipadib450e5e2008-03-25 16:51:26 -0700193 pr_debug(
venkatesh.pallipadi@intel.com6997ab42008-03-18 17:00:25 -0700194 "ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n",
Randy Dunlap4c8337a2008-04-10 15:09:50 -0700195 (unsigned long long)phys_addr,
196 (unsigned long long)(phys_addr + size),
venkatesh.pallipadi@intel.com6997ab42008-03-18 17:00:25 -0700197 prot_val, new_prot_val);
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700198 free_memtype(phys_addr, phys_addr + size);
199 return NULL;
200 }
201 prot_val = new_prot_val;
202 }
203
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -0700204 switch (prot_val) {
205 case _PAGE_CACHE_UC:
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +0100206 default:
Ingo Molnar55c62682008-03-26 06:19:45 +0100207 prot = PAGE_KERNEL_NOCACHE;
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +0100208 break;
Suresh Siddhade33c442008-04-25 17:07:22 -0700209 case _PAGE_CACHE_UC_MINUS:
210 prot = PAGE_KERNEL_UC_MINUS;
211 break;
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700212 case _PAGE_CACHE_WC:
213 prot = PAGE_KERNEL_WC;
214 break;
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -0700215 case _PAGE_CACHE_WB:
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +0100216 prot = PAGE_KERNEL;
217 break;
218 }
Haavard Skinnemoena148ecf2006-09-30 23:29:17 -0700219
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 * Ok, go for it..
222 */
Christoph Lameter23016962008-04-28 02:12:42 -0700223 area = get_vm_area_caller(size, VM_IOREMAP, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 if (!area)
225 return NULL;
226 area->phys_addr = phys_addr;
Thomas Gleixnere66aadb2008-02-04 16:48:05 +0100227 vaddr = (unsigned long) area->addr;
228 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) {
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700229 free_memtype(phys_addr, phys_addr + size);
Ingo Molnarb16bf712008-02-28 14:02:08 +0100230 free_vm_area(area);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 return NULL;
232 }
Thomas Gleixnere9332ca2008-01-30 13:34:05 +0100233
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -0700234 if (ioremap_change_attr(vaddr, size, prot_val) < 0) {
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700235 free_memtype(phys_addr, phys_addr + size);
Thomas Gleixnere66aadb2008-02-04 16:48:05 +0100236 vunmap(area->addr);
Thomas Gleixnere9332ca2008-01-30 13:34:05 +0100237 return NULL;
238 }
239
Pekka Paalanend61fc442008-05-12 21:20:57 +0200240 ret_addr = (void __iomem *) (vaddr + offset);
Pekka Paalanen87e547f2008-05-12 21:21:03 +0200241 mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
Pekka Paalanend61fc442008-05-12 21:20:57 +0200242
243 return ret_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245
246/**
247 * ioremap_nocache - map bus memory into CPU space
248 * @offset: bus address of the memory
249 * @size: size of the resource to map
250 *
251 * ioremap_nocache performs a platform specific sequence of operations to
252 * make bus memory CPU accessible via the readb/readw/readl/writeb/
253 * writew/writel functions and the other mmio helpers. The returned
254 * address is not guaranteed to be usable directly as a virtual
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100255 * address.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 *
257 * This version of ioremap ensures that the memory is marked uncachable
258 * on the CPU as well as honouring existing caching rules from things like
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100259 * the PCI bus. Note that there are other caches and buffers on many
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 * busses. In particular driver authors should read up on PCI writes
261 *
262 * It's useful if some control registers are in such an area and
263 * write combining or read caching is not desirable:
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100264 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265 * Must be freed with iounmap.
266 */
Linus Torvaldsb9e76a02008-03-24 11:22:39 -0700267void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268{
Suresh Siddhade33c442008-04-25 17:07:22 -0700269 /*
270 * Ideally, this should be:
Andreas Herrmann499f8f82008-06-10 16:06:21 +0200271 * pat_enabled ? _PAGE_CACHE_UC : _PAGE_CACHE_UC_MINUS;
Suresh Siddhade33c442008-04-25 17:07:22 -0700272 *
273 * Till we fix all X drivers to use ioremap_wc(), we will use
274 * UC MINUS.
275 */
276 unsigned long val = _PAGE_CACHE_UC_MINUS;
277
278 return __ioremap_caller(phys_addr, size, val,
Christoph Lameter23016962008-04-28 02:12:42 -0700279 __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280}
Alexey Dobriyan129f6942005-06-23 00:08:33 -0700281EXPORT_SYMBOL(ioremap_nocache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700283/**
284 * ioremap_wc - map memory into CPU space write combined
285 * @offset: bus address of the memory
286 * @size: size of the resource to map
287 *
288 * This version of ioremap ensures that the memory is marked write combining.
289 * Write combining allows faster writes to some hardware devices.
290 *
291 * Must be freed with iounmap.
292 */
293void __iomem *ioremap_wc(unsigned long phys_addr, unsigned long size)
294{
Andreas Herrmann499f8f82008-06-10 16:06:21 +0200295 if (pat_enabled)
Christoph Lameter23016962008-04-28 02:12:42 -0700296 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WC,
297 __builtin_return_address(0));
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700298 else
299 return ioremap_nocache(phys_addr, size);
300}
301EXPORT_SYMBOL(ioremap_wc);
302
Linus Torvaldsb9e76a02008-03-24 11:22:39 -0700303void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
Thomas Gleixner5f868152008-01-30 13:34:06 +0100304{
Christoph Lameter23016962008-04-28 02:12:42 -0700305 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WB,
306 __builtin_return_address(0));
Thomas Gleixner5f868152008-01-30 13:34:06 +0100307}
308EXPORT_SYMBOL(ioremap_cache);
309
Venkatesh Pallipadia361ee52008-07-10 10:09:59 +0200310static void __iomem *ioremap_default(resource_size_t phys_addr,
311 unsigned long size)
312{
313 unsigned long flags;
314 void *ret;
315 int err;
316
317 /*
318 * - WB for WB-able memory and no other conflicting mappings
319 * - UC_MINUS for non-WB-able memory with no other conflicting mappings
320 * - Inherit from confliting mappings otherwise
321 */
322 err = reserve_memtype(phys_addr, phys_addr + size, -1, &flags);
323 if (err < 0)
324 return NULL;
325
326 ret = (void *) __ioremap_caller(phys_addr, size, flags,
327 __builtin_return_address(0));
328
329 free_memtype(phys_addr, phys_addr + size);
330 return (void __iomem *)ret;
331}
332
Andi Kleenbf5421c2005-12-12 22:17:09 -0800333/**
334 * iounmap - Free a IO remapping
335 * @addr: virtual address from ioremap_*
336 *
337 * Caller must ensure there is only one unmapping for the same pointer.
338 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339void iounmap(volatile void __iomem *addr)
340{
Andi Kleenbf5421c2005-12-12 22:17:09 -0800341 struct vm_struct *p, *o;
Andrew Mortonc23a4e962005-07-07 17:56:02 -0700342
343 if ((void __force *)addr <= high_memory)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 return;
345
346 /*
347 * __ioremap special-cases the PCI/ISA range by not instantiating a
348 * vm_area and by simply returning an address into the kernel mapping
349 * of ISA space. So handle that here.
350 */
Thomas Gleixner6e92a5a2008-05-12 15:43:35 +0200351 if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
352 (void __force *)addr < phys_to_virt(ISA_END_ADDRESS))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 return;
354
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100355 addr = (volatile void __iomem *)
356 (PAGE_MASK & (unsigned long __force)addr);
Andi Kleenbf5421c2005-12-12 22:17:09 -0800357
Pekka Paalanend61fc442008-05-12 21:20:57 +0200358 mmiotrace_iounmap(addr);
359
Andi Kleenbf5421c2005-12-12 22:17:09 -0800360 /* Use the vm area unlocked, assuming the caller
361 ensures there isn't another iounmap for the same address
362 in parallel. Reuse of the virtual address is prevented by
363 leaving it in the global lists until we're done with it.
364 cpa takes care of the direct mappings. */
365 read_lock(&vmlist_lock);
366 for (p = vmlist; p; p = p->next) {
Thomas Gleixner6e92a5a2008-05-12 15:43:35 +0200367 if (p->addr == (void __force *)addr)
Andi Kleenbf5421c2005-12-12 22:17:09 -0800368 break;
369 }
370 read_unlock(&vmlist_lock);
371
372 if (!p) {
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100373 printk(KERN_ERR "iounmap: bad address %p\n", addr);
Andrew Mortonc23a4e962005-07-07 17:56:02 -0700374 dump_stack();
Andi Kleenbf5421c2005-12-12 22:17:09 -0800375 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 }
377
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700378 free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
379
Andi Kleenbf5421c2005-12-12 22:17:09 -0800380 /* Finally remove it */
Thomas Gleixner6e92a5a2008-05-12 15:43:35 +0200381 o = remove_vm_area((void __force *)addr);
Andi Kleenbf5421c2005-12-12 22:17:09 -0800382 BUG_ON(p != o || o == NULL);
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100383 kfree(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384}
Alexey Dobriyan129f6942005-06-23 00:08:33 -0700385EXPORT_SYMBOL(iounmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386
venkatesh.pallipadi@intel.come045fb22008-03-18 17:00:15 -0700387/*
388 * Convert a physical pointer to a virtual kernel pointer for /dev/mem
389 * access
390 */
391void *xlate_dev_mem_ptr(unsigned long phys)
392{
393 void *addr;
394 unsigned long start = phys & PAGE_MASK;
395
396 /* If page is RAM, we can use __va. Otherwise ioremap and unmap. */
397 if (page_is_ram(start >> PAGE_SHIFT))
398 return __va(phys);
399
Ingo Molnarae94b802008-07-12 07:29:02 +0200400 addr = (void __force *)ioremap_default(start, PAGE_SIZE);
venkatesh.pallipadi@intel.come045fb22008-03-18 17:00:15 -0700401 if (addr)
402 addr = (void *)((unsigned long)addr | (phys & ~PAGE_MASK));
403
404 return addr;
405}
406
407void unxlate_dev_mem_ptr(unsigned long phys, void *addr)
408{
409 if (page_is_ram(phys >> PAGE_SHIFT))
410 return;
411
412 iounmap((void __iomem *)((unsigned long)addr & PAGE_MASK));
413 return;
414}
415
Ingo Molnard18d6d62008-01-30 13:33:45 +0100416int __initdata early_ioremap_debug;
417
418static int __init early_ioremap_debug_setup(char *str)
419{
420 early_ioremap_debug = 1;
421
Huang, Ying793b24a2008-01-30 13:33:45 +0100422 return 0;
Ingo Molnard18d6d62008-01-30 13:33:45 +0100423}
Huang, Ying793b24a2008-01-30 13:33:45 +0100424early_param("early_ioremap_debug", early_ioremap_debug_setup);
Ingo Molnard18d6d62008-01-30 13:33:45 +0100425
Huang, Ying0947b2f2008-01-30 13:33:44 +0100426static __initdata int after_paging_init;
Jeremy Fitzhardingea7bf0bd2008-05-28 15:02:14 +0100427static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100428
Ian Campbell551889a62008-02-09 23:24:09 +0100429static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100430{
Jeremy Fitzhardinge37cc8d72008-02-13 16:20:35 +0100431 /* Don't assume we're using swapper_pg_dir at this point */
432 pgd_t *base = __va(read_cr3());
433 pgd_t *pgd = &base[pgd_index(addr)];
Ian Campbell551889a62008-02-09 23:24:09 +0100434 pud_t *pud = pud_offset(pgd, addr);
435 pmd_t *pmd = pmd_offset(pud, addr);
436
437 return pmd;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100438}
439
Ian Campbell551889a62008-02-09 23:24:09 +0100440static inline pte_t * __init early_ioremap_pte(unsigned long addr)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100441{
Ian Campbell551889a62008-02-09 23:24:09 +0100442 return &bm_pte[pte_index(addr)];
Huang, Ying0947b2f2008-01-30 13:33:44 +0100443}
444
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100445void __init early_ioremap_init(void)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100446{
Ian Campbell551889a62008-02-09 23:24:09 +0100447 pmd_t *pmd;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100448
Ingo Molnard18d6d62008-01-30 13:33:45 +0100449 if (early_ioremap_debug)
Ingo Molnaradafdf62008-01-30 13:34:08 +0100450 printk(KERN_INFO "early_ioremap_init()\n");
Ingo Molnard18d6d62008-01-30 13:33:45 +0100451
Ian Campbell551889a62008-02-09 23:24:09 +0100452 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
Huang, Ying0947b2f2008-01-30 13:33:44 +0100453 memset(bm_pte, 0, sizeof(bm_pte));
Ian Campbellb6fbb662008-02-09 23:24:09 +0100454 pmd_populate_kernel(&init_mm, pmd, bm_pte);
Ian Campbell551889a62008-02-09 23:24:09 +0100455
Ingo Molnar0e3a9542008-01-30 13:33:49 +0100456 /*
Ian Campbell551889a62008-02-09 23:24:09 +0100457 * The boot-ioremap range spans multiple pmds, for which
Ingo Molnar0e3a9542008-01-30 13:33:49 +0100458 * we are not prepared:
459 */
Ian Campbell551889a62008-02-09 23:24:09 +0100460 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
Ingo Molnar0e3a9542008-01-30 13:33:49 +0100461 WARN_ON(1);
Ian Campbell551889a62008-02-09 23:24:09 +0100462 printk(KERN_WARNING "pmd %p != %p\n",
463 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100464 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
Ian Campbell551889a62008-02-09 23:24:09 +0100465 fix_to_virt(FIX_BTMAP_BEGIN));
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100466 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
Ian Campbell551889a62008-02-09 23:24:09 +0100467 fix_to_virt(FIX_BTMAP_END));
Ingo Molnar0e3a9542008-01-30 13:33:49 +0100468
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100469 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
470 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
471 FIX_BTMAP_BEGIN);
Ingo Molnar0e3a9542008-01-30 13:33:49 +0100472 }
Huang, Ying0947b2f2008-01-30 13:33:44 +0100473}
474
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100475void __init early_ioremap_clear(void)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100476{
Ian Campbell551889a62008-02-09 23:24:09 +0100477 pmd_t *pmd;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100478
Ingo Molnard18d6d62008-01-30 13:33:45 +0100479 if (early_ioremap_debug)
Ingo Molnaradafdf62008-01-30 13:34:08 +0100480 printk(KERN_INFO "early_ioremap_clear()\n");
Ingo Molnard18d6d62008-01-30 13:33:45 +0100481
Ian Campbell551889a62008-02-09 23:24:09 +0100482 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
483 pmd_clear(pmd);
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700484 paravirt_release_pte(__pa(bm_pte) >> PAGE_SHIFT);
Huang, Ying0947b2f2008-01-30 13:33:44 +0100485 __flush_tlb_all();
486}
487
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100488void __init early_ioremap_reset(void)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100489{
490 enum fixed_addresses idx;
Ian Campbell551889a62008-02-09 23:24:09 +0100491 unsigned long addr, phys;
492 pte_t *pte;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100493
494 after_paging_init = 1;
Huang, Ying64a8f852008-01-30 13:33:44 +0100495 for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
Huang, Ying0947b2f2008-01-30 13:33:44 +0100496 addr = fix_to_virt(idx);
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100497 pte = early_ioremap_pte(addr);
Ian Campbell551889a62008-02-09 23:24:09 +0100498 if (pte_present(*pte)) {
499 phys = pte_val(*pte) & PAGE_MASK;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100500 set_fixmap(idx, phys);
501 }
502 }
503}
504
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100505static void __init __early_set_fixmap(enum fixed_addresses idx,
Huang, Ying0947b2f2008-01-30 13:33:44 +0100506 unsigned long phys, pgprot_t flags)
507{
Ian Campbell551889a62008-02-09 23:24:09 +0100508 unsigned long addr = __fix_to_virt(idx);
509 pte_t *pte;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100510
511 if (idx >= __end_of_fixed_addresses) {
512 BUG();
513 return;
514 }
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100515 pte = early_ioremap_pte(addr);
Jeremy Fitzhardinge4583ed52008-06-25 00:19:03 -0400516
Huang, Ying0947b2f2008-01-30 13:33:44 +0100517 if (pgprot_val(flags))
Ian Campbell551889a62008-02-09 23:24:09 +0100518 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
Huang, Ying0947b2f2008-01-30 13:33:44 +0100519 else
Jeremy Fitzhardinge4f9c11d2008-06-25 00:19:19 -0400520 pte_clear(&init_mm, addr, pte);
Huang, Ying0947b2f2008-01-30 13:33:44 +0100521 __flush_tlb_one(addr);
522}
523
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100524static inline void __init early_set_fixmap(enum fixed_addresses idx,
Huang, Ying0947b2f2008-01-30 13:33:44 +0100525 unsigned long phys)
526{
527 if (after_paging_init)
528 set_fixmap(idx, phys);
529 else
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100530 __early_set_fixmap(idx, phys, PAGE_KERNEL);
Huang, Ying0947b2f2008-01-30 13:33:44 +0100531}
532
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100533static inline void __init early_clear_fixmap(enum fixed_addresses idx)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100534{
535 if (after_paging_init)
536 clear_fixmap(idx);
537 else
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100538 __early_set_fixmap(idx, 0, __pgprot(0));
Huang, Ying0947b2f2008-01-30 13:33:44 +0100539}
540
Ingo Molnar1b42f512008-01-30 13:33:45 +0100541
542int __initdata early_ioremap_nested;
543
Ingo Molnard690b2a2008-01-30 13:33:47 +0100544static int __init check_early_ioremap_leak(void)
545{
546 if (!early_ioremap_nested)
547 return 0;
548
549 printk(KERN_WARNING
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100550 "Debug warning: early ioremap leak of %d areas detected.\n",
551 early_ioremap_nested);
Ingo Molnard690b2a2008-01-30 13:33:47 +0100552 printk(KERN_WARNING
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100553 "please boot with early_ioremap_debug and report the dmesg.\n");
Ingo Molnard690b2a2008-01-30 13:33:47 +0100554 WARN_ON(1);
555
556 return 1;
557}
558late_initcall(check_early_ioremap_leak);
559
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100560void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561{
562 unsigned long offset, last_addr;
Ingo Molnar1b42f512008-01-30 13:33:45 +0100563 unsigned int nrpages, nesting;
564 enum fixed_addresses idx0, idx;
565
566 WARN_ON(system_state != SYSTEM_BOOTING);
567
568 nesting = early_ioremap_nested;
Ingo Molnard18d6d62008-01-30 13:33:45 +0100569 if (early_ioremap_debug) {
Ingo Molnaradafdf62008-01-30 13:34:08 +0100570 printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100571 phys_addr, size, nesting);
Ingo Molnard18d6d62008-01-30 13:33:45 +0100572 dump_stack();
573 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574
575 /* Don't allow wraparound or zero size */
576 last_addr = phys_addr + size - 1;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100577 if (!size || last_addr < phys_addr) {
578 WARN_ON(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 return NULL;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100580 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100582 if (nesting >= FIX_BTMAPS_NESTING) {
583 WARN_ON(1);
Ingo Molnar1b42f512008-01-30 13:33:45 +0100584 return NULL;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100585 }
Ingo Molnar1b42f512008-01-30 13:33:45 +0100586 early_ioremap_nested++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 /*
588 * Mappings have to be page-aligned
589 */
590 offset = phys_addr & ~PAGE_MASK;
591 phys_addr &= PAGE_MASK;
592 size = PAGE_ALIGN(last_addr) - phys_addr;
593
594 /*
595 * Mappings have to fit in the FIX_BTMAP area.
596 */
597 nrpages = size >> PAGE_SHIFT;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100598 if (nrpages > NR_FIX_BTMAPS) {
599 WARN_ON(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 return NULL;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100601 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602
603 /*
604 * Ok, go for it..
605 */
Ingo Molnar1b42f512008-01-30 13:33:45 +0100606 idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
607 idx = idx0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 while (nrpages > 0) {
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100609 early_set_fixmap(idx, phys_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 phys_addr += PAGE_SIZE;
611 --idx;
612 --nrpages;
613 }
Ingo Molnard18d6d62008-01-30 13:33:45 +0100614 if (early_ioremap_debug)
615 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
Ingo Molnar1b42f512008-01-30 13:33:45 +0100616
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100617 return (void *) (offset + fix_to_virt(idx0));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618}
619
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100620void __init early_iounmap(void *addr, unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621{
622 unsigned long virt_addr;
623 unsigned long offset;
624 unsigned int nrpages;
625 enum fixed_addresses idx;
Ingo Molnar226e9a92008-05-27 09:56:49 +0200626 int nesting;
Ingo Molnar1b42f512008-01-30 13:33:45 +0100627
628 nesting = --early_ioremap_nested;
Ingo Molnar226e9a92008-05-27 09:56:49 +0200629 if (WARN_ON(nesting < 0))
630 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631
Ingo Molnard18d6d62008-01-30 13:33:45 +0100632 if (early_ioremap_debug) {
Ingo Molnaradafdf62008-01-30 13:34:08 +0100633 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100634 size, nesting);
Ingo Molnard18d6d62008-01-30 13:33:45 +0100635 dump_stack();
636 }
637
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 virt_addr = (unsigned long)addr;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100639 if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
640 WARN_ON(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641 return;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100642 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 offset = virt_addr & ~PAGE_MASK;
644 nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
645
Ingo Molnar1b42f512008-01-30 13:33:45 +0100646 idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 while (nrpages > 0) {
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100648 early_clear_fixmap(idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649 --idx;
650 --nrpages;
651 }
652}
Ingo Molnar1b42f512008-01-30 13:33:45 +0100653
654void __this_fixmap_does_not_exist(void)
655{
656 WARN_ON(1);
657}