Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* |
| 3 | * This file contains KASAN runtime code that manages shadow memory for |
| 4 | * generic and software tag-based KASAN modes. |
| 5 | * |
| 6 | * Copyright (c) 2014 Samsung Electronics Co., Ltd. |
| 7 | * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> |
| 8 | * |
| 9 | * Some code borrowed from https://github.com/xairy/kasan-prototype by |
| 10 | * Andrey Konovalov <andreyknvl@gmail.com> |
| 11 | */ |
| 12 | |
| 13 | #include <linux/init.h> |
| 14 | #include <linux/kasan.h> |
| 15 | #include <linux/kernel.h> |
Alexander Potapenko | 2b83052 | 2021-02-25 17:19:21 -0800 | [diff] [blame] | 16 | #include <linux/kfence.h> |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 17 | #include <linux/kmemleak.h> |
| 18 | #include <linux/memory.h> |
| 19 | #include <linux/mm.h> |
| 20 | #include <linux/string.h> |
| 21 | #include <linux/types.h> |
| 22 | #include <linux/vmalloc.h> |
| 23 | |
| 24 | #include <asm/cacheflush.h> |
| 25 | #include <asm/tlbflush.h> |
| 26 | |
| 27 | #include "kasan.h" |
| 28 | |
| 29 | bool __kasan_check_read(const volatile void *p, unsigned int size) |
| 30 | { |
Andrey Konovalov | f00748b | 2021-02-24 12:05:05 -0800 | [diff] [blame] | 31 | return kasan_check_range((unsigned long)p, size, false, _RET_IP_); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 32 | } |
| 33 | EXPORT_SYMBOL(__kasan_check_read); |
| 34 | |
| 35 | bool __kasan_check_write(const volatile void *p, unsigned int size) |
| 36 | { |
Andrey Konovalov | f00748b | 2021-02-24 12:05:05 -0800 | [diff] [blame] | 37 | return kasan_check_range((unsigned long)p, size, true, _RET_IP_); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 38 | } |
| 39 | EXPORT_SYMBOL(__kasan_check_write); |
| 40 | |
| 41 | #undef memset |
| 42 | void *memset(void *addr, int c, size_t len) |
| 43 | { |
Andrey Konovalov | f00748b | 2021-02-24 12:05:05 -0800 | [diff] [blame] | 44 | if (!kasan_check_range((unsigned long)addr, len, true, _RET_IP_)) |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 45 | return NULL; |
| 46 | |
| 47 | return __memset(addr, c, len); |
| 48 | } |
| 49 | |
| 50 | #ifdef __HAVE_ARCH_MEMMOVE |
| 51 | #undef memmove |
| 52 | void *memmove(void *dest, const void *src, size_t len) |
| 53 | { |
Andrey Konovalov | f00748b | 2021-02-24 12:05:05 -0800 | [diff] [blame] | 54 | if (!kasan_check_range((unsigned long)src, len, false, _RET_IP_) || |
| 55 | !kasan_check_range((unsigned long)dest, len, true, _RET_IP_)) |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 56 | return NULL; |
| 57 | |
| 58 | return __memmove(dest, src, len); |
| 59 | } |
| 60 | #endif |
| 61 | |
| 62 | #undef memcpy |
| 63 | void *memcpy(void *dest, const void *src, size_t len) |
| 64 | { |
Andrey Konovalov | f00748b | 2021-02-24 12:05:05 -0800 | [diff] [blame] | 65 | if (!kasan_check_range((unsigned long)src, len, false, _RET_IP_) || |
| 66 | !kasan_check_range((unsigned long)dest, len, true, _RET_IP_)) |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 67 | return NULL; |
| 68 | |
| 69 | return __memcpy(dest, src, len); |
| 70 | } |
| 71 | |
Andrey Konovalov | aa5c219 | 2021-04-29 22:59:59 -0700 | [diff] [blame] | 72 | void kasan_poison(const void *addr, size_t size, u8 value, bool init) |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 73 | { |
| 74 | void *shadow_start, *shadow_end; |
| 75 | |
Daniel Axtens | af3751f3 | 2021-06-28 19:40:42 -0700 | [diff] [blame] | 76 | if (!kasan_arch_is_ready()) |
| 77 | return; |
| 78 | |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 79 | /* |
| 80 | * Perform shadow offset calculation based on untagged address, as |
| 81 | * some of the callers (e.g. kasan_poison_object_data) pass tagged |
| 82 | * addresses to this function. |
| 83 | */ |
Andrey Konovalov | cde8a7e | 2021-02-25 17:20:27 -0800 | [diff] [blame] | 84 | addr = kasan_reset_tag(addr); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 85 | |
Alexander Potapenko | 2b83052 | 2021-02-25 17:19:21 -0800 | [diff] [blame] | 86 | /* Skip KFENCE memory if called explicitly outside of sl*b. */ |
Andrey Konovalov | cde8a7e | 2021-02-25 17:20:27 -0800 | [diff] [blame] | 87 | if (is_kfence_address(addr)) |
Alexander Potapenko | 2b83052 | 2021-02-25 17:19:21 -0800 | [diff] [blame] | 88 | return; |
| 89 | |
Andrey Konovalov | cde8a7e | 2021-02-25 17:20:27 -0800 | [diff] [blame] | 90 | if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK)) |
| 91 | return; |
| 92 | if (WARN_ON(size & KASAN_GRANULE_MASK)) |
| 93 | return; |
| 94 | |
| 95 | shadow_start = kasan_mem_to_shadow(addr); |
| 96 | shadow_end = kasan_mem_to_shadow(addr + size); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 97 | |
| 98 | __memset(shadow_start, value, shadow_end - shadow_start); |
| 99 | } |
Andrey Konovalov | 573a480 | 2021-02-24 12:05:21 -0800 | [diff] [blame] | 100 | EXPORT_SYMBOL(kasan_poison); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 101 | |
Andrey Konovalov | e2db1a9 | 2021-02-25 17:19:59 -0800 | [diff] [blame] | 102 | #ifdef CONFIG_KASAN_GENERIC |
Andrey Konovalov | cde8a7e | 2021-02-25 17:20:27 -0800 | [diff] [blame] | 103 | void kasan_poison_last_granule(const void *addr, size_t size) |
Andrey Konovalov | e2db1a9 | 2021-02-25 17:19:59 -0800 | [diff] [blame] | 104 | { |
Daniel Axtens | af3751f3 | 2021-06-28 19:40:42 -0700 | [diff] [blame] | 105 | if (!kasan_arch_is_ready()) |
| 106 | return; |
| 107 | |
Andrey Konovalov | e2db1a9 | 2021-02-25 17:19:59 -0800 | [diff] [blame] | 108 | if (size & KASAN_GRANULE_MASK) { |
Andrey Konovalov | cde8a7e | 2021-02-25 17:20:27 -0800 | [diff] [blame] | 109 | u8 *shadow = (u8 *)kasan_mem_to_shadow(addr + size); |
Andrey Konovalov | e2db1a9 | 2021-02-25 17:19:59 -0800 | [diff] [blame] | 110 | *shadow = size & KASAN_GRANULE_MASK; |
| 111 | } |
| 112 | } |
| 113 | #endif |
| 114 | |
Andrey Konovalov | aa5c219 | 2021-04-29 22:59:59 -0700 | [diff] [blame] | 115 | void kasan_unpoison(const void *addr, size_t size, bool init) |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 116 | { |
Andrey Konovalov | cde8a7e | 2021-02-25 17:20:27 -0800 | [diff] [blame] | 117 | u8 tag = get_tag(addr); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 118 | |
| 119 | /* |
| 120 | * Perform shadow offset calculation based on untagged address, as |
| 121 | * some of the callers (e.g. kasan_unpoison_object_data) pass tagged |
| 122 | * addresses to this function. |
| 123 | */ |
Andrey Konovalov | cde8a7e | 2021-02-25 17:20:27 -0800 | [diff] [blame] | 124 | addr = kasan_reset_tag(addr); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 125 | |
Alexander Potapenko | 2b83052 | 2021-02-25 17:19:21 -0800 | [diff] [blame] | 126 | /* |
| 127 | * Skip KFENCE memory if called explicitly outside of sl*b. Also note |
| 128 | * that calls to ksize(), where size is not a multiple of machine-word |
| 129 | * size, would otherwise poison the invalid portion of the word. |
| 130 | */ |
Andrey Konovalov | cde8a7e | 2021-02-25 17:20:27 -0800 | [diff] [blame] | 131 | if (is_kfence_address(addr)) |
Alexander Potapenko | 2b83052 | 2021-02-25 17:19:21 -0800 | [diff] [blame] | 132 | return; |
| 133 | |
Andrey Konovalov | cde8a7e | 2021-02-25 17:20:27 -0800 | [diff] [blame] | 134 | if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK)) |
| 135 | return; |
| 136 | |
| 137 | /* Unpoison all granules that cover the object. */ |
Andrey Konovalov | aa5c219 | 2021-04-29 22:59:59 -0700 | [diff] [blame] | 138 | kasan_poison(addr, round_up(size, KASAN_GRANULE_SIZE), tag, false); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 139 | |
Andrey Konovalov | e2db1a9 | 2021-02-25 17:19:59 -0800 | [diff] [blame] | 140 | /* Partially poison the last granule for the generic mode. */ |
| 141 | if (IS_ENABLED(CONFIG_KASAN_GENERIC)) |
Andrey Konovalov | cde8a7e | 2021-02-25 17:20:27 -0800 | [diff] [blame] | 142 | kasan_poison_last_granule(addr, size); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 143 | } |
| 144 | |
| 145 | #ifdef CONFIG_MEMORY_HOTPLUG |
| 146 | static bool shadow_mapped(unsigned long addr) |
| 147 | { |
| 148 | pgd_t *pgd = pgd_offset_k(addr); |
| 149 | p4d_t *p4d; |
| 150 | pud_t *pud; |
| 151 | pmd_t *pmd; |
| 152 | pte_t *pte; |
| 153 | |
| 154 | if (pgd_none(*pgd)) |
| 155 | return false; |
| 156 | p4d = p4d_offset(pgd, addr); |
| 157 | if (p4d_none(*p4d)) |
| 158 | return false; |
| 159 | pud = pud_offset(p4d, addr); |
| 160 | if (pud_none(*pud)) |
| 161 | return false; |
| 162 | |
| 163 | /* |
| 164 | * We can't use pud_large() or pud_huge(), the first one is |
| 165 | * arch-specific, the last one depends on HUGETLB_PAGE. So let's abuse |
| 166 | * pud_bad(), if pud is bad then it's bad because it's huge. |
| 167 | */ |
| 168 | if (pud_bad(*pud)) |
| 169 | return true; |
| 170 | pmd = pmd_offset(pud, addr); |
| 171 | if (pmd_none(*pmd)) |
| 172 | return false; |
| 173 | |
| 174 | if (pmd_bad(*pmd)) |
| 175 | return true; |
| 176 | pte = pte_offset_kernel(pmd, addr); |
| 177 | return !pte_none(*pte); |
| 178 | } |
| 179 | |
| 180 | static int __meminit kasan_mem_notifier(struct notifier_block *nb, |
| 181 | unsigned long action, void *data) |
| 182 | { |
| 183 | struct memory_notify *mem_data = data; |
| 184 | unsigned long nr_shadow_pages, start_kaddr, shadow_start; |
| 185 | unsigned long shadow_end, shadow_size; |
| 186 | |
| 187 | nr_shadow_pages = mem_data->nr_pages >> KASAN_SHADOW_SCALE_SHIFT; |
| 188 | start_kaddr = (unsigned long)pfn_to_kaddr(mem_data->start_pfn); |
| 189 | shadow_start = (unsigned long)kasan_mem_to_shadow((void *)start_kaddr); |
| 190 | shadow_size = nr_shadow_pages << PAGE_SHIFT; |
| 191 | shadow_end = shadow_start + shadow_size; |
| 192 | |
| 193 | if (WARN_ON(mem_data->nr_pages % KASAN_GRANULE_SIZE) || |
Andrey Konovalov | affc3f0 | 2020-12-22 12:00:35 -0800 | [diff] [blame] | 194 | WARN_ON(start_kaddr % KASAN_MEMORY_PER_SHADOW_PAGE)) |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 195 | return NOTIFY_BAD; |
| 196 | |
| 197 | switch (action) { |
| 198 | case MEM_GOING_ONLINE: { |
| 199 | void *ret; |
| 200 | |
| 201 | /* |
| 202 | * If shadow is mapped already than it must have been mapped |
| 203 | * during the boot. This could happen if we onlining previously |
| 204 | * offlined memory. |
| 205 | */ |
| 206 | if (shadow_mapped(shadow_start)) |
| 207 | return NOTIFY_OK; |
| 208 | |
| 209 | ret = __vmalloc_node_range(shadow_size, PAGE_SIZE, shadow_start, |
| 210 | shadow_end, GFP_KERNEL, |
| 211 | PAGE_KERNEL, VM_NO_GUARD, |
| 212 | pfn_to_nid(mem_data->start_pfn), |
| 213 | __builtin_return_address(0)); |
| 214 | if (!ret) |
| 215 | return NOTIFY_BAD; |
| 216 | |
| 217 | kmemleak_ignore(ret); |
| 218 | return NOTIFY_OK; |
| 219 | } |
| 220 | case MEM_CANCEL_ONLINE: |
| 221 | case MEM_OFFLINE: { |
| 222 | struct vm_struct *vm; |
| 223 | |
| 224 | /* |
| 225 | * shadow_start was either mapped during boot by kasan_init() |
| 226 | * or during memory online by __vmalloc_node_range(). |
| 227 | * In the latter case we can use vfree() to free shadow. |
| 228 | * Non-NULL result of the find_vm_area() will tell us if |
| 229 | * that was the second case. |
| 230 | * |
| 231 | * Currently it's not possible to free shadow mapped |
| 232 | * during boot by kasan_init(). It's because the code |
| 233 | * to do that hasn't been written yet. So we'll just |
| 234 | * leak the memory. |
| 235 | */ |
| 236 | vm = find_vm_area((void *)shadow_start); |
| 237 | if (vm) |
| 238 | vfree((void *)shadow_start); |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | return NOTIFY_OK; |
| 243 | } |
| 244 | |
| 245 | static int __init kasan_memhotplug_init(void) |
| 246 | { |
| 247 | hotplug_memory_notifier(kasan_mem_notifier, 0); |
| 248 | |
| 249 | return 0; |
| 250 | } |
| 251 | |
| 252 | core_initcall(kasan_memhotplug_init); |
| 253 | #endif |
| 254 | |
| 255 | #ifdef CONFIG_KASAN_VMALLOC |
| 256 | |
Kefeng Wang | 3252b1d | 2021-11-05 13:39:47 -0700 | [diff] [blame] | 257 | void __init __weak kasan_populate_early_vm_area_shadow(void *start, |
| 258 | unsigned long size) |
| 259 | { |
| 260 | } |
| 261 | |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 262 | static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr, |
| 263 | void *unused) |
| 264 | { |
| 265 | unsigned long page; |
| 266 | pte_t pte; |
| 267 | |
| 268 | if (likely(!pte_none(*ptep))) |
| 269 | return 0; |
| 270 | |
| 271 | page = __get_free_page(GFP_KERNEL); |
| 272 | if (!page) |
| 273 | return -ENOMEM; |
| 274 | |
| 275 | memset((void *)page, KASAN_VMALLOC_INVALID, PAGE_SIZE); |
| 276 | pte = pfn_pte(PFN_DOWN(__pa(page)), PAGE_KERNEL); |
| 277 | |
| 278 | spin_lock(&init_mm.page_table_lock); |
| 279 | if (likely(pte_none(*ptep))) { |
| 280 | set_pte_at(&init_mm, addr, ptep, pte); |
| 281 | page = 0; |
| 282 | } |
| 283 | spin_unlock(&init_mm.page_table_lock); |
| 284 | if (page) |
| 285 | free_page(page); |
| 286 | return 0; |
| 287 | } |
| 288 | |
| 289 | int kasan_populate_vmalloc(unsigned long addr, unsigned long size) |
| 290 | { |
| 291 | unsigned long shadow_start, shadow_end; |
| 292 | int ret; |
| 293 | |
| 294 | if (!is_vmalloc_or_module_addr((void *)addr)) |
| 295 | return 0; |
| 296 | |
| 297 | shadow_start = (unsigned long)kasan_mem_to_shadow((void *)addr); |
| 298 | shadow_start = ALIGN_DOWN(shadow_start, PAGE_SIZE); |
| 299 | shadow_end = (unsigned long)kasan_mem_to_shadow((void *)addr + size); |
| 300 | shadow_end = ALIGN(shadow_end, PAGE_SIZE); |
| 301 | |
| 302 | ret = apply_to_page_range(&init_mm, shadow_start, |
| 303 | shadow_end - shadow_start, |
| 304 | kasan_populate_vmalloc_pte, NULL); |
| 305 | if (ret) |
| 306 | return ret; |
| 307 | |
| 308 | flush_cache_vmap(shadow_start, shadow_end); |
| 309 | |
| 310 | /* |
| 311 | * We need to be careful about inter-cpu effects here. Consider: |
| 312 | * |
| 313 | * CPU#0 CPU#1 |
| 314 | * WRITE_ONCE(p, vmalloc(100)); while (x = READ_ONCE(p)) ; |
| 315 | * p[99] = 1; |
| 316 | * |
| 317 | * With compiler instrumentation, that ends up looking like this: |
| 318 | * |
| 319 | * CPU#0 CPU#1 |
| 320 | * // vmalloc() allocates memory |
| 321 | * // let a = area->addr |
| 322 | * // we reach kasan_populate_vmalloc |
Andrey Konovalov | f00748b | 2021-02-24 12:05:05 -0800 | [diff] [blame] | 323 | * // and call kasan_unpoison: |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 324 | * STORE shadow(a), unpoison_val |
| 325 | * ... |
| 326 | * STORE shadow(a+99), unpoison_val x = LOAD p |
| 327 | * // rest of vmalloc process <data dependency> |
| 328 | * STORE p, a LOAD shadow(x+99) |
| 329 | * |
Ingo Molnar | f0953a1 | 2021-05-06 18:06:47 -0700 | [diff] [blame] | 330 | * If there is no barrier between the end of unpoisoning the shadow |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 331 | * and the store of the result to p, the stores could be committed |
| 332 | * in a different order by CPU#0, and CPU#1 could erroneously observe |
| 333 | * poison in the shadow. |
| 334 | * |
| 335 | * We need some sort of barrier between the stores. |
| 336 | * |
| 337 | * In the vmalloc() case, this is provided by a smp_wmb() in |
| 338 | * clear_vm_uninitialized_flag(). In the per-cpu allocator and in |
| 339 | * get_vm_area() and friends, the caller gets shadow allocated but |
| 340 | * doesn't have any pages mapped into the virtual address space that |
| 341 | * has been reserved. Mapping those pages in will involve taking and |
| 342 | * releasing a page-table lock, which will provide the barrier. |
| 343 | */ |
| 344 | |
| 345 | return 0; |
| 346 | } |
| 347 | |
| 348 | /* |
| 349 | * Poison the shadow for a vmalloc region. Called as part of the |
| 350 | * freeing process at the time the region is freed. |
| 351 | */ |
| 352 | void kasan_poison_vmalloc(const void *start, unsigned long size) |
| 353 | { |
| 354 | if (!is_vmalloc_or_module_addr(start)) |
| 355 | return; |
| 356 | |
| 357 | size = round_up(size, KASAN_GRANULE_SIZE); |
Andrey Konovalov | aa5c219 | 2021-04-29 22:59:59 -0700 | [diff] [blame] | 358 | kasan_poison(start, size, KASAN_VMALLOC_INVALID, false); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 359 | } |
| 360 | |
| 361 | void kasan_unpoison_vmalloc(const void *start, unsigned long size) |
| 362 | { |
| 363 | if (!is_vmalloc_or_module_addr(start)) |
| 364 | return; |
| 365 | |
Andrey Konovalov | aa5c219 | 2021-04-29 22:59:59 -0700 | [diff] [blame] | 366 | kasan_unpoison(start, size, false); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 367 | } |
| 368 | |
| 369 | static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr, |
| 370 | void *unused) |
| 371 | { |
| 372 | unsigned long page; |
| 373 | |
| 374 | page = (unsigned long)__va(pte_pfn(*ptep) << PAGE_SHIFT); |
| 375 | |
| 376 | spin_lock(&init_mm.page_table_lock); |
| 377 | |
| 378 | if (likely(!pte_none(*ptep))) { |
| 379 | pte_clear(&init_mm, addr, ptep); |
| 380 | free_page(page); |
| 381 | } |
| 382 | spin_unlock(&init_mm.page_table_lock); |
| 383 | |
| 384 | return 0; |
| 385 | } |
| 386 | |
| 387 | /* |
| 388 | * Release the backing for the vmalloc region [start, end), which |
| 389 | * lies within the free region [free_region_start, free_region_end). |
| 390 | * |
| 391 | * This can be run lazily, long after the region was freed. It runs |
| 392 | * under vmap_area_lock, so it's not safe to interact with the vmalloc/vmap |
| 393 | * infrastructure. |
| 394 | * |
| 395 | * How does this work? |
| 396 | * ------------------- |
| 397 | * |
Ingo Molnar | f0953a1 | 2021-05-06 18:06:47 -0700 | [diff] [blame] | 398 | * We have a region that is page aligned, labeled as A. |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 399 | * That might not map onto the shadow in a way that is page-aligned: |
| 400 | * |
| 401 | * start end |
| 402 | * v v |
| 403 | * |????????|????????|AAAAAAAA|AA....AA|AAAAAAAA|????????| < vmalloc |
| 404 | * -------- -------- -------- -------- -------- |
| 405 | * | | | | | |
| 406 | * | | | /-------/ | |
| 407 | * \-------\|/------/ |/---------------/ |
| 408 | * ||| || |
| 409 | * |??AAAAAA|AAAAAAAA|AA??????| < shadow |
| 410 | * (1) (2) (3) |
| 411 | * |
| 412 | * First we align the start upwards and the end downwards, so that the |
| 413 | * shadow of the region aligns with shadow page boundaries. In the |
| 414 | * example, this gives us the shadow page (2). This is the shadow entirely |
| 415 | * covered by this allocation. |
| 416 | * |
| 417 | * Then we have the tricky bits. We want to know if we can free the |
| 418 | * partially covered shadow pages - (1) and (3) in the example. For this, |
| 419 | * we are given the start and end of the free region that contains this |
| 420 | * allocation. Extending our previous example, we could have: |
| 421 | * |
| 422 | * free_region_start free_region_end |
| 423 | * | start end | |
| 424 | * v v v v |
| 425 | * |FFFFFFFF|FFFFFFFF|AAAAAAAA|AA....AA|AAAAAAAA|FFFFFFFF| < vmalloc |
| 426 | * -------- -------- -------- -------- -------- |
| 427 | * | | | | | |
| 428 | * | | | /-------/ | |
| 429 | * \-------\|/------/ |/---------------/ |
| 430 | * ||| || |
| 431 | * |FFAAAAAA|AAAAAAAA|AAF?????| < shadow |
| 432 | * (1) (2) (3) |
| 433 | * |
| 434 | * Once again, we align the start of the free region up, and the end of |
| 435 | * the free region down so that the shadow is page aligned. So we can free |
| 436 | * page (1) - we know no allocation currently uses anything in that page, |
| 437 | * because all of it is in the vmalloc free region. But we cannot free |
| 438 | * page (3), because we can't be sure that the rest of it is unused. |
| 439 | * |
| 440 | * We only consider pages that contain part of the original region for |
| 441 | * freeing: we don't try to free other pages from the free region or we'd |
| 442 | * end up trying to free huge chunks of virtual address space. |
| 443 | * |
| 444 | * Concurrency |
| 445 | * ----------- |
| 446 | * |
| 447 | * How do we know that we're not freeing a page that is simultaneously |
| 448 | * being used for a fresh allocation in kasan_populate_vmalloc(_pte)? |
| 449 | * |
| 450 | * We _can_ have kasan_release_vmalloc and kasan_populate_vmalloc running |
| 451 | * at the same time. While we run under free_vmap_area_lock, the population |
| 452 | * code does not. |
| 453 | * |
| 454 | * free_vmap_area_lock instead operates to ensure that the larger range |
| 455 | * [free_region_start, free_region_end) is safe: because __alloc_vmap_area and |
| 456 | * the per-cpu region-finding algorithm both run under free_vmap_area_lock, |
| 457 | * no space identified as free will become used while we are running. This |
| 458 | * means that so long as we are careful with alignment and only free shadow |
| 459 | * pages entirely covered by the free region, we will not run in to any |
| 460 | * trouble - any simultaneous allocations will be for disjoint regions. |
| 461 | */ |
| 462 | void kasan_release_vmalloc(unsigned long start, unsigned long end, |
| 463 | unsigned long free_region_start, |
| 464 | unsigned long free_region_end) |
| 465 | { |
| 466 | void *shadow_start, *shadow_end; |
| 467 | unsigned long region_start, region_end; |
| 468 | unsigned long size; |
| 469 | |
Andrey Konovalov | affc3f0 | 2020-12-22 12:00:35 -0800 | [diff] [blame] | 470 | region_start = ALIGN(start, KASAN_MEMORY_PER_SHADOW_PAGE); |
| 471 | region_end = ALIGN_DOWN(end, KASAN_MEMORY_PER_SHADOW_PAGE); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 472 | |
Andrey Konovalov | affc3f0 | 2020-12-22 12:00:35 -0800 | [diff] [blame] | 473 | free_region_start = ALIGN(free_region_start, KASAN_MEMORY_PER_SHADOW_PAGE); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 474 | |
| 475 | if (start != region_start && |
| 476 | free_region_start < region_start) |
Andrey Konovalov | affc3f0 | 2020-12-22 12:00:35 -0800 | [diff] [blame] | 477 | region_start -= KASAN_MEMORY_PER_SHADOW_PAGE; |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 478 | |
Andrey Konovalov | affc3f0 | 2020-12-22 12:00:35 -0800 | [diff] [blame] | 479 | free_region_end = ALIGN_DOWN(free_region_end, KASAN_MEMORY_PER_SHADOW_PAGE); |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 480 | |
| 481 | if (end != region_end && |
| 482 | free_region_end > region_end) |
Andrey Konovalov | affc3f0 | 2020-12-22 12:00:35 -0800 | [diff] [blame] | 483 | region_end += KASAN_MEMORY_PER_SHADOW_PAGE; |
Andrey Konovalov | bb359db | 2020-12-22 12:00:32 -0800 | [diff] [blame] | 484 | |
| 485 | shadow_start = kasan_mem_to_shadow((void *)region_start); |
| 486 | shadow_end = kasan_mem_to_shadow((void *)region_end); |
| 487 | |
| 488 | if (shadow_end > shadow_start) { |
| 489 | size = shadow_end - shadow_start; |
| 490 | apply_to_existing_page_range(&init_mm, |
| 491 | (unsigned long)shadow_start, |
| 492 | size, kasan_depopulate_vmalloc_pte, |
| 493 | NULL); |
| 494 | flush_tlb_kernel_range((unsigned long)shadow_start, |
| 495 | (unsigned long)shadow_end); |
| 496 | } |
| 497 | } |
| 498 | |
| 499 | #else /* CONFIG_KASAN_VMALLOC */ |
| 500 | |
| 501 | int kasan_module_alloc(void *addr, size_t size) |
| 502 | { |
| 503 | void *ret; |
| 504 | size_t scaled_size; |
| 505 | size_t shadow_size; |
| 506 | unsigned long shadow_start; |
| 507 | |
| 508 | shadow_start = (unsigned long)kasan_mem_to_shadow(addr); |
| 509 | scaled_size = (size + KASAN_GRANULE_SIZE - 1) >> |
| 510 | KASAN_SHADOW_SCALE_SHIFT; |
| 511 | shadow_size = round_up(scaled_size, PAGE_SIZE); |
| 512 | |
| 513 | if (WARN_ON(!PAGE_ALIGNED(shadow_start))) |
| 514 | return -EINVAL; |
| 515 | |
| 516 | ret = __vmalloc_node_range(shadow_size, 1, shadow_start, |
| 517 | shadow_start + shadow_size, |
| 518 | GFP_KERNEL, |
| 519 | PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE, |
| 520 | __builtin_return_address(0)); |
| 521 | |
| 522 | if (ret) { |
| 523 | __memset(ret, KASAN_SHADOW_INIT, shadow_size); |
| 524 | find_vm_area(addr)->flags |= VM_KASAN; |
| 525 | kmemleak_ignore(ret); |
| 526 | return 0; |
| 527 | } |
| 528 | |
| 529 | return -ENOMEM; |
| 530 | } |
| 531 | |
| 532 | void kasan_free_shadow(const struct vm_struct *vm) |
| 533 | { |
| 534 | if (vm->flags & VM_KASAN) |
| 535 | vfree(kasan_mem_to_shadow(vm->addr)); |
| 536 | } |
| 537 | |
| 538 | #endif |