Thomas Gleixner | caab277 | 2019-06-03 07:44:50 +0200 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
Catalin Marinas | f1a0c4a | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (C) 2012 ARM Ltd. |
Catalin Marinas | f1a0c4a | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 4 | */ |
| 5 | #ifndef __ASM_CACHE_H |
| 6 | #define __ASM_CACHE_H |
| 7 | |
Will Deacon | 02f7760 | 2017-03-10 20:32:23 +0000 | [diff] [blame] | 8 | #include <asm/cputype.h> |
| 9 | |
| 10 | #define CTR_L1IP_SHIFT 14 |
| 11 | #define CTR_L1IP_MASK 3 |
Shanker Donthineni | 6ae4b6e | 2018-03-07 09:00:08 -0600 | [diff] [blame] | 12 | #define CTR_DMINLINE_SHIFT 16 |
Suzuki K Poulose | 4c4a39d | 2018-07-04 23:07:45 +0100 | [diff] [blame] | 13 | #define CTR_IMINLINE_SHIFT 0 |
James Morse | ee9d90b | 2019-10-17 18:42:59 +0100 | [diff] [blame] | 14 | #define CTR_IMINLINE_MASK 0xf |
Shanker Donthineni | 6ae4b6e | 2018-03-07 09:00:08 -0600 | [diff] [blame] | 15 | #define CTR_ERG_SHIFT 20 |
Will Deacon | 02f7760 | 2017-03-10 20:32:23 +0000 | [diff] [blame] | 16 | #define CTR_CWG_SHIFT 24 |
| 17 | #define CTR_CWG_MASK 15 |
Shanker Donthineni | 6ae4b6e | 2018-03-07 09:00:08 -0600 | [diff] [blame] | 18 | #define CTR_IDC_SHIFT 28 |
| 19 | #define CTR_DIC_SHIFT 29 |
Will Deacon | 02f7760 | 2017-03-10 20:32:23 +0000 | [diff] [blame] | 20 | |
Suzuki K Poulose | 4c4a39d | 2018-07-04 23:07:45 +0100 | [diff] [blame] | 21 | #define CTR_CACHE_MINLINE_MASK \ |
James Morse | ee9d90b | 2019-10-17 18:42:59 +0100 | [diff] [blame] | 22 | (0xf << CTR_DMINLINE_SHIFT | CTR_IMINLINE_MASK << CTR_IMINLINE_SHIFT) |
Suzuki K Poulose | 4c4a39d | 2018-07-04 23:07:45 +0100 | [diff] [blame] | 23 | |
Will Deacon | 02f7760 | 2017-03-10 20:32:23 +0000 | [diff] [blame] | 24 | #define CTR_L1IP(ctr) (((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK) |
| 25 | |
Will Deacon | dda288d | 2017-03-10 20:32:24 +0000 | [diff] [blame] | 26 | #define ICACHE_POLICY_VPIPT 0 |
Arnd Bergmann | 332576e | 2020-10-26 20:37:46 +0100 | [diff] [blame] | 27 | #define ICACHE_POLICY_RESERVED 1 |
Will Deacon | 02f7760 | 2017-03-10 20:32:23 +0000 | [diff] [blame] | 28 | #define ICACHE_POLICY_VIPT 2 |
| 29 | #define ICACHE_POLICY_PIPT 3 |
Catalin Marinas | a41dc0e | 2014-04-03 17:48:54 +0100 | [diff] [blame] | 30 | |
Catalin Marinas | d93277b | 2018-05-11 13:25:49 +0100 | [diff] [blame] | 31 | #define L1_CACHE_SHIFT (6) |
Catalin Marinas | f1a0c4a | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 32 | #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) |
| 33 | |
Suzuki K Poulose | 1602df0 | 2018-10-09 14:47:06 +0100 | [diff] [blame] | 34 | |
| 35 | #define CLIDR_LOUU_SHIFT 27 |
| 36 | #define CLIDR_LOC_SHIFT 24 |
| 37 | #define CLIDR_LOUIS_SHIFT 21 |
| 38 | |
| 39 | #define CLIDR_LOUU(clidr) (((clidr) >> CLIDR_LOUU_SHIFT) & 0x7) |
| 40 | #define CLIDR_LOC(clidr) (((clidr) >> CLIDR_LOC_SHIFT) & 0x7) |
| 41 | #define CLIDR_LOUIS(clidr) (((clidr) >> CLIDR_LOUIS_SHIFT) & 0x7) |
| 42 | |
Catalin Marinas | f1a0c4a | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 43 | /* |
| 44 | * Memory returned by kmalloc() may be used for DMA, so we must make |
| 45 | * sure that all such allocations are cache aligned. Otherwise, |
| 46 | * unrelated code may cause parts of the buffer to be read into the |
| 47 | * cache before the transfer is done, causing old data to be seen by |
| 48 | * the CPU. |
| 49 | */ |
Catalin Marinas | ebc7e21 | 2018-05-11 13:33:12 +0100 | [diff] [blame] | 50 | #define ARCH_DMA_MINALIGN (128) |
Catalin Marinas | a41dc0e | 2014-04-03 17:48:54 +0100 | [diff] [blame] | 51 | |
Andrey Konovalov | eb214f2 | 2019-01-08 15:23:11 -0800 | [diff] [blame] | 52 | #ifdef CONFIG_KASAN_SW_TAGS |
| 53 | #define ARCH_SLAB_MINALIGN (1ULL << KASAN_SHADOW_SCALE_SHIFT) |
Andrey Konovalov | 03c7581 | 2020-12-22 12:01:52 -0800 | [diff] [blame] | 54 | #elif defined(CONFIG_KASAN_HW_TAGS) |
| 55 | #define ARCH_SLAB_MINALIGN MTE_GRANULE_SIZE |
Andrey Konovalov | eb214f2 | 2019-01-08 15:23:11 -0800 | [diff] [blame] | 56 | #endif |
| 57 | |
Catalin Marinas | a41dc0e | 2014-04-03 17:48:54 +0100 | [diff] [blame] | 58 | #ifndef __ASSEMBLY__ |
| 59 | |
Will Deacon | 02f7760 | 2017-03-10 20:32:23 +0000 | [diff] [blame] | 60 | #include <linux/bitops.h> |
| 61 | |
| 62 | #define ICACHEF_ALIASING 0 |
Will Deacon | dda288d | 2017-03-10 20:32:24 +0000 | [diff] [blame] | 63 | #define ICACHEF_VPIPT 1 |
Will Deacon | 02f7760 | 2017-03-10 20:32:23 +0000 | [diff] [blame] | 64 | extern unsigned long __icache_flags; |
| 65 | |
| 66 | /* |
| 67 | * Whilst the D-side always behaves as PIPT on AArch64, aliasing is |
| 68 | * permitted in the I-cache. |
| 69 | */ |
| 70 | static inline int icache_is_aliasing(void) |
| 71 | { |
| 72 | return test_bit(ICACHEF_ALIASING, &__icache_flags); |
| 73 | } |
| 74 | |
James Morse | e43f133 | 2020-02-20 16:58:39 +0000 | [diff] [blame] | 75 | static __always_inline int icache_is_vpipt(void) |
Will Deacon | dda288d | 2017-03-10 20:32:24 +0000 | [diff] [blame] | 76 | { |
| 77 | return test_bit(ICACHEF_VPIPT, &__icache_flags); |
| 78 | } |
| 79 | |
Will Deacon | 02f7760 | 2017-03-10 20:32:23 +0000 | [diff] [blame] | 80 | static inline u32 cache_type_cwg(void) |
| 81 | { |
| 82 | return (read_cpuid_cachetype() >> CTR_CWG_SHIFT) & CTR_CWG_MASK; |
| 83 | } |
| 84 | |
Joe Perches | 33def84 | 2020-10-21 19:36:07 -0700 | [diff] [blame] | 85 | #define __read_mostly __section(".data..read_mostly") |
Jungseok Lee | e4f88d8 | 2014-12-02 17:49:24 +0000 | [diff] [blame] | 86 | |
Masayoshi Mizuma | 8f5c903 | 2019-06-14 09:11:41 -0400 | [diff] [blame] | 87 | static inline int cache_line_size_of_cpu(void) |
Catalin Marinas | a41dc0e | 2014-04-03 17:48:54 +0100 | [diff] [blame] | 88 | { |
| 89 | u32 cwg = cache_type_cwg(); |
Masayoshi Mizuma | 8f5c903 | 2019-06-14 09:11:41 -0400 | [diff] [blame] | 90 | |
Catalin Marinas | ebc7e21 | 2018-05-11 13:33:12 +0100 | [diff] [blame] | 91 | return cwg ? 4 << cwg : ARCH_DMA_MINALIGN; |
Catalin Marinas | a41dc0e | 2014-04-03 17:48:54 +0100 | [diff] [blame] | 92 | } |
| 93 | |
Shaokun Zhang | 7b8c87b | 2019-05-28 10:16:54 +0800 | [diff] [blame] | 94 | int cache_line_size(void); |
Catalin Marinas | a41dc0e | 2014-04-03 17:48:54 +0100 | [diff] [blame] | 95 | |
Suzuki K Poulose | 1602df0 | 2018-10-09 14:47:06 +0100 | [diff] [blame] | 96 | /* |
| 97 | * Read the effective value of CTR_EL0. |
| 98 | * |
| 99 | * According to ARM ARM for ARMv8-A (ARM DDI 0487C.a), |
| 100 | * section D10.2.33 "CTR_EL0, Cache Type Register" : |
| 101 | * |
| 102 | * CTR_EL0.IDC reports the data cache clean requirements for |
| 103 | * instruction to data coherence. |
| 104 | * |
| 105 | * 0 - dcache clean to PoU is required unless : |
| 106 | * (CLIDR_EL1.LoC == 0) || (CLIDR_EL1.LoUIS == 0 && CLIDR_EL1.LoUU == 0) |
| 107 | * 1 - dcache clean to PoU is not required for i-to-d coherence. |
| 108 | * |
| 109 | * This routine provides the CTR_EL0 with the IDC field updated to the |
| 110 | * effective state. |
| 111 | */ |
| 112 | static inline u32 __attribute_const__ read_cpuid_effective_cachetype(void) |
| 113 | { |
| 114 | u32 ctr = read_cpuid_cachetype(); |
| 115 | |
| 116 | if (!(ctr & BIT(CTR_IDC_SHIFT))) { |
| 117 | u64 clidr = read_sysreg(clidr_el1); |
| 118 | |
| 119 | if (CLIDR_LOC(clidr) == 0 || |
| 120 | (CLIDR_LOUIS(clidr) == 0 && CLIDR_LOUU(clidr) == 0)) |
| 121 | ctr |= BIT(CTR_IDC_SHIFT); |
| 122 | } |
| 123 | |
| 124 | return ctr; |
| 125 | } |
| 126 | |
Catalin Marinas | a41dc0e | 2014-04-03 17:48:54 +0100 | [diff] [blame] | 127 | #endif /* __ASSEMBLY__ */ |
Catalin Marinas | f1a0c4a | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 128 | |
| 129 | #endif |