Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved. |
| 3 | * |
| 4 | * SPDX-License-Identifier: GPL-2.0+ |
| 5 | */ |
| 6 | |
| 7 | #include <config.h> |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 8 | #include <common.h> |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 9 | #include <linux/compiler.h> |
| 10 | #include <linux/kernel.h> |
Alexey Brodkin | 97a6314 | 2017-06-26 11:46:47 +0300 | [diff] [blame] | 11 | #include <linux/log2.h> |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 12 | #include <asm/arcregs.h> |
Eugeniy Paltsev | 88ae27e | 2018-03-21 15:58:52 +0300 | [diff] [blame] | 13 | #include <asm/arc-bcr.h> |
Alexey Brodkin | 205e7a7 | 2015-02-03 13:58:13 +0300 | [diff] [blame] | 14 | #include <asm/cache.h> |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 15 | |
Eugeniy Paltsev | c27814b | 2018-03-21 15:58:50 +0300 | [diff] [blame] | 16 | /* |
| 17 | * [ NOTE 1 ]: |
| 18 | * Data cache (L1 D$ or SL$) entire invalidate operation or data cache disable |
| 19 | * operation may result in unexpected behavior and data loss even if we flush |
| 20 | * data cache right before invalidation. That may happens if we store any context |
| 21 | * on stack (like we store BLINK register on stack before function call). |
| 22 | * BLINK register is the register where return address is automatically saved |
| 23 | * when we do function call with instructions like 'bl'. |
| 24 | * |
| 25 | * There is the real example: |
| 26 | * We may hang in the next code as we store any BLINK register on stack in |
| 27 | * invalidate_dcache_all() function. |
| 28 | * |
| 29 | * void flush_dcache_all() { |
| 30 | * __dc_entire_op(OP_FLUSH); |
| 31 | * // Other code // |
| 32 | * } |
| 33 | * |
| 34 | * void invalidate_dcache_all() { |
| 35 | * __dc_entire_op(OP_INV); |
| 36 | * // Other code // |
| 37 | * } |
| 38 | * |
| 39 | * void foo(void) { |
| 40 | * flush_dcache_all(); |
| 41 | * invalidate_dcache_all(); |
| 42 | * } |
| 43 | * |
| 44 | * Now let's see what really happens during that code execution: |
| 45 | * |
| 46 | * foo() |
| 47 | * |->> call flush_dcache_all |
| 48 | * [return address is saved to BLINK register] |
| 49 | * [push BLINK] (save to stack) ![point 1] |
| 50 | * |->> call __dc_entire_op(OP_FLUSH) |
| 51 | * [return address is saved to BLINK register] |
| 52 | * [flush L1 D$] |
| 53 | * return [jump to BLINK] |
| 54 | * <<------ |
| 55 | * [other flush_dcache_all code] |
| 56 | * [pop BLINK] (get from stack) |
| 57 | * return [jump to BLINK] |
| 58 | * <<------ |
| 59 | * |->> call invalidate_dcache_all |
| 60 | * [return address is saved to BLINK register] |
| 61 | * [push BLINK] (save to stack) ![point 2] |
| 62 | * |->> call __dc_entire_op(OP_FLUSH) |
| 63 | * [return address is saved to BLINK register] |
| 64 | * [invalidate L1 D$] ![point 3] |
| 65 | * // Oops!!! |
| 66 | * // We lose return address from invalidate_dcache_all function: |
| 67 | * // we save it to stack and invalidate L1 D$ after that! |
| 68 | * return [jump to BLINK] |
| 69 | * <<------ |
| 70 | * [other invalidate_dcache_all code] |
| 71 | * [pop BLINK] (get from stack) |
| 72 | * // we don't have this data in L1 dcache as we invalidated it in [point 3] |
| 73 | * // so we get it from next memory level (for example DDR memory) |
| 74 | * // but in the memory we have value which we save in [point 1], which |
| 75 | * // is return address from flush_dcache_all function (instead of |
| 76 | * // address from current invalidate_dcache_all function which we |
| 77 | * // saved in [point 2] !) |
| 78 | * return [jump to BLINK] |
| 79 | * <<------ |
| 80 | * // As BLINK points to invalidate_dcache_all, we call it again and |
| 81 | * // loop forever. |
| 82 | * |
| 83 | * Fortunately we may fix that by using flush & invalidation of D$ with a single |
| 84 | * one instruction (instead of flush and invalidation instructions pair) and |
| 85 | * enabling force function inline with '__attribute__((always_inline))' gcc |
| 86 | * attribute to avoid any function call (and BLINK store) between cache flush |
| 87 | * and disable. |
| 88 | */ |
| 89 | |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 90 | /* Bit values in IC_CTRL */ |
Eugeniy Paltsev | 19b10a4 | 2018-01-16 19:20:29 +0300 | [diff] [blame] | 91 | #define IC_CTRL_CACHE_DISABLE BIT(0) |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 92 | |
| 93 | /* Bit values in DC_CTRL */ |
Eugeniy Paltsev | 19b10a4 | 2018-01-16 19:20:29 +0300 | [diff] [blame] | 94 | #define DC_CTRL_CACHE_DISABLE BIT(0) |
| 95 | #define DC_CTRL_INV_MODE_FLUSH BIT(6) |
| 96 | #define DC_CTRL_FLUSH_STATUS BIT(8) |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 97 | |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 98 | #define OP_INV BIT(0) |
| 99 | #define OP_FLUSH BIT(1) |
| 100 | #define OP_FLUSH_N_INV (OP_FLUSH | OP_INV) |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 101 | |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 102 | /* Bit val in SLC_CONTROL */ |
| 103 | #define SLC_CTRL_DIS 0x001 |
| 104 | #define SLC_CTRL_IM 0x040 |
| 105 | #define SLC_CTRL_BUSY 0x100 |
| 106 | #define SLC_CTRL_RGN_OP_INV 0x200 |
| 107 | |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 108 | /* |
| 109 | * By default that variable will fall into .bss section. |
| 110 | * But .bss section is not relocated and so it will be initilized before |
| 111 | * relocation but will be used after being zeroed. |
| 112 | */ |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 113 | int l1_line_sz __section(".data"); |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 114 | |
| 115 | #define CACHE_LINE_MASK (~(l1_line_sz - 1)) |
| 116 | |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 117 | int slc_line_sz __section(".data"); |
Eugeniy Paltsev | 3cf2393 | 2017-11-30 17:41:32 +0300 | [diff] [blame] | 118 | bool ioc_exists __section(".data") = false; |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 119 | |
Eugeniy Paltsev | b0146f9 | 2018-01-16 19:20:28 +0300 | [diff] [blame] | 120 | /* To force enable IOC set ioc_enable to 'true' */ |
| 121 | bool ioc_enable __section(".data") = false; |
| 122 | |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 123 | static inline bool pae_exists(void) |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 124 | { |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 125 | /* TODO: should we compare mmu version from BCR and from CONFIG? */ |
| 126 | #if (CONFIG_ARC_MMU_VER >= 4) |
Eugeniy Paltsev | 88ae27e | 2018-03-21 15:58:52 +0300 | [diff] [blame] | 127 | union bcr_mmu_4 mmu4; |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 128 | |
Eugeniy Paltsev | 88ae27e | 2018-03-21 15:58:52 +0300 | [diff] [blame] | 129 | mmu4.word = read_aux_reg(ARC_AUX_MMU_BCR); |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 130 | |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 131 | if (mmu4.fields.pae) |
| 132 | return true; |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 133 | #endif /* (CONFIG_ARC_MMU_VER >= 4) */ |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 134 | |
| 135 | return false; |
| 136 | } |
| 137 | |
| 138 | static inline bool icache_exists(void) |
| 139 | { |
| 140 | union bcr_di_cache ibcr; |
| 141 | |
| 142 | ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); |
| 143 | return !!ibcr.fields.ver; |
| 144 | } |
| 145 | |
| 146 | static inline bool dcache_exists(void) |
| 147 | { |
| 148 | union bcr_di_cache dbcr; |
| 149 | |
| 150 | dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); |
| 151 | return !!dbcr.fields.ver; |
| 152 | } |
| 153 | |
| 154 | static inline bool slc_exists(void) |
| 155 | { |
| 156 | if (is_isa_arcv2()) { |
| 157 | union bcr_generic sbcr; |
| 158 | |
| 159 | sbcr.word = read_aux_reg(ARC_BCR_SLC); |
| 160 | return !!sbcr.fields.ver; |
| 161 | } |
| 162 | |
| 163 | return false; |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 164 | } |
| 165 | |
| 166 | static void __slc_entire_op(const int op) |
| 167 | { |
| 168 | unsigned int ctrl; |
| 169 | |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 170 | if (!slc_exists()) |
Eugeniy Paltsev | ea9f6f1 | 2018-03-21 15:58:55 +0300 | [diff] [blame] | 171 | return; |
| 172 | |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 173 | ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); |
| 174 | |
| 175 | if (!(op & OP_FLUSH)) /* i.e. OP_INV */ |
| 176 | ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ |
| 177 | else |
| 178 | ctrl |= SLC_CTRL_IM; |
| 179 | |
| 180 | write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); |
| 181 | |
| 182 | if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ |
| 183 | write_aux_reg(ARC_AUX_SLC_INVALIDATE, 0x1); |
| 184 | else |
| 185 | write_aux_reg(ARC_AUX_SLC_FLUSH, 0x1); |
| 186 | |
| 187 | /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ |
| 188 | read_aux_reg(ARC_AUX_SLC_CTRL); |
| 189 | |
| 190 | /* Important to wait for flush to complete */ |
| 191 | while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY); |
| 192 | } |
| 193 | |
| 194 | static void slc_upper_region_init(void) |
| 195 | { |
| 196 | /* |
| 197 | * ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1 are always == 0 |
| 198 | * as we don't use PAE40. |
| 199 | */ |
| 200 | write_aux_reg(ARC_AUX_SLC_RGN_END1, 0); |
| 201 | write_aux_reg(ARC_AUX_SLC_RGN_START1, 0); |
| 202 | } |
| 203 | |
| 204 | static void __slc_rgn_op(unsigned long paddr, unsigned long sz, const int op) |
| 205 | { |
Eugeniy Paltsev | 05c6a26 | 2018-03-21 15:58:54 +0300 | [diff] [blame] | 206 | #ifdef CONFIG_ISA_ARCV2 |
| 207 | |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 208 | unsigned int ctrl; |
| 209 | unsigned long end; |
| 210 | |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 211 | if (!slc_exists()) |
Eugeniy Paltsev | ea9f6f1 | 2018-03-21 15:58:55 +0300 | [diff] [blame] | 212 | return; |
| 213 | |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 214 | /* |
| 215 | * The Region Flush operation is specified by CTRL.RGN_OP[11..9] |
| 216 | * - b'000 (default) is Flush, |
| 217 | * - b'001 is Invalidate if CTRL.IM == 0 |
| 218 | * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 |
| 219 | */ |
| 220 | ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); |
| 221 | |
| 222 | /* Don't rely on default value of IM bit */ |
| 223 | if (!(op & OP_FLUSH)) /* i.e. OP_INV */ |
| 224 | ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ |
| 225 | else |
| 226 | ctrl |= SLC_CTRL_IM; |
| 227 | |
| 228 | if (op & OP_INV) |
| 229 | ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ |
| 230 | else |
| 231 | ctrl &= ~SLC_CTRL_RGN_OP_INV; |
| 232 | |
| 233 | write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); |
| 234 | |
| 235 | /* |
| 236 | * Lower bits are ignored, no need to clip |
| 237 | * END needs to be setup before START (latter triggers the operation) |
| 238 | * END can't be same as START, so add (l2_line_sz - 1) to sz |
| 239 | */ |
| 240 | end = paddr + sz + slc_line_sz - 1; |
| 241 | |
| 242 | /* |
| 243 | * Upper addresses (ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1) |
| 244 | * are always == 0 as we don't use PAE40, so we only setup lower ones |
| 245 | * (ARC_AUX_SLC_RGN_END and ARC_AUX_SLC_RGN_START) |
| 246 | */ |
| 247 | write_aux_reg(ARC_AUX_SLC_RGN_END, end); |
| 248 | write_aux_reg(ARC_AUX_SLC_RGN_START, paddr); |
| 249 | |
| 250 | /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ |
| 251 | read_aux_reg(ARC_AUX_SLC_CTRL); |
| 252 | |
| 253 | while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY); |
Eugeniy Paltsev | 05c6a26 | 2018-03-21 15:58:54 +0300 | [diff] [blame] | 254 | |
| 255 | #endif /* CONFIG_ISA_ARCV2 */ |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 256 | } |
Eugeniy Paltsev | a6f557c | 2018-03-21 15:58:51 +0300 | [diff] [blame] | 257 | |
| 258 | static void arc_ioc_setup(void) |
| 259 | { |
| 260 | /* IOC Aperture start is equal to DDR start */ |
| 261 | unsigned int ap_base = CONFIG_SYS_SDRAM_BASE; |
| 262 | /* IOC Aperture size is equal to DDR size */ |
| 263 | long ap_size = CONFIG_SYS_SDRAM_SIZE; |
| 264 | |
| 265 | flush_n_invalidate_dcache_all(); |
| 266 | |
| 267 | if (!is_power_of_2(ap_size) || ap_size < 4096) |
| 268 | panic("IOC Aperture size must be power of 2 and bigger 4Kib"); |
| 269 | |
| 270 | /* |
| 271 | * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB, |
| 272 | * so setting 0x11 implies 512M, 0x12 implies 1G... |
| 273 | */ |
| 274 | write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE, |
| 275 | order_base_2(ap_size / 1024) - 2); |
| 276 | |
| 277 | /* IOC Aperture start must be aligned to the size of the aperture */ |
| 278 | if (ap_base % ap_size != 0) |
| 279 | panic("IOC Aperture start must be aligned to the size of the aperture"); |
| 280 | |
| 281 | write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, ap_base >> 12); |
| 282 | write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1); |
| 283 | write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1); |
| 284 | } |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 285 | |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 286 | static void read_decode_cache_bcr_arcv2(void) |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 287 | { |
Eugeniy Paltsev | 05c6a26 | 2018-03-21 15:58:54 +0300 | [diff] [blame] | 288 | #ifdef CONFIG_ISA_ARCV2 |
| 289 | |
Eugeniy Paltsev | 88ae27e | 2018-03-21 15:58:52 +0300 | [diff] [blame] | 290 | union bcr_slc_cfg slc_cfg; |
| 291 | union bcr_clust_cfg cbcr; |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 292 | |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 293 | if (slc_exists()) { |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 294 | slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG); |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 295 | slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64; |
| 296 | } |
Alexey Brodkin | db6ce23 | 2015-12-14 17:15:13 +0300 | [diff] [blame] | 297 | |
Alexey Brodkin | db6ce23 | 2015-12-14 17:15:13 +0300 | [diff] [blame] | 298 | cbcr.word = read_aux_reg(ARC_BCR_CLUSTER); |
Eugeniy Paltsev | b0146f9 | 2018-01-16 19:20:28 +0300 | [diff] [blame] | 299 | if (cbcr.fields.c && ioc_enable) |
Eugeniy Paltsev | 3cf2393 | 2017-11-30 17:41:32 +0300 | [diff] [blame] | 300 | ioc_exists = true; |
Eugeniy Paltsev | 05c6a26 | 2018-03-21 15:58:54 +0300 | [diff] [blame] | 301 | |
| 302 | #endif /* CONFIG_ISA_ARCV2 */ |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 303 | } |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 304 | |
| 305 | void read_decode_cache_bcr(void) |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 306 | { |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 307 | int dc_line_sz = 0, ic_line_sz = 0; |
Eugeniy Paltsev | 88ae27e | 2018-03-21 15:58:52 +0300 | [diff] [blame] | 308 | union bcr_di_cache ibcr, dbcr; |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 309 | |
| 310 | ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); |
| 311 | if (ibcr.fields.ver) { |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 312 | l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len; |
| 313 | if (!ic_line_sz) |
| 314 | panic("Instruction exists but line length is 0\n"); |
| 315 | } |
| 316 | |
| 317 | dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); |
Eugeniy Paltsev | 19b10a4 | 2018-01-16 19:20:29 +0300 | [diff] [blame] | 318 | if (dbcr.fields.ver) { |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 319 | l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len; |
| 320 | if (!dc_line_sz) |
| 321 | panic("Data cache exists but line length is 0\n"); |
| 322 | } |
| 323 | |
| 324 | if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz)) |
| 325 | panic("Instruction and data cache line lengths differ\n"); |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 326 | } |
| 327 | |
| 328 | void cache_init(void) |
| 329 | { |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 330 | read_decode_cache_bcr(); |
| 331 | |
Eugeniy Paltsev | 05c6a26 | 2018-03-21 15:58:54 +0300 | [diff] [blame] | 332 | if (is_isa_arcv2()) |
| 333 | read_decode_cache_bcr_arcv2(); |
Alexey Brodkin | db6ce23 | 2015-12-14 17:15:13 +0300 | [diff] [blame] | 334 | |
Eugeniy Paltsev | 05c6a26 | 2018-03-21 15:58:54 +0300 | [diff] [blame] | 335 | if (is_isa_arcv2() && ioc_exists) |
Eugeniy Paltsev | a6f557c | 2018-03-21 15:58:51 +0300 | [diff] [blame] | 336 | arc_ioc_setup(); |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 337 | |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 338 | /* |
| 339 | * ARC_AUX_SLC_RGN_START1 and ARC_AUX_SLC_RGN_END1 register exist |
| 340 | * only if PAE exists in current HW. So we had to check pae_exist |
| 341 | * before using them. |
| 342 | */ |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 343 | if (is_isa_arcv2() && slc_exists() && pae_exists()) |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 344 | slc_upper_region_init(); |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 345 | } |
| 346 | |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 347 | int icache_status(void) |
| 348 | { |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 349 | if (!icache_exists()) |
Igor Guryanov | f8cf3d1 | 2014-12-24 16:07:07 +0300 | [diff] [blame] | 350 | return 0; |
| 351 | |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 352 | if (read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE) |
| 353 | return 0; |
| 354 | else |
| 355 | return 1; |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 356 | } |
| 357 | |
| 358 | void icache_enable(void) |
| 359 | { |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 360 | if (icache_exists()) |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 361 | write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) & |
| 362 | ~IC_CTRL_CACHE_DISABLE); |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 363 | } |
| 364 | |
| 365 | void icache_disable(void) |
| 366 | { |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 367 | if (icache_exists()) |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 368 | write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) | |
| 369 | IC_CTRL_CACHE_DISABLE); |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 370 | } |
| 371 | |
Eugeniy Paltsev | 16aeee8 | 2018-03-21 15:58:46 +0300 | [diff] [blame] | 372 | /* IC supports only invalidation */ |
| 373 | static inline void __ic_entire_invalidate(void) |
| 374 | { |
| 375 | if (!icache_status()) |
| 376 | return; |
| 377 | |
| 378 | /* Any write to IC_IVIC register triggers invalidation of entire I$ */ |
| 379 | write_aux_reg(ARC_AUX_IC_IVIC, 1); |
| 380 | /* |
| 381 | * As per ARC HS databook (see chapter 5.3.3.2) |
| 382 | * it is required to add 3 NOPs after each write to IC_IVIC. |
| 383 | */ |
| 384 | __builtin_arc_nop(); |
| 385 | __builtin_arc_nop(); |
| 386 | __builtin_arc_nop(); |
| 387 | read_aux_reg(ARC_AUX_IC_CTRL); /* blocks */ |
| 388 | } |
| 389 | |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 390 | void invalidate_icache_all(void) |
| 391 | { |
Eugeniy Paltsev | 16aeee8 | 2018-03-21 15:58:46 +0300 | [diff] [blame] | 392 | __ic_entire_invalidate(); |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 393 | |
Eugeniy Paltsev | ea9f6f1 | 2018-03-21 15:58:55 +0300 | [diff] [blame] | 394 | if (is_isa_arcv2()) |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 395 | __slc_entire_op(OP_INV); |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 396 | } |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 397 | |
| 398 | int dcache_status(void) |
| 399 | { |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 400 | if (!dcache_exists()) |
Igor Guryanov | f8cf3d1 | 2014-12-24 16:07:07 +0300 | [diff] [blame] | 401 | return 0; |
| 402 | |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 403 | if (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE) |
| 404 | return 0; |
| 405 | else |
| 406 | return 1; |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 407 | } |
| 408 | |
| 409 | void dcache_enable(void) |
| 410 | { |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 411 | if (!dcache_exists()) |
Igor Guryanov | f8cf3d1 | 2014-12-24 16:07:07 +0300 | [diff] [blame] | 412 | return; |
| 413 | |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 414 | write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) & |
| 415 | ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE)); |
| 416 | } |
| 417 | |
| 418 | void dcache_disable(void) |
| 419 | { |
Eugeniy Paltsev | 7579087 | 2018-03-21 15:58:56 +0300 | [diff] [blame^] | 420 | if (!dcache_exists()) |
Igor Guryanov | f8cf3d1 | 2014-12-24 16:07:07 +0300 | [diff] [blame] | 421 | return; |
| 422 | |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 423 | write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) | |
| 424 | DC_CTRL_CACHE_DISABLE); |
| 425 | } |
| 426 | |
Eugeniy Paltsev | c4ef14d | 2018-03-21 15:58:47 +0300 | [diff] [blame] | 427 | /* Common Helper for Line Operations on D-cache */ |
| 428 | static inline void __dcache_line_loop(unsigned long paddr, unsigned long sz, |
| 429 | const int cacheop) |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 430 | { |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 431 | unsigned int aux_cmd; |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 432 | int num_lines; |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 433 | |
Eugeniy Paltsev | c4ef14d | 2018-03-21 15:58:47 +0300 | [diff] [blame] | 434 | /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ |
| 435 | aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL; |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 436 | |
| 437 | sz += paddr & ~CACHE_LINE_MASK; |
| 438 | paddr &= CACHE_LINE_MASK; |
| 439 | |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 440 | num_lines = DIV_ROUND_UP(sz, l1_line_sz); |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 441 | |
| 442 | while (num_lines-- > 0) { |
| 443 | #if (CONFIG_ARC_MMU_VER == 3) |
Eugeniy Paltsev | c4ef14d | 2018-03-21 15:58:47 +0300 | [diff] [blame] | 444 | write_aux_reg(ARC_AUX_DC_PTAG, paddr); |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 445 | #endif |
| 446 | write_aux_reg(aux_cmd, paddr); |
Alexey Brodkin | 379b328 | 2015-12-14 17:14:46 +0300 | [diff] [blame] | 447 | paddr += l1_line_sz; |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 448 | } |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 449 | } |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 450 | |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 451 | static void __before_dc_op(const int op) |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 452 | { |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 453 | unsigned int ctrl; |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 454 | |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 455 | ctrl = read_aux_reg(ARC_AUX_DC_CTRL); |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 456 | |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 457 | /* IM bit implies flush-n-inv, instead of vanilla inv */ |
| 458 | if (op == OP_INV) |
| 459 | ctrl &= ~DC_CTRL_INV_MODE_FLUSH; |
| 460 | else |
| 461 | ctrl |= DC_CTRL_INV_MODE_FLUSH; |
| 462 | |
| 463 | write_aux_reg(ARC_AUX_DC_CTRL, ctrl); |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 464 | } |
| 465 | |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 466 | static void __after_dc_op(const int op) |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 467 | { |
| 468 | if (op & OP_FLUSH) /* flush / flush-n-inv both wait */ |
Eugeniy Paltsev | 19b10a4 | 2018-01-16 19:20:29 +0300 | [diff] [blame] | 469 | while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS); |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 470 | } |
| 471 | |
| 472 | static inline void __dc_entire_op(const int cacheop) |
| 473 | { |
| 474 | int aux; |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 475 | |
Eugeniy Paltsev | c877a89 | 2018-03-21 15:58:53 +0300 | [diff] [blame] | 476 | if (!dcache_status()) |
| 477 | return; |
| 478 | |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 479 | __before_dc_op(cacheop); |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 480 | |
| 481 | if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */ |
| 482 | aux = ARC_AUX_DC_IVDC; |
| 483 | else |
| 484 | aux = ARC_AUX_DC_FLSH; |
| 485 | |
| 486 | write_aux_reg(aux, 0x1); |
| 487 | |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 488 | __after_dc_op(cacheop); |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 489 | } |
| 490 | |
| 491 | static inline void __dc_line_op(unsigned long paddr, unsigned long sz, |
| 492 | const int cacheop) |
| 493 | { |
Eugeniy Paltsev | c877a89 | 2018-03-21 15:58:53 +0300 | [diff] [blame] | 494 | if (!dcache_status()) |
| 495 | return; |
| 496 | |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 497 | __before_dc_op(cacheop); |
Eugeniy Paltsev | c4ef14d | 2018-03-21 15:58:47 +0300 | [diff] [blame] | 498 | __dcache_line_loop(paddr, sz, cacheop); |
Eugeniy Paltsev | 5d7a24d | 2018-03-21 15:58:48 +0300 | [diff] [blame] | 499 | __after_dc_op(cacheop); |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 500 | } |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 501 | |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 502 | void invalidate_dcache_range(unsigned long start, unsigned long end) |
| 503 | { |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 504 | if (start >= end) |
| 505 | return; |
| 506 | |
Eugeniy Paltsev | 05c6a26 | 2018-03-21 15:58:54 +0300 | [diff] [blame] | 507 | /* |
| 508 | * ARCv1 -> call __dc_line_op |
| 509 | * ARCv2 && no IOC -> call __dc_line_op; call __slc_rgn_op |
| 510 | * ARCv2 && IOC enabled -> nothing |
| 511 | */ |
| 512 | if (!is_isa_arcv2() || !ioc_exists) |
Alexey Brodkin | db6ce23 | 2015-12-14 17:15:13 +0300 | [diff] [blame] | 513 | __dc_line_op(start, end - start, OP_INV); |
| 514 | |
Eugeniy Paltsev | ea9f6f1 | 2018-03-21 15:58:55 +0300 | [diff] [blame] | 515 | if (is_isa_arcv2() && !ioc_exists) |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 516 | __slc_rgn_op(start, end - start, OP_INV); |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 517 | } |
| 518 | |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 519 | void flush_dcache_range(unsigned long start, unsigned long end) |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 520 | { |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 521 | if (start >= end) |
| 522 | return; |
| 523 | |
Eugeniy Paltsev | 05c6a26 | 2018-03-21 15:58:54 +0300 | [diff] [blame] | 524 | /* |
| 525 | * ARCv1 -> call __dc_line_op |
| 526 | * ARCv2 && no IOC -> call __dc_line_op; call __slc_rgn_op |
| 527 | * ARCv2 && IOC enabled -> nothing |
| 528 | */ |
| 529 | if (!is_isa_arcv2() || !ioc_exists) |
Alexey Brodkin | db6ce23 | 2015-12-14 17:15:13 +0300 | [diff] [blame] | 530 | __dc_line_op(start, end - start, OP_FLUSH); |
| 531 | |
Eugeniy Paltsev | ea9f6f1 | 2018-03-21 15:58:55 +0300 | [diff] [blame] | 532 | if (is_isa_arcv2() && !ioc_exists) |
Eugeniy Paltsev | 41cada4 | 2018-01-16 19:20:26 +0300 | [diff] [blame] | 533 | __slc_rgn_op(start, end - start, OP_FLUSH); |
Alexey Brodkin | 2f16ac9 | 2014-02-04 12:56:14 +0400 | [diff] [blame] | 534 | } |
| 535 | |
| 536 | void flush_cache(unsigned long start, unsigned long size) |
| 537 | { |
| 538 | flush_dcache_range(start, start + size); |
| 539 | } |
Alexey Brodkin | 6eb15e5 | 2015-03-30 13:36:04 +0300 | [diff] [blame] | 540 | |
Eugeniy Paltsev | c27814b | 2018-03-21 15:58:50 +0300 | [diff] [blame] | 541 | /* |
| 542 | * As invalidate_dcache_all() is not used in generic U-Boot code and as we |
| 543 | * don't need it in arch/arc code alone (invalidate without flush) we implement |
| 544 | * flush_n_invalidate_dcache_all (flush and invalidate in 1 operation) because |
| 545 | * it's much safer. See [ NOTE 1 ] for more details. |
| 546 | */ |
| 547 | void flush_n_invalidate_dcache_all(void) |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 548 | { |
Eugeniy Paltsev | c27814b | 2018-03-21 15:58:50 +0300 | [diff] [blame] | 549 | __dc_entire_op(OP_FLUSH_N_INV); |
Alexey Brodkin | db6ce23 | 2015-12-14 17:15:13 +0300 | [diff] [blame] | 550 | |
Eugeniy Paltsev | ea9f6f1 | 2018-03-21 15:58:55 +0300 | [diff] [blame] | 551 | if (is_isa_arcv2()) |
Eugeniy Paltsev | c27814b | 2018-03-21 15:58:50 +0300 | [diff] [blame] | 552 | __slc_entire_op(OP_FLUSH_N_INV); |
Alexey Brodkin | 6eb15e5 | 2015-03-30 13:36:04 +0300 | [diff] [blame] | 553 | } |
| 554 | |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 555 | void flush_dcache_all(void) |
Alexey Brodkin | 6eb15e5 | 2015-03-30 13:36:04 +0300 | [diff] [blame] | 556 | { |
Alexey Brodkin | 2a8382c | 2016-04-16 15:28:30 +0300 | [diff] [blame] | 557 | __dc_entire_op(OP_FLUSH); |
Alexey Brodkin | db6ce23 | 2015-12-14 17:15:13 +0300 | [diff] [blame] | 558 | |
Eugeniy Paltsev | ea9f6f1 | 2018-03-21 15:58:55 +0300 | [diff] [blame] | 559 | if (is_isa_arcv2()) |
Alexey Brodkin | ef639e6 | 2015-05-18 16:56:26 +0300 | [diff] [blame] | 560 | __slc_entire_op(OP_FLUSH); |
Alexey Brodkin | 6eb15e5 | 2015-03-30 13:36:04 +0300 | [diff] [blame] | 561 | } |