diff --git a/arch/arm/mach-mvebu/cpu.c b/arch/arm/mach-mvebu/cpu.c
index 7335620..23c2a97 100644
--- a/arch/arm/mach-mvebu/cpu.c
+++ b/arch/arm/mach-mvebu/cpu.c
@@ -183,13 +183,25 @@
 int arch_cpu_init(void)
 {
 #ifndef CONFIG_SPL_BUILD
-	/*
-	 * Only with disabled MMU its possible to switch the base
-	 * register address on Armada 38x. Without this the SDRAM
-	 * located at >= 0x4000.0000 is also not accessible, as its
-	 * still locked to cache.
-	 */
-	mmu_disable();
+	if (mvebu_soc_family() == MVEBU_SOC_A38X) {
+		struct pl310_regs *const pl310 =
+			(struct pl310_regs *)CONFIG_SYS_PL310_BASE;
+
+		/*
+		 * Only with disabled MMU its possible to switch the base
+		 * register address on Armada 38x. Without this the SDRAM
+		 * located at >= 0x4000.0000 is also not accessible, as its
+		 * still locked to cache.
+		 *
+		 * So to fully release / unlock this area from cache, we need
+		 * to first flush all caches, then disable the MMU and
+		 * disable the L2 cache.
+		 */
+		icache_disable();
+		dcache_disable();
+		mmu_disable();
+		clrbits_le32(&pl310->pl310_ctrl, L2X0_CTRL_EN);
+	}
 #endif
 
 	/* Linux expects the internal registers to be at 0xf1000000 */