arm: mach-k3: common: Drop main r5 start

Only start-up the non-linux remote cores if we are running in legacy
boot mode. HSM rearch is not yet supporting this.

Signed-off-by: Dave Gerlach <d-gerlach@ti.com>
Signed-off-by: Tero Kristo <t-kristo@ti.com>
Signed-off-by: Tero Kristo <kristo@kernel.org>
diff --git a/arch/arm/mach-k3/common.c b/arch/arm/mach-k3/common.c
index 0a1638e..ab6d9bd 100644
--- a/arch/arm/mach-k3/common.c
+++ b/arch/arm/mach-k3/common.c
@@ -212,11 +212,12 @@
 		panic("rproc failed to be initialized (%d)\n", ret);
 
 	init_env();
-	start_non_linux_remote_cores();
-	if (!fit_image_info[IMAGE_ID_DM_FW].image_start)
+
+	if (!fit_image_info[IMAGE_ID_DM_FW].image_start) {
+		start_non_linux_remote_cores();
 		size = load_firmware("name_mcur5f0_0fw", "addr_mcur5f0_0load",
 				     &loadaddr);
-
+	}
 
 	/*
 	 * It is assumed that remoteproc device 1 is the corresponding