Fix dynarec crashes on 3DS

After the dynarec writes new instructions, it has to flush the
instruction and data caches. Some of these flush operations are
privileged on the 3DS, so the clear cache functions have to run
through svcBackdoor. The Nintendo implementation (and CFW
reimplementation) of svcBackdoor has a problem where interrupts and
context switches will cause crashes.

Even though we can disable interrupts in the flush function, there's
still a window of time between svcBackdoor being called and the
function being run where an interrupt will corrupt the stack.

Luma3DS implemements a svcCustomBackdoor call we can use that also
runs a function in supervisor mode, but uses an implementation that
avoids this problem.

4 files changed, 58 insertions, 4 deletions

diff --git a/Makefile.libretro b/Makefile.libretro
index 133ee6d..d9b528a 100644
--- a/Makefile.libretro
+++ b/Makefile.libretro
@@ -213,6 +213,8 @@ else ifeq ($(platform), ctr)
 	CFLAGS += -Ifrontend/3ds
 	CFLAGS += -Werror=implicit-function-declaration
 
+	OBJS += frontend/3ds/utils.o
+
 #	CFLAGS += -DPCSX
 	BUILTIN_GPU = unai
 	DYNAREC = ari64
diff --git a/frontend/3ds/3ds_utils.h b/frontend/3ds/3ds_utils.h
index 1f12b84..cad4cbe 100644
--- a/frontend/3ds/3ds_utils.h
+++ b/frontend/3ds/3ds_utils.h
@@ -8,6 +8,8 @@
 #define MEMOP_MAP       4
 #define MEMOP_UNMAP     5
 
+#define GET_VERSION_MAJOR(version)    ((version) >>24)
+
 void* linearMemAlign(size_t size, size_t alignment);
 void linearFree(void* mem);
 
@@ -21,6 +23,7 @@ int32_t threadJoin(int32_t thread, int64_t timeout_ns);
 void threadFree(int32_t thread);
 void threadExit(int32_t rc)  __attribute__((noreturn));
 
+int32_t svcGetSystemInfo(int64_t* out, uint32_t type, int32_t param);
 int32_t svcBackdoor(int32_t (*callback)(void));
 
 #define DEBUG_HOLD() do{printf("%s@%s:%d.\n",__FUNCTION__, __FILE__, __LINE__);fflush(stdout);wait_for_input();}while(0)
@@ -29,6 +32,24 @@ void wait_for_input(void);
 
 extern __attribute__((weak)) int  __ctr_svchax;
 
+bool has_rosalina;
+
+static void check_rosalina() {
+  int64_t version;
+  uint32_t major;
+
+  has_rosalina = false;
+
+  if (!svcGetSystemInfo(&version, 0x10000, 0)) {
+     major = GET_VERSION_MAJOR(version);
+
+     if (major >= 8)
+       has_rosalina = true;
+  }
+}
+
+void ctr_clear_cache(void);
+
 typedef int32_t (*ctr_callback_type)(void);
 
 static inline void ctr_invalidate_ICache_kernel(void)
@@ -57,12 +78,14 @@ static inline void ctr_flush_DCache(void)
    svcBackdoor((ctr_callback_type)ctr_flush_DCache_kernel);
 }
 
-
 static inline void ctr_flush_invalidate_cache(void)
 {
-   ctr_flush_DCache();
-   ctr_invalidate_ICache();
+   if (has_rosalina) {
+      ctr_clear_cache();
+   } else {
+      ctr_flush_DCache();
+      ctr_invalidate_ICache();
+   }
 }
 
-
 #endif // _3DS_UTILS_H
diff --git a/frontend/3ds/utils.S b/frontend/3ds/utils.S
new file mode 100644
index 0000000..c8df651
--- /dev/null
+++ b/frontend/3ds/utils.S
@@ -0,0 +1,25 @@
+  .text
+  .arm
+  .balign 4
+
+  .func ctr_clear_cache_kernel
+ctr_clear_cache_kernel:
+  cpsid aif
+  mov r0, #0
+  mcr p15, 0, r0, c7, c10, 0    @ Clean entire data cache
+  mcr p15, 0, r0, c7, c10, 5    @ Data Memory Barrier
+  mcr p15, 0, r0, c7, c5, 0     @ Invalidate entire instruction cache / Flush BTB
+  mcr p15, 0, r0, c7, c10, 4    @ Data Sync Barrier
+  bx lr
+  .endfunc
+
+  @@ Clear the entire data cache / invalidate the instruction cache. Uses
+  @@ Rosalina svcCustomBackdoor to avoid svcBackdoor stack corruption
+  @@ during interrupts.
+  .global ctr_clear_cache
+  .func ctr_clear_cache
+ctr_clear_cache:
+  ldr r0, =ctr_clear_cache_kernel
+  svc 0x80                      @ svcCustomBackdoor
+  bx lr
+  .endfunc
diff --git a/libpcsxcore/new_dynarec/new_dynarec.c b/libpcsxcore/new_dynarec/new_dynarec.c
index dfa17a7..bb6ff0b 100644
--- a/libpcsxcore/new_dynarec/new_dynarec.c
+++ b/libpcsxcore/new_dynarec/new_dynarec.c
@@ -7085,6 +7085,10 @@ void new_dynarec_init(void)
 {
   SysPrintf("Init new dynarec\n");
 
+#ifdef _3DS
+  check_rosalina();
+#endif
+
   // allocate/prepare a buffer for translation cache
   // see assem_arm.h for some explanation
 #if   defined(BASE_ADDR_FIXED)