diff --git a/CMakeModules/DuckStationUtils.cmake b/CMakeModules/DuckStationUtils.cmake
index bec071287c..9d7c5ad465 100644
--- a/CMakeModules/DuckStationUtils.cmake
+++ b/CMakeModules/DuckStationUtils.cmake
@@ -92,17 +92,22 @@ function(detect_architecture)
   elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "riscv64")
     message(STATUS "Building RISC-V 64 binaries.")
     set(CPU_ARCH_RISCV64 TRUE PARENT_SCOPE)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -finline-atomics" PARENT_SCOPE)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -finline-atomics" PARENT_SCOPE)
 
-    # Still need this, apparently.
-    link_libraries("-latomic")
+    # Don't want function calls for atomics.
+    if(COMPILER_GCC)
+      set(EXTRA_CFLAGS "${EXTRA_CFLAGS} -finline-atomics")
+
+      # Still need this, apparently.
+      link_libraries("-latomic")
+    endif()
 
     if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
       # Frame pointers generate an annoying amount of code on leaf functions.
-      set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer" PARENT_SCOPE)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fomit-frame-pointer" PARENT_SCOPE)
+      set(EXTRA_CFLAGS "${EXTRA_CFLAGS} -fomit-frame-pointer")
     endif()
+
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_CFLAGS}" PARENT_SCOPE)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CFLAGS}" PARENT_SCOPE)
   else()
     message(FATAL_ERROR "Unknown system processor: ${CMAKE_SYSTEM_PROCESSOR}")
   endif()