diff --git a/meson.build b/meson.build
index b562ec3512..9d0b487eb0 100644
--- a/meson.build
+++ b/meson.build
@@ -37,10 +37,6 @@ add_project_arguments(cppc.get_supported_arguments([
   '-Wno-missing-braces',
 ]), language: 'cpp')
 
-add_project_arguments(cppc.get_supported_arguments([
-  '-ffast-math',
-]), language: 'cpp')
-
 pipewire_dep = dependency('libpipewire-0.3', required: get_option('pipewire'))
 librt_dep = cppc.find_library('rt', required : get_option('pipewire'))
 hwdata_dep = dependency('hwdata', required : false)
diff --git a/src/Utils/Directives.h b/src/Utils/Directives.h
new file mode 100644
index 0000000000..8137e5152a
--- /dev/null
+++ b/src/Utils/Directives.h
@@ -0,0 +1,48 @@
+#pragma once
+
+#ifdef __x86__
+#include <xmmintrin.h>
+#define SET_FAST_MATH_FLAGS FlagSwitcher switcher{};
+
+struct FlagSwitcher {
+  unsigned int m_csr;
+  FlagSwitcher() : m_csr{_mm_getcsr()} {
+    _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON);
+    _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
+  }
+  ~FlagSwitcher() {
+    _mm_setcsr(m_csr);
+  }
+};
+#elif __aarch64__
+#define SET_FAST_MATH_FLAGS FlagSwitcher switcher{};
+
+static constexpr unsigned long long fz_bit = 0x1000000;
+//based on this stuff: https://github.com/DLTcollab/sse2neon/blob/706d3b58025364c2371cafcf9b16e32ff7e630ed/sse2neon.h#L2433
+struct FlagSwitcher {
+  unsigned long long m_csr;
+  FlagSwitcher() : m_csr{__builtin_aarch64_get_fpcr64()} {
+    __builtin_aarch64_set_fpcr64(fz_bit);
+  }
+  ~FlagSwitcher() {
+    __builtin_aarch64_set_fpcr64(m_csr | fz_bit);
+  }
+};
+
+#else
+#define SET_FAST_MATH_FLAGS
+
+#endif
+
+#ifdef __clang__
+#define FAST_MATH_ON _Pragma("float_control(push)");        \
+                     _Pragma("float_control(precise, off)") //https://clang.llvm.org/docs/LanguageExtensions.html#extensions-to-specify-floating-point-flags
+#define FAST_MATH_OFF _Pragma("float_control(pop)")
+#elif defined(__GNUC__)
+#define FAST_MATH_ON  _Pragma("GCC push_options");             \
+                      _Pragma("GCC optimize(\"-ffast-math\")")
+#define FAST_MATH_OFF _Pragma("GCC pop_options")
+#else
+#define FAST_MATH_ON
+#define FAST_MATH_OFF
+#endif
diff --git a/src/color_helpers.cpp b/src/color_helpers.cpp
index 2075eca463..b74d139f17 100644
--- a/src/color_helpers.cpp
+++ b/src/color_helpers.cpp
@@ -1,6 +1,8 @@
 #define COLOR_HELPERS_CPP
 #include "color_helpers_impl.h"
 
+FAST_MATH_ON
+
 #include <algorithm>
 #include <cstdint>
 #include <cmath>
@@ -214,7 +216,7 @@ inline void lerp_rgb(float* out, const float* a, const float* b, const float* c,
 
 inline float ClampAndSanitize( float a, float min, float max )
 {
-#ifndef __FAST_MATH__
+#if !( defined(__FAST_MATH__) || defined(__FINITE_MATH_ONLY__) )
     return std::isfinite( a ) ? std::min(std::max(min, a), max) : min;
 #else
     return std::min(std::max(min, a), max);
@@ -679,6 +681,8 @@ void calcColorTransform( lut1d_t * pShaper, int nLutSize1d,
     const colormapping_t & mapping, const nightmode_t & nightmode, const tonemapping_t & tonemapping,
     const lut3d_t * pLook, float flGain )
 {
+    SET_FAST_MATH_FLAGS
+
     // Generate shaper lut
     // Note: while this is typically a 1D approximation of our end to end transform,
     // it need not be! Conceptually this is just to determine the interpolation properties...
@@ -910,3 +914,5 @@ const glm::mat3 k_xyz_from_2020 = normalised_primary_matrix( displaycolorimetry_
 const glm::mat3 k_2020_from_xyz = glm::inverse( k_xyz_from_2020 );
 
 const glm::mat3 k_2020_from_709 = k_2020_from_xyz * k_xyz_from_709;
+
+FAST_MATH_OFF
\ No newline at end of file
diff --git a/src/color_helpers_impl.h b/src/color_helpers_impl.h
index 3e8c2d5a5e..22352a23ae 100644
--- a/src/color_helpers_impl.h
+++ b/src/color_helpers_impl.h
@@ -1,4 +1,8 @@
 #pragma once
+#include "Utils/Directives.h"
+
+FAST_MATH_ON
+
 #include "color_helpers.h"
 
 namespace rendervulkan {
@@ -17,4 +21,6 @@ namespace ns_color_tests {
 
 #ifdef COLOR_HELPERS_CPP
 REGISTER_LUT_EDGE_SIZE(rendervulkan::s_nLutEdgeSize3d);
-#endif
\ No newline at end of file
+#endif
+
+FAST_MATH_OFF
\ No newline at end of file
diff --git a/src/steamcompmgr.cpp b/src/steamcompmgr.cpp
index a8d959f78a..a146e908f1 100644
--- a/src/steamcompmgr.cpp
+++ b/src/steamcompmgr.cpp
@@ -5069,7 +5069,7 @@ steamcompmgr_latch_frame_done( steamcompmgr_win_t *w, uint64_t vblank_idx )
 
 static inline float santitize_float( float f )
 {
-#ifndef __FAST_MATH__
+#if !( defined(__FAST_MATH__) || defined(__FINITE_MATH_ONLY__) )
 	return ( std::isfinite( f ) ? f : 0.f );
 #else
 	return f;