diff --git a/bin/resources/shaders/opengl/common_header.glsl b/bin/resources/shaders/opengl/common_header.glsl
index 46416a35f3..146dac72b1 100644
--- a/bin/resources/shaders/opengl/common_header.glsl
+++ b/bin/resources/shaders/opengl/common_header.glsl
@@ -47,8 +47,6 @@ out gl_PerVertex {
 #ifdef FRAGMENT_SHADER
 layout(std140, binding = 15) uniform cb15
 {
-    ivec4 ScalingFactor;
-
     int EMODA;
     int EMODC;
     ivec2 pad_cb15;
diff --git a/bin/resources/shaders/opengl/convert.glsl b/bin/resources/shaders/opengl/convert.glsl
index ffd152a43e..a7858be8fa 100644
--- a/bin/resources/shaders/opengl/convert.glsl
+++ b/bin/resources/shaders/opengl/convert.glsl
@@ -226,9 +226,9 @@ void ps_convert_rgba_8i()
     int txN  = tb.x | (int(gl_FragCoord.x) & 7);
     int txH  = tb.x | ((int(gl_FragCoord.x) + 4) & 7);
 
-    txN *= ScalingFactor.x;
-    txH *= ScalingFactor.x;
-    ty  *= ScalingFactor.y;
+    txN *= PS_SCALE_FACTOR;
+    txH *= PS_SCALE_FACTOR;
+    ty  *= PS_SCALE_FACTOR;
 
     // TODO investigate texture gather
     vec4 cN = texelFetch(TextureSampler, ivec2(txN, ty), 0);
diff --git a/bin/resources/shaders/opengl/tfx_fs.glsl b/bin/resources/shaders/opengl/tfx_fs.glsl
index 554820afa1..93e5f7b16b 100644
--- a/bin/resources/shaders/opengl/tfx_fs.glsl
+++ b/bin/resources/shaders/opengl/tfx_fs.glsl
@@ -282,7 +282,7 @@ ivec2 clamp_wrap_uv_depth(ivec2 uv)
 
 vec4 sample_depth(vec2 st)
 {
-    vec2 uv_f = vec2(clamp_wrap_uv_depth(ivec2(st))) * vec2(ScalingFactor.xy) * vec2(1.0f/16.0f);
+    vec2 uv_f = vec2(clamp_wrap_uv_depth(ivec2(st))) * vec2(float(PS_SCALE_FACTOR)) * vec2(1.0f/16.0f);
     ivec2 uv = ivec2(uv_f);
 
     vec4 t = vec4(0.0f);
@@ -619,7 +619,7 @@ void ps_dither(inout vec3 C)
     #if PS_DITHER == 2
     ivec2 fpos = ivec2(gl_FragCoord.xy);
     #else
-    ivec2 fpos = ivec2(gl_FragCoord.xy / ScalingFactor.x);
+    ivec2 fpos = ivec2(gl_FragCoord.xy / float(PS_SCALE_FACTOR));
     #endif
     C += DitherMatrix[fpos.y&3][fpos.x&3];
 #endif
diff --git a/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp b/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp
index 3ebd4938d0..4710dcbe6d 100644
--- a/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp
+++ b/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp
@@ -67,6 +67,7 @@ GSDeviceOGL::GSDeviceOGL()
 	GLState::Clear();
 
 	m_mipmap = theApp.GetConfigI("mipmap");
+	m_upscale_multiplier = std::max(1, theApp.GetConfigI("upscale_multiplier"));
 	if (theApp.GetConfigB("UserHacks"))
 		m_filter = static_cast<TriFiltering>(theApp.GetConfigI("UserHacks_TriFilter"));
 	else
@@ -381,11 +382,6 @@ bool GSDeviceOGL::Create(const WindowInfo& wi)
 		GL_PUSH("GSDeviceOGL::Convert");
 
 		m_convert.cb = new GSUniformBufferOGL("Misc UBO", g_convert_index, sizeof(MiscConstantBuffer));
-		// Upload once and forget about it.
-		// Use value of 1 when upscale multiplier is 0 for ScalingFactor,
-		// this is to avoid doing math with 0 in shader. It helps custom res be less broken.
-		m_misc_cb_cache.ScalingFactor = GSVector4i(std::max(1, theApp.GetConfigI("upscale_multiplier")));
-		m_convert.cb->cache_upload(&m_misc_cb_cache);
 
 		const auto shader = Host::ReadResourceFileToString("shaders/opengl/convert.glsl");
 		if (!shader.has_value())
@@ -397,7 +393,10 @@ bool GSDeviceOGL::Create(const WindowInfo& wi)
 		for (size_t i = 0; i < std::size(m_convert.ps); i++)
 		{
 			const char* name = shaderName(static_cast<ShaderConvert>(i));
-			ps = m_shader->Compile("convert.glsl", name, GL_FRAGMENT_SHADER, m_shader_common_header, shader->c_str());
+			const std::string macro_sel = (static_cast<ShaderConvert>(i) == ShaderConvert::RGBA_TO_8I) ?
+                                              format("#define PS_SCALE_FACTOR %d\n", m_upscale_multiplier) :
+                                              std::string();
+			ps = m_shader->Compile("convert.glsl", name, GL_FRAGMENT_SHADER, m_shader_common_header, shader->c_str(), macro_sel);
 			std::string pretty_name = std::string("Convert pipe ") + name;
 			m_convert.ps[i] = m_shader->LinkPipeline(pretty_name, vs, 0, ps);
 		}
@@ -1009,6 +1008,7 @@ GLuint GSDeviceOGL::CompilePS(PSSelector sel)
 		+ format("#define PS_DITHER %d\n", sel.dither)
 		+ format("#define PS_ZCLAMP %d\n", sel.zclamp)
 		+ format("#define PS_PABE %d\n", sel.pabe)
+		+ format("#define PS_SCALE_FACTOR %d\n", m_upscale_multiplier)
 	;
 
 	if (GLLoader::buggy_sso_dual_src)
diff --git a/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.h b/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.h
index e2085c252f..79df0fc53f 100644
--- a/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.h
+++ b/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.h
@@ -183,7 +183,6 @@ public:
 
 	struct alignas(32) MiscConstantBuffer
 	{
-		GSVector4i ScalingFactor;
 		GSVector4i EMOD_AC;
 
 		MiscConstantBuffer() { memset(this, 0, sizeof(*this)); }
@@ -195,6 +194,7 @@ public:
 private:
 	std::unique_ptr<GL::Context> m_gl_context;
 	int m_mipmap;
+	int m_upscale_multiplier;
 	TriFiltering m_filter;
 
 	static bool m_debug_gl_call;