diff --git a/Source/Core/VideoBackends/D3D/NativeVertexFormat.cpp b/Source/Core/VideoBackends/D3D/NativeVertexFormat.cpp
index 70dfe23df0..d70d128415 100644
--- a/Source/Core/VideoBackends/D3D/NativeVertexFormat.cpp
+++ b/Source/Core/VideoBackends/D3D/NativeVertexFormat.cpp
@@ -88,12 +88,13 @@ void D3DVertexFormat::Initialize(const PortableVertexDeclaration &_vtx_decl)
 
 	for (int i = 0; i < 2; i++)
 	{
-		if (_vtx_decl.color_offset[i] > 0)
+		format = &_vtx_decl.colors[i];
+		if (format->enable)
 		{
 			m_elems[m_num_elems].SemanticName = "COLOR";
 			m_elems[m_num_elems].SemanticIndex = i;
-			m_elems[m_num_elems].AlignedByteOffset = _vtx_decl.color_offset[i];
-			m_elems[m_num_elems].Format = VarToD3D(_vtx_decl.color_gl_type, 4, false);
+			m_elems[m_num_elems].AlignedByteOffset = format->offset;
+			m_elems[m_num_elems].Format = VarToD3D(format->type, format->components, format->integer);
 			m_elems[m_num_elems].InputSlotClass = D3D11_INPUT_PER_VERTEX_DATA;
 			++m_num_elems;
 		}
diff --git a/Source/Core/VideoBackends/OGL/NativeVertexFormat.cpp b/Source/Core/VideoBackends/OGL/NativeVertexFormat.cpp
index 88147aa755..784191c1d2 100644
--- a/Source/Core/VideoBackends/OGL/NativeVertexFormat.cpp
+++ b/Source/Core/VideoBackends/OGL/NativeVertexFormat.cpp
@@ -79,10 +79,7 @@ void GLVertexFormat::Initialize(const PortableVertexDeclaration &_vtx_decl)
 	}
 
 	for (int i = 0; i < 2; i++) {
-		if (vtx_decl.color_offset[i] != -1) {
-			glEnableVertexAttribArray(SHADER_COLOR0_ATTRIB+i);
-			glVertexAttribPointer(SHADER_COLOR0_ATTRIB+i, 4, GL_UNSIGNED_BYTE, GL_TRUE, vtx_decl.stride, (u8*)NULL + vtx_decl.color_offset[i]);
-		}
+		SetPointer(SHADER_COLOR0_ATTRIB+i, vertex_stride, vtx_decl.colors[i]);
 	}
 
 	for (int i = 0; i < 8; i++) {
diff --git a/Source/Core/VideoCommon/NativeVertexFormat.h b/Source/Core/VideoCommon/NativeVertexFormat.h
index 9e389a2d64..ae1b8fc763 100644
--- a/Source/Core/VideoCommon/NativeVertexFormat.h
+++ b/Source/Core/VideoCommon/NativeVertexFormat.h
@@ -87,9 +87,8 @@ struct PortableVertexDeclaration
 
 	AttributeFormat position;
 	AttributeFormat normals[3];
+	AttributeFormat colors[2];
 
-	VarType color_gl_type;  // always GL_UNSIGNED_BYTE
-	int color_offset[2];
 	VarType texcoord_gl_type[8];
 	//int texcoord_gl_size[8];
 	int texcoord_offset[8];
diff --git a/Source/Core/VideoCommon/VertexLoader.cpp b/Source/Core/VideoCommon/VertexLoader.cpp
index 42fa6a7b0c..8251f76bf1 100644
--- a/Source/Core/VideoCommon/VertexLoader.cpp
+++ b/Source/Core/VideoCommon/VertexLoader.cpp
@@ -627,17 +627,14 @@ void VertexLoader::CompileVertexTranslator()
 			components |= VB_HAS_NRM1 | VB_HAS_NRM2;
 	}
 
-	vtx_decl.color_gl_type = VAR_UNSIGNED_BYTE;
-	vtx_decl.color_offset[0] = -1;
-	vtx_decl.color_offset[1] = -1;
 	for (int i = 0; i < 2; i++)
 	{
-		components |= VB_HAS_COL0 << i;
+		vtx_decl.colors[i].components = 4;
+		vtx_decl.colors[i].type = VAR_UNSIGNED_BYTE;
+		vtx_decl.colors[i].integer = false;
 		switch (col[i])
 		{
 		case NOT_PRESENT:
-			components &= ~(VB_HAS_COL0 << i);
-			vtx_decl.color_offset[i] = -1;
 			break;
 		case DIRECT:
 			switch (m_VtxAttr.color[i].Comp)
@@ -681,7 +678,9 @@ void VertexLoader::CompileVertexTranslator()
 		// Common for the three bottom cases
 		if (col[i] != NOT_PRESENT)
 		{
-			vtx_decl.color_offset[i] = nat_offset;
+			components |= VB_HAS_COL0 << i;
+			vtx_decl.colors[i].offset = nat_offset;
+			vtx_decl.colors[i].enable = true;
 			nat_offset += 4;
 		}
 	}