dolphin/Data/User/Shaders/asciiart.txt

uniform sampler2D samp8; // textures
uniform sampler2D samp9;

const int char_width = 8;
const int char_height = 13;
const int char_count = 95;
const int char_pixels = char_width*char_height;
const vec2 char_dim = vec2(char_width, char_height);
const vec2 font_scale = vec2(1.0/char_width/char_count, 1.0/char_height);

out vec4 ocol0;
in vec2 uv0;

uniform vec4 resolution;

void main()
{
  vec2 char_pos = floor(uv0*resolution.xy/char_dim);
  vec2 pixel_offset = floor(uv0*resolution.xy) - char_pos*char_dim;

  float mindiff = char_width*char_height*100; // just a big number
  float minc = 0.0;
  vec4 mina = vec4(0.0, 0.0, 0.0, 0.0);
  vec4 minb = vec4(0.0, 0.0, 0.0, 0.0);
  
  for(int i=0; i<char_count; i++) {
    vec4 ff = vec4(0.0, 0.0, 0.0, 0.0);
    vec4 f = vec4(0.0, 0.0, 0.0, 0.0);
    vec4 ft = vec4(0.0, 0.0, 0.0, 0.0);
    vec4 t = vec4(0.0, 0.0, 0.0, 0.0);
    vec4 tt = vec4(0.0, 0.0, 0.0, 0.0);
    
    for(int x=0; x<char_width; x++) {
      for(int y=0; y<char_height; y++) {
      
	vec2 tex_pos = char_pos*char_dim + vec2(x,y) + 0.5;
	vec4 tex = texture(samp9, tex_pos * resolution.zw);
      
	vec2 font_pos = vec2(x+i*char_width, y) + 0.5;
	vec4 font = texture(samp8, font_pos * font_scale);
	
	// generates sum of texture and font and their squares
	ff += font*font;
	f += font;
	ft += font*tex;
	t += tex;
	tt += tex*tex;
      }
    }
    
    /*
      The next lines are a bit harder, hf :-)
      
      The idea is to find the perfect char with the perfect background color and the perfect font color.
      As this is an equation with three unknowns, we can't just try all chars and color combinations.
      
      As criterion how "perfect" the selection is, we compare the "mean squared error" of the resulted colors of all chars.
      So, now the big issue: how to calculate the MSE without knowing the two colors ...
      
      In the next steps, "a" is the font color, "b" is the background color, "f" is the font value at this pixel, "t" is the texture value
      
      So the square error of one pixel is: 
        e = ( t - a⋅f - b⋅(1-f) ) ^ 2
        
      In longer:
        e = a^2⋅f^2 - 2⋅a⋅b⋅f^2 + 2⋅a⋅b⋅f - 2⋅a⋅f⋅t + b^2⋅f^2 - 2⋅b^2⋅f + b^2 + 2⋅b⋅f⋅t - 2⋅b⋅t + t^2
        
      The sum of all errors is: (as shortcut, ff,f,ft,t,tt are now the sums like declared above, sum(1) is the count of pixels)
        sum(e) = a^2⋅ff - 2⋅a^2⋅ff + 2⋅a⋅b⋅f - 2⋅a⋅ft + b^2⋅ff - 2⋅b^2⋅f + b^2⋅sum(1) + 2⋅b⋅ft - 2⋅b⋅t + tt
        
      To find the minimum, we have to derive this by "a" and "b":
        d/da sum(e) = 2⋅a⋅ff + 2⋅b⋅f - 2⋅b⋅ff - 2⋅ft
        d/db sum(e) = 2⋅a⋅f - 2⋅a⋅ff - 4⋅b⋅f + 2⋅b⋅ff + 2⋅b⋅sum(1) + 2⋅ft - 2⋅t
      
      So, both equations must be zero at minimum and there is only one solution.
    */
    vec4 a = (f*ft - ff*t + f*t - ft*char_pixels) / (f*f  - ff*char_pixels);
    vec4 b = (f*ft - ff*t)                        / (f*f  - ff*char_pixels);
    
    vec4 diff = a*a*ff + 2.0*a*b*f - 2.0*a*b*ff - 2.0*a*ft + b*b *(-2.0*f + ff + char_pixels) + 2.0*b*ft - 2.0*b*t + tt;
    float diff_f = dot(diff, vec4(1.0, 1.0, 1.0, 1.0));
    
    if(diff_f < mindiff) {
      mindiff = diff_f;
      minc = i;
      mina = a;
      minb = b;
    }
  }
  
  vec2 font_pos_res = vec2(minc*char_width, 0) + pixel_offset + 0.5;

  vec4 col = texture(samp8, font_pos_res * font_scale);
  ocol0 = mina * col + minb * (vec4(1.0,1.0,1.0,1.0) - col);
}