backend: gl: use effective texture size for rounded corner calculation

The rounded corner calculation only took the actual texture dimensions
into account, effectively turning all pixels outside completely
transparent. Furthermore, `texelFetch()` in the default window shader
does not work with any `GL_TEXTURE_WRAP_x` parameter.

As a result, a to-be-tiled root pixmap wasn't properly tiled and only
visible in the original top-left corner.

- Changed the default window shader to use `textureSize()` to scale the
  texture coordinates into the proper range for use with `texture2D()`.
- Added a new uniform to the default post-processing shader containing
  the effective texture size for rounded corners calculation instead of
  relying on `textureSize()`.

man/picom.1.asciidoc

@@ -387,6 +387,7 @@ uniform float corner_radius;  // corner radius of the window (pixels)
 uniform float border_width;   // estimated border width of the window (pixels)
 uniform bool invert_color;    // whether to invert the color of the window
 uniform sampler2D tex;        // texture of the window
+uniform vec2 effective_size;  // effective dimensions of the texture (repeats pixels if larger than tex)
 uniform sampler2D brightness; // estimated brightness of the window, 1x1 texture
 uniform float max_brightness; // configured maximum brightness of the window (0.0 - 1.0)
 uniform float time;           // time in milliseconds, counting from an unspecified starting point
@@ -412,7 +413,8 @@ vec4 default_post_processing(vec4 c);
 // 1) fetch the specified pixel
 // 2) apply default post-processing
 vec4 window_shader() {
-    vec4 c = texelFetch(tex, ivec2(texcoord), 0);
+    vec2 texsize = textureSize(tex, 0);
+    vec4 c = texture2D(tex, texcoord / texsize, 0);
     return default_post_processing(c);
 }
 ----

src/backend/gl/gl_common.c

@@ -385,6 +385,10 @@ static void _gl_compose(backend_t *base, struct backend_image *img, GLuint targe
 	if (win_shader->uniform_tex >= 0) {
 		glUniform1i(win_shader->uniform_tex, 0);
 	}
+	if (win_shader->uniform_effective_size >= 0) {
+		glUniform2f(win_shader->uniform_effective_size, (float)img->ewidth,
+		            (float)img->eheight);
+	}
 	if (win_shader->uniform_dim >= 0) {
 		glUniform1f(win_shader->uniform_dim, (float)img->dim);
 	}
@@ -596,6 +600,7 @@ static bool gl_win_shader_from_stringv(const char **vshader_strv,
 	bind_uniform(ret, opacity);
 	bind_uniform(ret, invert_color);
 	bind_uniform(ret, tex);
+	bind_uniform(ret, effective_size);
 	bind_uniform(ret, dim);
 	bind_uniform(ret, brightness);
 	bind_uniform(ret, max_brightness);

src/backend/gl/gl_common.h

@@ -35,6 +35,7 @@ typedef struct {
 	GLint uniform_opacity;
 	GLint uniform_invert_color;
 	GLint uniform_tex;
+	GLint uniform_effective_size;
 	GLint uniform_dim;
 	GLint uniform_brightness;
 	GLint uniform_max_brightness;

src/backend/gl/shaders.c

@@ -97,6 +97,7 @@ const char win_shader_glsl[] = GLSL(330,
 	uniform bool invert_color;
 	in vec2 texcoord;
 	uniform sampler2D tex;
+	uniform vec2 effective_size;
 	uniform sampler2D brightness;
 	uniform float max_brightness;
 	// Signed distance field for rectangle center at (0, 0), with size of
@@ -130,7 +131,7 @@ const char win_shader_glsl[] = GLSL(330,
 		// Using mix() to avoid a branch here.
 		vec4 rim_color = mix(c, border_color, clamp(border_width, 0.0f, 1.0f));
 
-		vec2 outer_size = vec2(textureSize(tex, 0));
+		vec2 outer_size = effective_size;
 		vec2 inner_size = outer_size - vec2(corner_radius) * 2.0f;
 		float rect_distance = rectangle_sdf(texcoord - outer_size / 2.0f,
 		    inner_size / 2.0f) - corner_radius;
@@ -157,7 +158,8 @@ const char win_shader_default[] = GLSL(330,
 	uniform sampler2D tex;
 	vec4 default_post_processing(vec4 c);
 	vec4 window_shader() {
-		vec4 c = texelFetch(tex, ivec2(texcoord), 0);
+		vec2 texsize = textureSize(tex, 0);
+		vec4 c = texture2D(tex, texcoord / texsize, 0);
 		return default_post_processing(c);
 	}
 );