From 5110a5d25c055a838579be7a81bd7bee5ee36d9c Mon Sep 17 00:00:00 2001 From: Yoyobuae Date: Sun, 21 Apr 2024 11:50:30 -0500 Subject: [PATCH 1/2] fix(client_core) Fix FFE shader for PhoneVR --- alvr/client_core/cpp/ffr.cpp | 143 ++++++++++++++++++++++------------- 1 file changed, 92 insertions(+), 51 deletions(-) diff --git a/alvr/client_core/cpp/ffr.cpp b/alvr/client_core/cpp/ffr.cpp index b8179fc867..af8c80551a 100644 --- a/alvr/client_core/cpp/ffr.cpp +++ b/alvr/client_core/cpp/ffr.cpp @@ -5,6 +5,8 @@ #include "utils.h" +#include + using namespace std; using namespace gl_render_utils; @@ -12,13 +14,23 @@ namespace { const string FFR_COMMON_SHADER_FORMAT = R"glsl(#version 300 es precision highp float; - const uvec2 TARGET_RESOLUTION = uvec2(%u, %u); - const uvec2 OPTIMIZED_RESOLUTION = uvec2(%u, %u); const vec2 EYE_SIZE_RATIO = vec2(%f, %f); - const vec2 CENTER_SIZE = vec2(%f, %f); - const vec2 CENTER_SHIFT = vec2(%f, %f); const vec2 EDGE_RATIO = vec2(%f, %f); + const vec2 c1 = vec2(%f, %f); + const vec2 c2 = vec2(%f, %f); + const vec2 loBound = vec2(%f, %f); + const vec2 hiBound = vec2(%f, %f); + const vec2 loBoundC = vec2(%f, %f); + const vec2 hiBoundC = vec2(%f, %f); + + const vec2 aleft = vec2(%f, %f); + const vec2 bleft = vec2(%f, %f); + + const vec2 aright = vec2(%f, %f); + const vec2 bright = vec2(%f, %f); + const vec2 cright = vec2(%f, %f); + vec2 TextureToEyeUV(vec2 textureUV, bool isRightEye) { // flip distortion horizontally for right eye // left: x * 2; right: (1 - x) * 2 @@ -31,50 +43,49 @@ const string FFR_COMMON_SHADER_FORMAT = R"glsl(#version 300 es } )glsl"; +// Fragment shader which reverses the FFE compressed image on client side. +// Essentially it implements this function, for both the X and Y axis (or U and V in texture): +// https://www.desmos.com/calculator/cmvjr7ljje +// +// The function is the same for each axis. The code refers to the "left" and "right" edges, but +// that kinda only refers to the left/right sides of the function. The actual edge being calculated +// in the image might not be the left and right ones (might be top and bottom instead). Also for +// the right eye the UVs are mirrored so the "left" edge is actually the right one. const string DECOMPRESS_AXIS_ALIGNED_FRAGMENT_SHADER = R"glsl( uniform sampler2D tex0; in vec2 uv; out vec4 color; void main() { + // Source UV spans across both eyes, so first transform + // the UV coordinates to be per-eye. bool isRightEye = uv.x > 0.5; vec2 eyeUV = TextureToEyeUV(uv, isRightEye); - vec2 c0 = (1. - CENTER_SIZE) * 0.5; - vec2 c1 = (EDGE_RATIO - 1.) * c0 * (CENTER_SHIFT + 1.) / EDGE_RATIO; - vec2 c2 = (EDGE_RATIO - 1.) * CENTER_SIZE + 1.; - - vec2 loBound = c0 * (CENTER_SHIFT + 1.); - vec2 hiBound = c0 * (CENTER_SHIFT - 1.) + 1.; - vec2 underBound = vec2(eyeUV.x < loBound.x, eyeUV.y < loBound.y); - vec2 inBound = vec2(loBound.x < eyeUV.x && eyeUV.x < hiBound.x, - loBound.y < eyeUV.y && eyeUV.y < hiBound.y); - vec2 overBound = vec2(eyeUV.x > hiBound.x, eyeUV.y > hiBound.y); - + // Now calculate the uncompressed UVs for the various regions of the image. + // There's three regions to consider: the "left", the middle and the "right" vec2 center = (eyeUV - c1) * EDGE_RATIO / c2; - - vec2 loBoundC = c0 * (CENTER_SHIFT + 1.) / c2; - vec2 hiBoundC = c0 * (CENTER_SHIFT - 1.) / c2 + 1.; - - vec2 leftEdge = (-(c1 + c2 * loBoundC) / loBoundC + - sqrt(((c1 + c2 * loBoundC) / loBoundC) * ((c1 + c2 * loBoundC) / loBoundC) + - 4. * c2 * (1. - EDGE_RATIO) / (EDGE_RATIO * loBoundC) * eyeUV)) / - (2. * c2 * (1. - EDGE_RATIO)) * (EDGE_RATIO * loBoundC); - vec2 rightEdge = - (-(c2 - EDGE_RATIO * c1 - 2. * EDGE_RATIO * c2 + c2 * EDGE_RATIO * (1. - hiBoundC) + - EDGE_RATIO) / - (EDGE_RATIO * (1. - hiBoundC)) + - sqrt(((c2 - EDGE_RATIO * c1 - 2. * EDGE_RATIO * c2 + c2 * EDGE_RATIO * (1. - hiBoundC) + - EDGE_RATIO) / - (EDGE_RATIO * (1. - hiBoundC))) * - ((c2 - EDGE_RATIO * c1 - 2. * EDGE_RATIO * c2 + - c2 * EDGE_RATIO * (1. - hiBoundC) + EDGE_RATIO) / - (EDGE_RATIO * (1. - hiBoundC))) - - 4. * ((c2 * EDGE_RATIO - c2) * (c1 - hiBoundC + hiBoundC * c2) / - (EDGE_RATIO * (1. - hiBoundC) * (1. - hiBoundC)) - - eyeUV * (c2 * EDGE_RATIO - c2) / (EDGE_RATIO * (1. - hiBoundC))))) / - (2. * c2 * (EDGE_RATIO - 1.)) * (EDGE_RATIO * (1. - hiBoundC)); - - vec2 uncompressedUV = underBound * leftEdge + inBound * center + overBound * rightEdge; + vec2 leftEdge = (-bleft + sqrt(bleft * bleft + 4. * aleft * eyeUV)) / + (2. * aleft); + vec2 rightEdge = (-bright + sqrt(bright * bright - 4. * (cright - aright * eyeUV))) / (2. * aright); + + // Now figure out which UV coordinates to actually output depending on which + // UV region is being processed. Do each axis separately to cover all the nine + // possible combinations. + vec2 uncompressedUV = vec2(0., 0.); + + if (eyeUV.x < loBound.x) + uncompressedUV.x = leftEdge.x; + else if (eyeUV.x > hiBound.x) + uncompressedUV.x = rightEdge.x; + else + uncompressedUV.x = center.x; + + if (eyeUV.y < loBound.y) + uncompressedUV.y = leftEdge.y; + else if (eyeUV.y > hiBound.y) + uncompressedUV.y = rightEdge.y; + else + uncompressedUV.y = center.y; color = texture(tex0, EyeToTextureUV(uncompressedUV * EYE_SIZE_RATIO, isRightEye)); } @@ -138,19 +149,49 @@ FoveationVars CalculateFoveationVars(FFRData data) { FFR::FFR(Texture *inputSurface) : mInputSurface(inputSurface) {} void FFR::Initialize(FoveationVars fv) { + using glm::vec2; + + // Precalculate a bunch of constants that will be used in fragment shader + auto CENTER_SIZE = vec2(fv.centerSizeY, fv.centerSizeY); // Size of the center, non-distorted, region + auto CENTER_SHIFT = vec2(fv.centerShiftX, fv.centerShiftY); // How much to shift the center region + auto EDGE_RATIO = vec2(fv.edgeRatioX, fv.edgeRatioY); // Ratio of edge region VS center region + + auto c0 = (vec2(1., 1.) - CENTER_SIZE) * vec2(0.5, 0.5); + auto c1 = (EDGE_RATIO - vec2(1., 1.)) * c0 * (CENTER_SHIFT + vec2(1., 1.)) / EDGE_RATIO; + auto c2 = (EDGE_RATIO - vec2(1., 1.)) * CENTER_SIZE + vec2(1., 1.); + + auto loBound = c0 * (CENTER_SHIFT + vec2(1., 1.)); // Lower bound bellow which "left" edge begins + auto hiBound = c0 * (CENTER_SHIFT - vec2(1., 1.)) + vec2(1., 1.); // Upper bound above which "right" edge begins + auto loBoundC = c0 * (CENTER_SHIFT + vec2(1., 1.)) / c2; // Same as loBound but rescaled for distorted image + auto hiBoundC = c0 * (CENTER_SHIFT - vec2(1., 1.)) / c2 + vec2(1., 1.); // Same as hiBound but rescaled for distorted image + + // Constants for function: + // leftEdge(x) = (-bleft + sqrt(bleft^2 + 4 * aleft * x)) / (2 * aleft) + auto aleft = c2 * (vec2(1., 1.) - EDGE_RATIO) / (EDGE_RATIO * loBoundC); + auto bleft = (c1 + c2 * loBoundC) / loBoundC; + + // Constants for function: + // rightEdge(x) = (-bright + sqrt(bright^2 + 4 * (cright - aright * x)) / (2 * aright) + auto aright = c2 * (EDGE_RATIO - vec2(1., 1.)) / (EDGE_RATIO * (vec2(1., 1.) - hiBoundC)); + auto bright = (c2 - EDGE_RATIO * c1 - vec2(2., 2.) * EDGE_RATIO * c2 + c2 * EDGE_RATIO * (vec2(1., 1.) - hiBoundC) + EDGE_RATIO) / (EDGE_RATIO * (vec2(1., 1.) - hiBoundC)); + auto cright = ((c2 * EDGE_RATIO - c2) * (c1 - hiBoundC + c2 * hiBoundC)) / (EDGE_RATIO * (vec2(1., 1.) - hiBoundC) * (vec2(1., 1.) - hiBoundC)); + + // Put all the constants into the shader auto ffrCommonShaderStr = string_format(FFR_COMMON_SHADER_FORMAT, - fv.targetEyeWidth, - fv.targetEyeHeight, - fv.optimizedEyeWidth, - fv.optimizedEyeHeight, - fv.eyeWidthRatio, - fv.eyeHeightRatio, - fv.centerSizeX, - fv.centerSizeY, - fv.centerShiftX, - fv.centerShiftY, - fv.edgeRatioX, - fv.edgeRatioY); + fv.eyeWidthRatio, fv.eyeHeightRatio, + fv.edgeRatioX, fv.edgeRatioY, + c1.x, c1.y, + c2.x, c2.y, + loBound.x, loBound.y, + hiBound.x, hiBound.y, + loBoundC.x, loBoundC.y, + hiBoundC.x, hiBoundC.y, + aleft.x, aleft.y, + bleft.x, bleft.y, + aright.x, aright.y, + bright.x, bright.y, + cright.x, cright.y + ); mExpandedTexture.reset(new Texture(false, 0, false, fv.targetEyeWidth * 2, fv.targetEyeHeight)); mExpandedTextureState = make_unique(mExpandedTexture.get()); From 4919e7ce8be4a366b45c8414f5e7e32a7f10e6ea Mon Sep 17 00:00:00 2001 From: Yoyobuae Date: Wed, 24 Apr 2024 11:41:53 -0500 Subject: [PATCH 2/2] fix(client_core) Fix typo from previous commit --- alvr/client_core/cpp/ffr.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alvr/client_core/cpp/ffr.cpp b/alvr/client_core/cpp/ffr.cpp index af8c80551a..e724845cf3 100644 --- a/alvr/client_core/cpp/ffr.cpp +++ b/alvr/client_core/cpp/ffr.cpp @@ -152,7 +152,7 @@ void FFR::Initialize(FoveationVars fv) { using glm::vec2; // Precalculate a bunch of constants that will be used in fragment shader - auto CENTER_SIZE = vec2(fv.centerSizeY, fv.centerSizeY); // Size of the center, non-distorted, region + auto CENTER_SIZE = vec2(fv.centerSizeX, fv.centerSizeY); // Size of the center, non-distorted region auto CENTER_SHIFT = vec2(fv.centerShiftX, fv.centerShiftY); // How much to shift the center region auto EDGE_RATIO = vec2(fv.edgeRatioX, fv.edgeRatioY); // Ratio of edge region VS center region