fix: general cylinder inside rendering

viewer/packages/rendering/src/glsl/sector/primitives/generalCylinder.frag

@@ -51,15 +51,15 @@ void main()
     vec3 rayTarget = v_viewPos;
     vec3 rayDirection = normalize(rayTarget); // rayOrigin is (0,0,0) in camera space
 
-    vec3 diff = rayTarget - v_centerB;
+    vec3 diff = - v_centerB;
     vec3 E = diff * v_modelBasis;
     vec3 D = rayDirection * v_modelBasis;
 
     float a = dot(D.xy, D.xy);
     float b = dot(E.xy, D.xy);
     float c = dot(E.xy, E.xy) - v_radius*v_radius;
 
-    // Calculate a dicriminant of the above quadratic equation
+    // Calculate a discriminant of the above quadratic equation
     float d = b*b - a*c;
 
     // d < 0.0 means the ray hits outside an infinitely long cone
@@ -76,36 +76,40 @@ void main()
     theta += theta < v_angles[0] ? 2.0 * PI : 0.0;
 
     // Intersection point in camera space
-    vec3 p = rayTarget + dist * rayDirection;
+    vec3 p = dist * rayDirection;
 
     vec3 planeACenter = vec3(0.0, 0.0, v_planeA.w);
     vec3 planeANormal = v_planeA.xyz;
     vec3 planeBCenter = vec3(0.0, 0.0, v_planeB.w);
     vec3 planeBNormal = v_planeB.xyz;
 
+    float normalFactor = 1.0;
+
     if (dot(intersectionPoint - planeACenter, planeANormal) > 0.0 ||
         dot(intersectionPoint - planeBCenter, planeBNormal) > 0.0 ||
         theta > v_angles[1] + v_angles[0] ||
-        isClipped(appearance, p)
+        isClipped(appearance, p) ||
+        dist < 0.0
        ) {
         // Missed the first point, check the other point
         dist = max(dist1, dist2);
         intersectionPoint = E + dist * D;
         theta = atan(intersectionPoint.y, intersectionPoint.x);
         theta += theta < v_angles[0] ? 2.0 * PI : 0.0;
-        p = rayTarget + dist*rayDirection;
+        p = dist * rayDirection;
         if (dot(intersectionPoint - planeACenter, planeANormal) > 0.0 ||
             dot(intersectionPoint - planeBCenter, planeBNormal) > 0.0 ||
-            theta > v_angles[1] + v_angles[0] || isClipped(appearance, p)
+            theta > v_angles[1] + v_angles[0] || isClipped(appearance, p) ||
+            dist < 0.0
            ) {
             // Missed the other point too
             discard;
         }
+        normalFactor = -1.0;
     }
 
-    //TODO - christjt 2022/10/10: This seems wrong when hitting inner surface
     vec3 p_local = p - v_centerB;
-    vec3 normal = normalize(p_local - v_modelBasis[2] * dot(p_local, v_modelBasis[2]));
+    vec3 normal = normalize(p_local - v_modelBasis[2] * dot(p_local, v_modelBasis[2])) * normalFactor;
 
     float fragDepth = updateFragmentDepth(p, projectionMatrix);
     updateFragmentColor(renderMode, color, v_treeIndex, normal, fragDepth, matCapTexture, GeometryType.Primitive);

viewer/packages/rendering/src/glsl/sector/primitives/generalCylinder.vert

@@ -38,6 +38,20 @@ out float v_radius;
 
 out highp vec2 v_treeIndexPacked;
 
+
+bool isWithinSpan(vec3 point, vec3 span) {
+    return all(lessThan(abs(point), span));
+}
+
+vec3 transformQuadToCoverScreenInViewSpace(vec3 position, mat4 projectionMatrix, float near) {
+    float tanFov = 1.0 / projectionMatrix[1][1];
+
+    float aspect = projectionMatrix[1][1] / projectionMatrix[0][0];
+    float maxAspect = max(aspect, 1.0 / aspect);
+    vec3 fullScreenQuadCorner = vec3(position.xy * maxAspect * tanFov * near, -near - 1e-6);
+    return fullScreenQuadCorner;
+}
+
 void main() {
     v_treeIndexPacked = packTreeIndex(a_treeIndex);
 
@@ -67,10 +81,24 @@ void main() {
     vec3 left = normalize(cross(objectToCameraModelSpace, lDir));
     vec3 up = normalize(cross(left, lDir));
 
-    vec3 localBillboardPosition = center + mat3(halfHeight * lDir, a_radius * left, a_radius * up) * position;
+    mat3 billboardWorldRotation = mat3(lDir, left, up);
+    vec3 cylinderAxisScales = vec3(halfHeight, a_radius, a_radius);
+    mat3 inverseBillboardWorldRotation = transpose(billboardWorldRotation);
+    vec3 cameraPosInCylinderSpace = inverseBillboardWorldRotation * (rayOrigin - center);
+
+    mat3 billboardWorldScaleRotation = mat3(halfHeight * lDir, a_radius * left, a_radius * up);
+
+    vec3 localBillboardPosition = center + billboardWorldScaleRotation * position;
     vec3 viewBillboardPosition = mul3(modelToView, localBillboardPosition);
 
-    gl_Position = projectionMatrix * vec4(viewBillboardPosition, 1.0 );
+    float near = projectionMatrix[3][2] / (projectionMatrix[2][2] - 1.0);
+
+    // Check whether we are inside the primitive, in which case the quad must cover the entire screen
+    if (isWithinSpan(cameraPosInCylinderSpace, cylinderAxisScales + vec3(near))) {
+        viewBillboardPosition = transformQuadToCoverScreenInViewSpace(position, projectionMatrix, near);
+    }
+
+    gl_Position = projectionMatrix * vec4(viewBillboardPosition, 1.0);
 
     // varying data
     v_treeIndex = a_treeIndex;