"Raytraced primitives in unity3d (part 4): casting and receiving shadows"

Posted on 2016-04-12 in shaders

Alright. I expected this to take less time, but things happened.

Here's what I have right now.

All lights on

Point liights, Point light

spot lights, Spot Light

and directional lights Directional Light

are supported and cast shadows.

Getting there, however, was unexpectedly difficult.

Here's the code. It is in need of some refactoring, but it works.

RaycastObject.shader:

Shader "Unlit/RaycastObjectShader"{
    Properties  {
        _MainTex ("Texture", 2D) = "white" {}
        _Color ("Color", Color) = (1, 1, 1, 1)
        _SpecularColor ("Specular Color", Color) = (1, 1, 1, 1)
        _SpecularPower ("Specular power", Range(1.0, 100.0)) = 25.0
    }
    SubShader   {
        Tags { "RenderType"="Opaque" }
        LOD 100

        //basepass
        Pass{
            Tags {"LightMode" = "ForwardBase"}
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            #pragma multi_compile_fwdbase_fullshadows

            #define RAYCAST_USE_AMBIENT
            #define RAYCAST_USE_LIGHT
            #define RAYCAST_USE_SPECULAR
            #define RAYCAST_USE_ATTENUATION
            #include "SphereRender.cginc"
            ENDCG
        }

        //light pass
        Pass{
            Tags {"LightMode" = "ForwardAdd"}
            Blend One One
            ZWrite Off
            ZTest LEqual
            CGPROGRAM
            #pragma multi_compile_fwdadd_fullshadows
            #pragma vertex vert
            #pragma fragment frag

            #define RAYCAST_USE_LIGHT
            #define RAYCAST_USE_SPECULAR
            #define RAYCAST_USE_ATTENUATION
            #include "SphereRender.cginc"
            ENDCG
        }

        Pass{
            Tags {"LightMode" = "Shadowcaster"}
            ZWrite On
            ZTest LEqual
            CGPROGRAM
            #pragma multi_compile_shadowcaster
            #pragma vertex vert
            #pragma fragment frag

            #define RAYCAST_NO_TEXTURE
            #define RAYCAST_SHADOW_PASS
            #include "SphereRender.cginc"
            ENDCG
        }
    }
}

SphereRender.cginc:

#include "UnityCG.cginc"
#include "Raycast.cginc"
#include "LightAndShadow.cginc"

struct appdata{
    float4 vertex : POSITION;
};

struct v2f{
    float4 vertex : SV_POSITION;
    float3 rayDir: TEXCOORD0;
    float3 rayPos: TEXCOORD1;
};

sampler2D _MainTex;
float4 _MainTex_ST;
float4 _Color;
float4 _SpecularColor;
float _SpecularPower;

#if defined(RAYCAST_SHADOW_PASS)

#define RAYCAST_RAY_FROM_LIGHT_POS
#ifdef SHADOWS_CUBE
float4 calculateCubeShadowDepth(float3 worldPos){
    float3 diff = worldPos - _LightPositionRange.xyz;//_WorldSpaceLightPos0.xyz;
    float depth = (length(diff) + unity_LightShadowBias.x) * _LightPositionRange.w;
    return UnityEncodeCubeShadowDepth(depth);
}
#else
float calculateShadowDepth(float3 worldPos){
    float4 projPos = mul(UNITY_MATRIX_VP, float4(worldPos, 1));
    projPos = UnityApplyLinearShadowBias(projPos);
    return projPos.z/projPos.w;
}
#endif
#endif

float3 getRayToCamera(float3 worldPos){
    //return -UNITY_MATRIX_V[2].xyz;;
    if (unity_OrthoParams.w > 0){
        return -UNITY_MATRIX_V[2].xyz;;
    }
    else
        return worldPos  - _WorldSpaceCameraPos;
}

v2f vert (appdata v){
    v2f o;
    o.vertex = mul(UNITY_MATRIX_MVP, v.vertex);
    float3 worldPos = mul(_Object2World, v.vertex);
    o.rayPos = worldPos;

#if defined(RAYCAST_RAY_FROM_LIGHT_POS)
    if (_WorldSpaceLightPos0.w > 0){
        o.rayDir = worldPos.xyz - _WorldSpaceLightPos0.xyz;
        //o.rayDir = getRayToCamera(worldPos);
    }
    else{
#if defined(DIRECTIONAL)
        if ((UNITY_MATRIX_P[3].x == 0.0) && (UNITY_MATRIX_P[3].y == 0.0) && (UNITY_MATRIX_P[3].z == 0.0)){
            o.rayDir = -UNITY_MATRIX_V[2].xyz;
        }
        else{
            o.rayDir = getRayToCamera(worldPos);
        }
#else
        o.rayDir = getRayToCamera(worldPos);
#endif
    }
#else
    o.rayDir = getRayToCamera(worldPos);
#endif
    return o;
}

struct FragOut{
    float4 col: SV_Target;
    float depth: SV_Depth;
};

FragOut frag (v2f i){
    // sample the texture
    float3 rayDir = normalize(i.rayDir);
    float3 rayPos = i.rayPos;
    //fixed4 col = 1.0;//tex2D(_MainTex, i.uv);

    float3 spherePos = float3(0.0, 2.0, 0.0);
    float sphereRadius = 1.0;

    float t = getDistanceToSphere(rayPos, rayDir, spherePos, sphereRadius);
    if (t < 0)
        discard;

    float3 worldPos = rayPos + rayDir*t;

    ContactInfo sphereContact = calculateSphereContact(worldPos, spherePos, sphereRadius);
    //col.xyz = sphereContact.n;

    FragOut result;
    float4 col = 0.0;
#ifdef RAYCAST_NO_TEXTURE
    float4 baseColor = 1.0;//texColor;
#else
    float4 uv = calculateSphereTexcoordsFromNormal(sphereContact.n);
    float4 texColor = tex2D(_MainTex, uv.xy);
    float4 baseColor = texColor;
#endif
    //baseColor = 1.0;

#ifdef RAYCAST_USE_AMBIENT
    float4 ambColor = UNITY_LIGHTMODEL_AMBIENT;
    col = col + baseColor * ambColor;
#endif
#ifdef RAYCAST_USE_LIGHT
    float3 lightDir = _WorldSpaceLightPos0.xyz - worldPos*_WorldSpaceLightPos0.w;
    lightDir = normalize(lightDir);

    float shadowAtten = getShadowAttenuation(worldPos);
    float lightAtten = getLightAttenuation(worldPos);
    float lightDot = dot(lightDir, sphereContact.n);
    float lightFactor = max(0.0, lightDot);
    col = col + baseColor * lightFactor * _LightColor0 * shadowAtten * lightAtten;
#ifdef RAYCAST_USE_SPECULAR
    float3 reflectedLight = reflect(lightDir, sphereContact.n);
    float specFactor = max(dot(reflectedLight, rayDir), 0.0);
    specFactor = pow(specFactor, _SpecularPower);
    col = col + _LightColor0 * lightAtten * _SpecularColor * specFactor * shadowAtten;
#endif
#endif
#ifdef RAYCAST_USE_UNLIT_TEXTURE
    col = baseColor;
#endif

#if defined(RAYCAST_SHADOW_PASS)
#ifdef SHADOWS_CUBE
    float4 shadowDepth = calculateCubeShadowDepth(worldPos);
    result.depth = calculateFragmentDepth(worldPos);
    result.col = shadowDepth;
#else
    float shadowDepth = calculateShadowDepth(worldPos);
    result.depth = shadowDepth;
    result.col = shadowDepth;
#endif
#else
    result.depth = calculateFragmentDepth(worldPos);
    result.col = col;
#endif

    return result;
}

Raycast.cginc:

float getDistanceToSphere(float3 rayPos, float3 rayDir, float3 spherePos, float sphereRadius){
    float3 diff = spherePos - rayPos;

    float r2 = sphereRadius;

    float3 projectedDiff = dot(diff, rayDir)*rayDir;
    float3 perp = diff - projectedDiff;
    float perpSqLen = dot(perp, perp);
    float perpSqDepth = r2 - perpSqLen;

    if (perpSqDepth < 0)
        return -1.0;
        //return -1.0;

    float dt = sqrt(r2 - perpSqLen);
    float t = sqrt(dot(projectedDiff, projectedDiff)) - dt;
    return t;
}

struct ContactInfo{
    float3 relPos;
    float3 n;
};

ContactInfo calculateSphereContact(float3 worldPos, float3 spherePos, float sphereRadius){
    ContactInfo contact;
    contact.relPos = worldPos - spherePos;
    contact.n = normalize(contact.relPos);
    return contact;
}

float4 calculateSphereTexcoordsFromNormal(float3 n){
    float4 result = 0.0;

    float u = 0.0;
    float v = 0.0;

    float pi = 3.14159265358979323846264338327;
    v = 1.0 - acos(n.y)/pi, 0.0, 1.0;
    u = 1.0 - atan2(n.x, n.z)/(2.0*pi);

    result.x = u;
    result.y = v;

    return result;
}

float4 calculateSphereTexcoords(float3 worldPos, float3 spherePos, float sphereRadius){
    float3 dir = normalize(worldPos - spherePos);
    return calculateSphereTexcoordsFromNormal(dir);
}

float calculateFragmentDepth(float3 worldPos){
    float4 depthVec = mul(UNITY_MATRIX_VP, float4(worldPos, 1.0));
    return depthVec.z/depthVec.w;
}

LightAndShadow.cginc:

#include "Lighting.cginc"
#include "AutoLight.cginc"

float getLightAttenuation(float3 worldPos){
#if defined(POINT)
    {
        unityShadowCoord3 lightCoord = mul(_LightMatrix0, float4(worldPos, 1.0)).xyz;
        float result = tex2D(_LightTexture0, dot(lightCoord, lightCoord).rr).UNITY_ATTEN_CHANNEL;
        return result;
    }   
#elif defined(SPOT)
    {
        unityShadowCoord4 lightCoord = mul(_LightMatrix0, float4(worldPos, 1.0));
        float result = (lightCoord.z > 0) * UnitySpotCookie(lightCoord) * UnitySpotAttenuate(lightCoord.xyz);
        return result;
    }
#elif defined(DIRECTIONAL)
    {
        return 1.0;
    }
#elif defined(POINT_COOKIE)
    {
        unityShadowCoord3 lightCoord = mul(_LightMatrix0, float4(worldPos, 1.0)).xyz;
        float result = tex2D(_LightTextureB0, dot(lightCoord, lightCoord).rr).UNITY_ATTEN_CHANNEL * texCube(_LightTexture0, lightCoord).w;
        return result;
    }
#elif defined(DIRECTIONAL_COOKIE)
    {
        unityShadowCoord2 lightCoord = mul(_LightMatrix0, float4(worldPos, 1.0)).xy;
        float result = tex2D(_LightTexture0, lightCoord).w;
        return result;
    }
#else
    return 1.0;
#endif
}

float getShadowAttenuation(float3 worldPos){
#if defined(SHADOWS_CUBE)
    {
        unityShadowCoord3 shadowCoord = worldPos - _LightPositionRange.xyz;
        float result = UnitySampleShadowmap(shadowCoord);
        return result;
    }
#elif defined(SHADOWS_SCREEN)
    {
    #ifdef UNITY_NO_SCREENSPACE_SHADOWS
        unityShadowCoord4 shadowCoord = mul( unity_World2Shadow[0], worldPos);  
    #else
        unityShadowCoord4 shadowCoord = ComputeScreenPos(mul(UNITY_MATRIX_VP, float4(worldPos, 1.0)));
    #endif
        float result = unitySampleShadow(shadowCoord);
        return result;
    }       
#elif defined(SHADOWS_DEPTH) && defined(SPOT)
    {       
        unityShadowCoord4 shadowCoord = mul(unity_World2Shadow[0], float4(worldPos, 1.0));
        float result = UnitySampleShadowmap(shadowCoord);
        return result;
    }
#else
    return 1.0;
#endif  
}

Now, some explanations.

I got stuck for a while making direcitonal lights work. The only reason why I even managed to make it there, is Unity's frame debugger. If you aren't aware of it, you should give it a try. It shed some light on my shadowcasting issues.

The root of the problem is orthographic projections and special treatment of directional lights. First thing is, I eventually noticed that previous versions of the shader simply won't work if camera is orthographic. Why? Because calculation was not correct. In orthographic space all lines "shot" out of the camera must be parallel. The shader, however, calculated vector from camera to portal using vector substraction, which deformed raytraced primitive. This led to this bugfix:

float3 getRayToCamera(float3 worldPos){
    if (unity_OrthoParams.w > 0){
        return -UNITY_MATRIX_V[2].xyz;;
    }
    else
        return worldPos  - _WorldSpaceCameraPos;
}

This approach fixed orthographic issue, and the earth stopped trying to be cubical. If you wonder what the heck unity_OrthoParams is, it is one of the built-in shader variables.

This approach did not, however, solve issues with directional lights. While the other lights - spot and point ones - worked as expected (well, more or less, because I found out another bug in point light depth calculation just before posting this), directional light resulted in jumbled shadows, or no shadows at all (which you can see in earlier posting).

For example, this: Broken directional light

^^^ this is one of the many variations of the strange visual artifacts I encountered. while trying to make directional lights work. Obviously at some point shadow wasn't being correctly computed, and I wasn't quite sure where did I mess up this time.

I was planning to abandon directional lights, until I remembered that "Frame Debugger" is a thing, and is worth checking out. Using the frame debugger, I managed to figure out what the unity does while rendering shadows, and what's wrong with my shaders.

Frame Debugger

First thing I noticed was wrong pragma. I noticed it becase parameters and macro definitions passed to standard shader (which are visible on the "Shader Properties" tab) did not match the ones used by unity standard shader in the same pass. it turned out that I used #pragma multi_compile_fwdadd_fullshadows instead of #pragma multi_compile_shadowcaster in shadowcaster pass.

Next thing I noticed (in frame debugger) was that the sphere didn't have the right silhouette while being rendered into directional light's shadow buffer and unity renders directional light shadow in a very interesting way.

As it can be visible on earlier framedebugger screenshot, when directional light is being rendered, Unity:

  1. Creates depth texture.
  2. Renders objects being affected by directional lights - several times.
  3. Combines shadows into one screen-space shadow texture.

On a rendering preview you would ssee that with each rendering pass used in #2 part of the process, unity renders progressively bigger portion of the map.

I believe this is most likely an implementation of cascaded shadow maps I remember being introduced in one of GDC papers during early 2000s.

The interesting thing about those shadow passes is that they're rendered as spot lights (SPOT macro is enabled) that use orthographic projection matrix, and the only correct way to get direction to the "light" is to pretty much use unity view matrix. However, here's an important caveat that took me some time to figure out. Remember unity_OrthoParams from earlier? This built-in shader variable indicates whether camera is in orthographic mode or not. Except that while the directional lights are being rendered, it is not correctly set, so even if the scene is being rendered with ortho camera, the .w value will not correctly reflect that. This issue turned out to be a problem, because initial pass where unity rendered scene depth used very similar set of flags compared to a pass where unity was being used by lights. So fixing one broke another.

In the end I settled to this very ugly bugfix:

#if defined(RAYCAST_RAY_FROM_LIGHT_POS)
    if (_WorldSpaceLightPos0.w > 0){
        o.rayDir = worldPos.xyz - _WorldSpaceLightPos0.xyz;
        //o.rayDir = getRayToCamera(worldPos);
    }
    else{
#if defined(DIRECTIONAL)
        if ((UNITY_MATRIX_P[3].x == 0.0) && (UNITY_MATRIX_P[3].y == 0.0) && (UNITY_MATRIX_P[3].z == 0.0)){
            o.rayDir = -UNITY_MATRIX_V[2].xyz;
        }
        else{
            o.rayDir = getRayToCamera(worldPos);
        }
#else
        o.rayDir = getRayToCamera(worldPos);
#endif

See that looong UNITY_MATRIX_P[3].... line? It tests whether the current projection matrix is orthographic or not. Directly. Not the best thing to do in a shader, but that was pretty much the only way to go about it. (I reported this as a bug, by the way). At the very least, UNITY_MATRIX_P is a uniform, so the branching shouldn't be going nuts because of it alone.

By the way, this is a line that returns forward vector for an orthographic camera:

        return -UNITY_MATRIX_V[2].xyz;

This is the end of today's post.

Hopefully now that the issue of integrating raytraced primitives into unity lighting system is out of the way, I can get to testing more interesting stuff. Like complex objects, reflections, and the like.

unity3d shaders raytracing