XNA + Pixel Shader Difficulties

Question

I've written a basic 2d pixel shader, and i can't seem to get it to work. If i draw with the effect active, then nothing draws to the screen. But if i disable it, then the texture draws to the screen as expected.

My aim is to be able to draw an arbitrary texture to the screen, then have this pixel shader "carve" circular hunks of pixels out of it, for use in an overlay system for ranges and such.

Here's my pixel shader code:

sampler TextureSampler : register(s0);
//A list of positions for circles. They are specified in texture space rather than screen space.
float2 PositionData[64];
//A matching list of radiuses. These are specified in pixels, though.
float Radii[64];
//how much of the array is filled with data.
int DataSize;
//the size of the texture being drawn.
float2 TextureSize;

float4 RenderSolidCircles(float2 texCoord : TEXCOORD0) : COLOR
{
    float opacityAcc = 1;
    float2 screenSpaceTexCoord = texCoord * TextureSize;
    for (int i = 0; i < DataSize; i++)
    {
        float2 properPosCoordinate = PositionData[i] * TextureSize;
        float dist = length(screenSpaceTexCoord - properPosCoordinate) - Radii[i];
        if (dist < 0)
        {
            opacityAcc -= min(abs(dist), 1);
        }
    }
    opacityAcc = max(0, opacityAcc);
    float4 outPix = tex2D(TextureSampler, texCoord);
    outPix.a *= opacityAcc;
    return outPix;
}


technique SolidCircles
{
    pass P0
    {
        PixelShader = compile ps_3_0 RenderSolidCircles();
    }
}

float4 PassThrough(float2 texCoord : TEXCOORD0) : COLOR
{
    return tex2D(TextureSampler, texCoord);
}
technique PassThrough
{
    pass P0
    {
        PixelShader = compile ps_3_0 PassThrough();
    }
}

Here's the ASM version of the SolidCircles technique:

//
// Generated by Microsoft (R) D3DX9 Shader Compiler 9.15.779.0000
//
// Parameters:
//
//   int DataSize;
//   float2 PositionData[64];
//   float Radii[64];
//   sampler2D TextureSampler;
//   float2 TextureSize;
//
//
// Registers:
//
//   Name           Reg   Size
//   -------------- ----- ----
//   PositionData   c0      64
//   Radii          c64     64
//   DataSize       c128     1
//   TextureSize    c129     1
//   TextureSampler s0       1
//
//
// Default values:
//snipped comments here

    ps_3_0
    def c130, 1, 0, -1, 2
    def c131, 3, 4, 5, 6
    def c132, 7, 8, 9, 10
    def c133, 11, 12, 13, 14
    def c134, 15, 16, 17, 18
    def c135, 19, 20, 21, 22
    def c136, 23, 24, 25, 26
    def c137, 27, 28, 29, 30
    def c138, 31, 32, 33, 34
    def c139, 35, 36, 37, 38
    def c140, 39, 40, 41, 42
    def c141, 43, 44, 45, 46
    def c142, 47, 48, 49, 50
    def c143, 51, 52, 53, 54
    def c144, 55, 56, 57, 58
    def c145, 59, 60, 61, 62
    def c146, 63, 0, 0, 0
    dcl_texcoord v0.xy  // texCoord<0,1>
    dcl_2d s0

#line 22 "C:\Users\RCIX\Documents\Visual Studio 2008\Projects\2DFXFilesTest\2DFXFilesTest\Content\OverlayFx.fx"
    mov r0.w, c130.x  // opacityAcc<0>
    mul r2.xy, v0, c129  // screenSpaceTexCoord<0,1>
    mov r5.w, -c128.x
    add r0.z, r5.w, c130.y
    cmp r12.w, r0.z, c130.y, c130.x
    mul r11.w, r12.w, c130.x
    if_ne r11.w, -r11.w
      mov r13.xy, c129  // ::TextureSize<0,1>
      mul r12.xy, r13, c0  // properPosCoordinate<0,1>
      mov r12.xy, -r12
      add r11.xy, r2, r12
      mul r16.xy, r11, r11
      add r11.z, r16.x, r16.y
      rsq r10.w, r11.z
      rcp r8.w, r10.w
      mov r9.w, -c64.x
      add r4.w, r8.w, r9.w  // dist<0>
      add r7.w, r4.w, c130.y
      cmp r6.w, r7.w, c130.y, c130.x
      mov r3.w, -r4.w
      mov r5.z, -r3.w
      add r1.w, r4.w, r5.z
      cmp r15.w, r1.w, r4.w, r3.w
      add r14.w, r15.w, c130.z
      cmp r2.w, r14.w, c130.x, r15.w
      mov r2.w, -r2.w
      add r13.w, r2.w, c130.x  // opacityAcc<0>
      mov r6.w, -r6.w
      cmp r0.w, r6.w, r0.w, r13.w  // opacityAcc<0>

#line 24
    endif

//snipped 63 blocks of unrolled loop code

#line 33
    mov r1.w, -r0.w
    add r15.w, r1.w, c130.y
    cmp r14.w, r15.w, c130.y, r0.w  // opacityAcc<0>
    texld r0, v0, s0  // outPix<0,1,2,3>
    mul r2.x, r14.w, r0.w  // outPix<3>
    mov oC0.xyz, r0  // ::RenderSolidCircles<0,1,2>
    mov oC0.w, r2.x  // ::RenderSolidCircles<3>

// approximately 1866 instruction slots used (1 texture, 1865 arithmetic)

and here's the relevant portion of my Draw function in my Game class:

GraphicsDevice.Clear(Color.CornflowerBlue);
overlayEffect.Parameters["PositionData"].SetValue(Positions.ToArray());
overlayEffect.Parameters["Radii"].SetValue(Radii.ToArray());
overlayEffect.Parameters["DataSize"].SetValue(64);
overlayEffect.Parameters["TextureSize"].SetValue(new Vector2(500));

spriteBatch.Begin(SpriteBlendMode.AlphaBlend,
    SpriteSortMode.Immediate,
    SaveStateMode.None);

overlayEffect.Begin();
overlayEffect.CurrentTechnique.Passes[0].Begin();

spriteBatch.Draw(pixTex, new Rectangle(0, 0, 500, 500), Color.White);
spriteBatch.End();

overlayEffect.CurrentTechnique.Passes[0].End();
overlayEffect.End();

base.Draw(gameTime);

Finally, here's my function that builds the list of positions and radii:

private void RebuildPositionsList()
{
    spriteBatch = new SpriteBatch(GraphicsDevice);
    Positions = new List<Vector2>();
    Radii = new List<float>();
    for (int i = 0; i < 64; i++)
    {
        Positions.Add(
            new Vector2(
                (float)r.NextDouble(),
                (float)r.NextDouble())
                );
        Radii.Add(((float)r.NextDouble() * 100) + 40);
    }
}

The lines that make my texture:

pixTex = new Texture2D(GraphicsDevice, 1, 1);
pixTex.SetData<Color>(new Color[] { new Color(0f, 0f, 0f, 1f) });

Positions and Radii are Lists of Vectors and floats respectively, of size 64. pixTex is a 1 pixel solid black texture.

Why does the shader not work?

Answer 1

Before calling effect.End() you need to call spriteBatch.End().

The reason for this (and the fact that it is "fixed" in XNA 4.0) is described in this article on Shawn Hargreaves' blog. (This entry may also be worth reading.)

Basically (in XNA 3.1): SpriteSortMode.Immediate is not as immediate as you might expect it to be. You need to call spriteBatch.End() to actually push the final batch of sprites to the GPU, before you end your effect.

The Sprite Effects sample shows how to correctly apply effects to sprites.

Answer 2

OK, first of all you need to apply the two fixes in my other answer (end the batch before ending the effect, and (from comments) use SpriteBlendMode.AlphaBlend).

Now - the problem in your shader. It's actually working - but possibly not in the way you expect. It seems you're confusing the screen-space coordinates your sprite is drawn at (width = 500, height = 500), with the texture-space coordinates that your pixel shader works in (width = 1, height = 1).

So first of all - when you cut holes in your sprite you need to do it in texture-space, like so:

Positions.Add(new Vector2(0.5f, 0.5f));
Radii.Add(0.25f);

Positions.Add(new Vector2(0.25f, 0.25f));
Radii.Add(0.1f);

And second of all, what looks like an attempt at anti-aliasing is causing your cut outs to be more of a fade-out. It needs to take into account the size of the texture as it is drawn on screen. The easiest fix is to change this line:

opacityAcc -= min(abs(dist), 1);

To this:

opacityAcc -= min(abs(dist * 500), 1);

Of course - this assumes that your sprite is drawn at 500 by 500. You should pass the actual value in as a shader parameter.

If you're going to draw your sprite non-square, then you'll need to do a little extra maths to make the coordinate systems "line up". I'll leave that as an exercise.

Answer 3

So in the comments for this answer it was determined that yet another problem here is that the vertex shader for Sprite Batch in XNA 3.1 is vs_1_1. And strictly speaking you cannot mix SM 3.0 vertex or pixel shaders with shaders of different versions. It seems that, in practice, most cards will let you get away with it, but apparently RCIX's card (a Radeon HD 4850) will not.

(This is why it's worth having the DirectX debug runtimes on hand (from the SDK), as they will warn you about things like this. You can use DebugView to view its output.)

There are a number of solutions to this issue:

1) By far the easiest solution is to upgrade to XNA 4.0 (the downsides are the breaking changes and the fact that it's currently only in beta). In this version of XNA you can easily specify your own vertex shader for SpriteBatch.

2) You could use a custom vertex shader with SpriteBatch in XNA 3.1, it's just not as easy. A good starting point would be the source code for the vertex shader used by XNA (up until XNA 4.0, see above).

3) Finally: perhaps you could just make your shader use ps_2_0? Do you really need 64 cutouts per texture?