Invalid algorithm specified on Windows 2003 Server only

Question

I am decoding a file using the following method:

string outFileName = zfoFileName.Replace(".zfo", "_tmp.zfo");
FileStream inFile = null;
FileStream outFile = null;
inFile = File.Open(zfoFileName, FileMode.Open);
outFile = File.Create(outFileName);
LargeCMS.CMS cms = new LargeCMS.CMS();
cms.Decode(inFile, outFile);

This is working fine on my Win 7 dev machine, but on a Windows 2003 server production machine it fails with the following exception:

Exception: System.Exception: CryptMsgUpdate error #-2146893816 ---> System.ComponentModel.Win32Exception: Invalid algorithm specified --- End of inner exception stack trace --- at LargeCMS.CMS.Decode(FileStream inFile, FileStream outFile)

Here are the classes below which I call to do the decoding, if needed I can upload a sample file for decoding, its just strange it works on Win 7, and not on Win2k3 server:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Runtime.InteropServices;
using System.ComponentModel;

namespace LargeCMS
{
    class CMS
    {
        // File stream to use in callback function
        private FileStream m_callbackFile;

        // Streaming callback function for encoding
        private Boolean StreamOutputCallback(IntPtr pvArg, IntPtr pbData, int cbData, Boolean fFinal)
        {
            // Write all bytes to encoded file
            Byte[] bytes = new Byte[cbData];
            Marshal.Copy(pbData, bytes, 0, cbData);
            m_callbackFile.Write(bytes, 0, cbData);

            if (fFinal)
            {
                // This is the last piece. Close the file
                m_callbackFile.Flush();
                m_callbackFile.Close();
                m_callbackFile = null;
            }

            return true;
        }



        // Decode CMS with streaming to support large data
        public void Decode(FileStream inFile, FileStream outFile)
        {
            // Variables
            Win32.CMSG_STREAM_INFO StreamInfo;
            Win32.CERT_CONTEXT SignerCertContext;

            BinaryReader stream = null;
            GCHandle gchandle = new GCHandle();

            IntPtr hMsg = IntPtr.Zero;
            IntPtr pSignerCertInfo = IntPtr.Zero;
            IntPtr pSignerCertContext = IntPtr.Zero;
            IntPtr pbPtr = IntPtr.Zero;
            IntPtr hStore = IntPtr.Zero;
            Byte[] pbData;
            Boolean bResult = false;
            int dwFileSize;
            int dwRemaining;
            int dwSize;
            int cbSignerCertInfo;

            try
            {
                // Get data to decode
                dwFileSize = (int)inFile.Length;
                stream = new BinaryReader(inFile);
                pbData = stream.ReadBytes(dwFileSize);

                // Prepare stream for decoded info
                m_callbackFile = outFile;

                // Populate Stream Info struct
                StreamInfo = new Win32.CMSG_STREAM_INFO();
                StreamInfo.cbContent = dwFileSize;
                StreamInfo.pfnStreamOutput = new Win32.StreamOutputCallbackDelegate(StreamOutputCallback);

                // Open message to decode
                hMsg = Win32.CryptMsgOpenToDecode(
                    Win32.X509_ASN_ENCODING | Win32.PKCS_7_ASN_ENCODING,
                    0,
                    0,
                    IntPtr.Zero,
                    IntPtr.Zero,
                    ref StreamInfo
                );
                if (hMsg.Equals(IntPtr.Zero))
                {
                    throw new Exception("CryptMsgOpenToDecode error #" + Marshal.GetLastWin32Error().ToString(), new Win32Exception(Marshal.GetLastWin32Error()));
                }

                // Process the whole message
                gchandle = GCHandle.Alloc(pbData, GCHandleType.Pinned);
                pbPtr = gchandle.AddrOfPinnedObject();
                dwRemaining = dwFileSize;
                dwSize = (dwFileSize < 1024 * 1000 * 100) ? dwFileSize : 1024 * 1000 * 100;
                while (dwRemaining > 0)
                {
                    // Update message piece by piece     
                    bResult = Win32.CryptMsgUpdate(
                        hMsg,
                        pbPtr,
                        dwSize,
                        (dwRemaining <= dwSize) ? true : false
                    );
                    if (!bResult)
                    {
                        throw new Exception("CryptMsgUpdate error #" + Marshal.GetLastWin32Error().ToString(), new Win32Exception(Marshal.GetLastWin32Error()));
                    }

                    // Move to the next piece
                    pbPtr = new IntPtr(pbPtr.ToInt64() + dwSize);
                    dwRemaining -= dwSize;
                    if (dwRemaining < dwSize)
                    {
                        dwSize = dwRemaining;
                    }
                }

                // Get signer certificate info
                cbSignerCertInfo = 0;
                bResult = Win32.CryptMsgGetParam(
                    hMsg,
                    Win32.CMSG_SIGNER_CERT_INFO_PARAM,
                    0,
                    IntPtr.Zero,
                    ref cbSignerCertInfo
                );
                if (!bResult)
                {
                    throw new Exception("CryptMsgGetParam error #" + Marshal.GetLastWin32Error().ToString(), new Win32Exception(Marshal.GetLastWin32Error()));
                }

                pSignerCertInfo = Marshal.AllocHGlobal(cbSignerCertInfo);

                bResult = Win32.CryptMsgGetParam(
                    hMsg,
                    Win32.CMSG_SIGNER_CERT_INFO_PARAM,
                    0,
                    pSignerCertInfo,
                    ref cbSignerCertInfo
                );
                if (!bResult)
                {
                    throw new Exception("CryptMsgGetParam error #" + Marshal.GetLastWin32Error().ToString(), new Win32Exception(Marshal.GetLastWin32Error()));
                }

                // Open a cert store in memory with the certs from the message
                hStore = Win32.CertOpenStore(
                    Win32.CERT_STORE_PROV_MSG,
                    Win32.X509_ASN_ENCODING | Win32.PKCS_7_ASN_ENCODING,
                    IntPtr.Zero,
                    0,
                    hMsg
                );
                if (hStore.Equals(IntPtr.Zero))
                {
                    throw new Exception("CertOpenStore error #" + Marshal.GetLastWin32Error().ToString(), new Win32Exception(Marshal.GetLastWin32Error()));
                }

                // Find the signer's cert in the store
                pSignerCertContext = Win32.CertGetSubjectCertificateFromStore(
                    hStore,
                    Win32.X509_ASN_ENCODING | Win32.PKCS_7_ASN_ENCODING,
                    pSignerCertInfo
                );
                if (pSignerCertContext.Equals(IntPtr.Zero))
                {
                    throw new Exception("CertGetSubjectCertificateFromStore error #" + Marshal.GetLastWin32Error().ToString(), new Win32Exception(Marshal.GetLastWin32Error()));
                }

                // Set message for verifying
                SignerCertContext = (Win32.CERT_CONTEXT)Marshal.PtrToStructure(pSignerCertContext, typeof(Win32.CERT_CONTEXT));
                bResult = Win32.CryptMsgControl(
                    hMsg,
                    0,
                    Win32.CMSG_CTRL_VERIFY_SIGNATURE,
                    SignerCertContext.pCertInfo
                );
                if (!bResult)
                {
                    throw new Exception("CryptMsgControl error #" + Marshal.GetLastWin32Error().ToString(), new Win32Exception(Marshal.GetLastWin32Error()));
                }
            }
            finally
            {
                // Clean up
                if (gchandle.IsAllocated)
                {
                    gchandle.Free();
                }
                if (!pSignerCertContext.Equals(IntPtr.Zero))
                {
                    Win32.CertFreeCertificateContext(pSignerCertContext);
                }
                if (!pSignerCertInfo.Equals(IntPtr.Zero))
                {
                    Marshal.FreeHGlobal(pSignerCertInfo);
                }
                if (!hStore.Equals(IntPtr.Zero))
                {
                    Win32.CertCloseStore(hStore, Win32.CERT_CLOSE_STORE_FORCE_FLAG);
                }
                if (stream != null)
                {
                    stream.Close();
                }
                if (m_callbackFile != null)
                {
                    m_callbackFile.Close();
                }
                if (!hMsg.Equals(IntPtr.Zero))
                {
                    Win32.CryptMsgClose(hMsg);
                }
            }
        }
    }
}

and

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.InteropServices;
using System.Security.Cryptography.X509Certificates;
using System.ComponentModel;
using System.Security.Cryptography;

namespace LargeCMS
{
    class Win32
    {
        #region "CONSTS"

        public const int X509_ASN_ENCODING = 0x00000001;
        public const int PKCS_7_ASN_ENCODING = 0x00010000;
        public const int CMSG_SIGNED = 2;
        public const int CMSG_DETACHED_FLAG = 0x00000004;
        public const int AT_KEYEXCHANGE = 1;
        public const int AT_SIGNATURE = 2;
        public const String szOID_OIWSEC_sha1 = "1.3.14.3.2.26";
        public const int CMSG_CTRL_VERIFY_SIGNATURE = 1;
        public const int CMSG_CERT_PARAM = 12;
        public const int CMSG_SIGNER_CERT_INFO_PARAM = 7;
        public const int CERT_STORE_PROV_MSG = 1;
        public const int CERT_CLOSE_STORE_FORCE_FLAG = 1;

        #endregion

        #region "STRUCTS"

        [StructLayout(LayoutKind.Sequential)]
        public struct CRYPT_ALGORITHM_IDENTIFIER
        {
            public String pszObjId;
            BLOB Parameters;
        }

        [StructLayout(LayoutKind.Sequential)]
        public struct CERT_ID
        {
            public int dwIdChoice;
            public BLOB IssuerSerialNumberOrKeyIdOrHashId;
        }

        [StructLayout(LayoutKind.Sequential)]
        public struct CMSG_SIGNER_ENCODE_INFO
        {
            public int cbSize;
            public IntPtr pCertInfo;
            public IntPtr hCryptProvOrhNCryptKey;
            public int dwKeySpec;
            public CRYPT_ALGORITHM_IDENTIFIER HashAlgorithm;
            public IntPtr pvHashAuxInfo;
            public int cAuthAttr;
            public IntPtr rgAuthAttr;
            public int cUnauthAttr;
            public IntPtr rgUnauthAttr;
            public CERT_ID SignerId;
            public CRYPT_ALGORITHM_IDENTIFIER HashEncryptionAlgorithm;
            public IntPtr pvHashEncryptionAuxInfo;
        }

        [StructLayout(LayoutKind.Sequential)]
        public struct CERT_CONTEXT
        {
            public int dwCertEncodingType;
            public IntPtr pbCertEncoded;
            public int cbCertEncoded;
            public IntPtr pCertInfo;
            public IntPtr hCertStore;
        }

        [StructLayout(LayoutKind.Sequential)]
        public struct BLOB
        {
            public int cbData;
            public IntPtr pbData;
        }

        [StructLayout(LayoutKind.Sequential)]
        public struct CMSG_SIGNED_ENCODE_INFO
        {
            public int cbSize;
            public int cSigners;
            public IntPtr rgSigners;
            public int cCertEncoded;
            public IntPtr rgCertEncoded;
            public int cCrlEncoded;
            public IntPtr rgCrlEncoded;
            public int cAttrCertEncoded;
            public IntPtr rgAttrCertEncoded;
        }

        [StructLayout(LayoutKind.Sequential)]
        public struct CMSG_STREAM_INFO
        {
            public int cbContent;
            public StreamOutputCallbackDelegate pfnStreamOutput;
            public IntPtr pvArg;
        }

        #endregion

        #region "DELEGATES"

        public delegate Boolean StreamOutputCallbackDelegate(IntPtr pvArg, IntPtr pbData, int cbData, Boolean fFinal);

        #endregion

        #region "API"

        [DllImport("advapi32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        public static extern Boolean CryptAcquireContext(
          ref IntPtr hProv,
          String pszContainer,
          String pszProvider,
          int dwProvType,
          int dwFlags
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern IntPtr CryptMsgOpenToEncode(
            int dwMsgEncodingType,
            int dwFlags,
            int dwMsgType,
            ref CMSG_SIGNED_ENCODE_INFO pvMsgEncodeInfo,
            String pszInnerContentObjID,
            ref CMSG_STREAM_INFO pStreamInfo
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern IntPtr CryptMsgOpenToDecode(
            int dwMsgEncodingType,
            int dwFlags,
            int dwMsgType,
            IntPtr hCryptProv,
            IntPtr pRecipientInfo,
            ref CMSG_STREAM_INFO pStreamInfo
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern Boolean CryptMsgClose(
            IntPtr hCryptMsg
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern Boolean CryptMsgUpdate(
            IntPtr hCryptMsg,
            Byte[] pbData,
            int cbData,
            Boolean fFinal
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern Boolean CryptMsgUpdate(
            IntPtr hCryptMsg,
            IntPtr pbData,
            int cbData,
            Boolean fFinal
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern Boolean CryptMsgGetParam(
            IntPtr hCryptMsg,
            int dwParamType,
            int dwIndex,
            IntPtr pvData,
            ref int pcbData
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern Boolean CryptMsgControl(
            IntPtr hCryptMsg,
            int dwFlags,
            int dwCtrlType,
            IntPtr pvCtrlPara
        );

        [DllImport("advapi32.dll", SetLastError = true)]
        public static extern Boolean CryptReleaseContext(
            IntPtr hProv,
            int dwFlags
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern IntPtr CertCreateCertificateContext(
            int dwCertEncodingType,
            IntPtr pbCertEncoded,
            int cbCertEncoded
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern Boolean CertFreeCertificateContext(
            IntPtr pCertContext
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern IntPtr CertOpenStore(
            int lpszStoreProvider,
            int dwMsgAndCertEncodingType,
            IntPtr hCryptProv,
            int dwFlags,
            IntPtr pvPara
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern IntPtr CertGetSubjectCertificateFromStore(
            IntPtr hCertStore,
            int dwCertEncodingType,
            IntPtr pCertId
        );

        [DllImport("Crypt32.dll", SetLastError = true)]
        public static extern IntPtr CertCloseStore(
            IntPtr hCertStore,
            int dwFlags
        );

        #endregion
    }
}

Answer 1

Could it be that the CSP you use aren't installed on your Win2003? I remember reading that XP before SP3 had some problems with SHA-2 or something, and XP and Win2003 are more or less the same basic OS I think.

I think you can see what's installed on the following registry key
HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Cryptography\Defaults

Maybe you can compare between your machines.

Answer 2

MD5 and other legacy/broken cryptographic algorithms can be disabled by group policy (needed for US Government use).