I guess the problem you're trying to solve is finding the largest (with more nodes) BST in BT. In that case you'll need to traverse all the tree nodes checking if it is a BST, and once you find one you'll have to check if it has more nodes than the largest one found till the moment.
class TreeNode
{
public int value;
public TreeNode left;
public TreeNode right;
}
void LargestBST(TreeNode bt, IDictionary<TreeNode, bool> isBST, IDictionary<TreeNode, int> nodeCount, ref TreeNode largestBST)
{
if (bt == null)
return;
if (IsBST(bt, isBST) && (largestBST == null || NodeCount(bt, nodeCount) > NodeCount(largestBST, nodeCount))
largestBST = bt;
else
{
LargestBST(bt.left, isBST, nodeCount, ref largestBST);
LargestBST(bt.Right, isBST, nodeCount, ref largestBST);
}
}
bool IsBST(TreeNode node, IDictionary<TreeNode, bool> isBST)
{
if (node == null)
return true;
bool result;
if (!isBST.TryGetValue(node, out result))
{
TreeNode maxLeft = Max(node.Left);
TreeNode minRight = Min(node.Right);
result = (maxLeft == null || maxLeft.value <= node.value) &&
(minRight == null || minRight.value >= node.value) &&
IsBST(node.left, isBST) && IsBST(node.Right, isBST);
isBST.Add(node, result);
}
return result;
}
TreeNode Max(TreeNode node)
{
if (node == null)
return null;
while (node.right != null)
node = node.right;
return node;
}
TreeNode Min(TreeNode node)
{
if (node == null)
return null;
while (node.left != null)
node = node.left;
return node;
}
int NodeCount(TreeNode node, IDictionary<TreeNode, int> nodeCount)
{
if (node == null)
return 0;
int result;
if (!nodeCount.TryGetValue(node, out result))
{
result = 1 + NodeCount(node.left, nodeCount) + NodeCount(node.right, nodeCount);
nodeCount.Add(node, result);
}
return result;
}