Recognize objects in image

Question

Hello I am in the process of doing a school project, where we have a robot driving on the ground in between Flamingo plates. We need to create an algorithm that can identify the locations of these plates, so we can create paths around them (We are using A Star for that).

So far have we worked with AForged Library and we have created the following class, the only problem with this is that when it create the rectangles dose it not take in account that the plates are not always parallel with the camera border, and it that case will it just create a rectangle that cover the whole plate. So we need to some way find the rotation on the object, or another way to identify this. I have create an image that might help explain this

Image the describe the problem: http://img683.imageshack.us/img683/9835/imagerectangle.png

Any help on how I can do this would be greatly appreciated.

Any other information or ideers are always welcome.

public class PasteMap
{
    private Bitmap image;
    private Bitmap processedImage;
    private Rectangle[] rectangels;

    public void initialize(Bitmap image)
    {
        this.image = image;
    }

    public void process()
    {
        processedImage = image;
        processedImage = applyFilters(processedImage);
        processedImage = filterWhite(processedImage);
        rectangels = extractRectangles(processedImage);
        //rectangels = filterRectangles(rectangels);
        processedImage = drawRectangelsToImage(processedImage, rectangels);
    }

    public Bitmap getProcessedImage
    {
        get
        {
            return processedImage;
        }
    }

    public Rectangle[] getRectangles
    {
        get
        {
            return rectangels;
        }
    }

    private Bitmap applyFilters(Bitmap image)
    {
        image = new ContrastCorrection(2).Apply(image);
        image = new GaussianBlur(10, 10).Apply(image);
        return image;
    }

    private Bitmap filterWhite(Bitmap image)
    {
        Bitmap test = new Bitmap(image.Width, image.Height);

        for (int width = 0; width < image.Width; width++)
        {
            for (int height = 0; height < image.Height; height++)
            {
                if (image.GetPixel(width, height).R > 200 &&
                    image.GetPixel(width, height).G > 200 &&
                    image.GetPixel(width, height).B > 200)
                {
                    test.SetPixel(width, height, Color.White);
                }
                else
                    test.SetPixel(width, height, Color.Black);
            }
        }
        return test;
    }

    private Rectangle[] extractRectangles(Bitmap image)
    {
        BlobCounter bc = new BlobCounter();
        bc.FilterBlobs = true;
        bc.MinWidth  = 5;
        bc.MinHeight = 5;
        // process binary image
        bc.ProcessImage( image );
        Blob[] blobs = bc.GetObjects(image, false);
        // process blobs
        List<Rectangle> rects = new List<Rectangle>();
        foreach (Blob blob in blobs)
        {
            if (blob.Area > 1000)
            {
                rects.Add(blob.Rectangle);
            }
        }

        return rects.ToArray();
    }

    private Rectangle[] filterRectangles(Rectangle[] rects)
    {
        List<Rectangle> Rectangles = new List<Rectangle>();
        foreach (Rectangle rect in rects)
        {
            if (rect.Width > 75 && rect.Height > 75)
                Rectangles.Add(rect);
        }

        return Rectangles.ToArray();
    }

    private Bitmap drawRectangelsToImage(Bitmap image, Rectangle[] rects)
    {
        BitmapData data = image.LockBits(new Rectangle(0, 0, image.Width, image.Height),
                ImageLockMode.ReadWrite, PixelFormat.Format24bppRgb);
        foreach (Rectangle rect in rects)
            Drawing.FillRectangle(data, rect, Color.Red);
        image.UnlockBits(data);
        return image;
    }
}

Answer 1

The most straight forward solution is probably to find the corners of each detected blob and then geometrically calculate which point-pairs make up the different sides of the squares. This assumes that the camera is looking straight down such that a square is actually a square in the image (no perspective distorsion).

I am however a bit curious why you need to know the rotation of the rectangles. In all the example images the rectangles are more or less aligned with the image borders, so a bounding box for a rectangle blob would be very close to what you are trying to find. At least it should be good enough for path finding.

Answer 2

You should be using neural networks. See: http://en.wikipedia.org/wiki/Neural_network

Answer 3

You need to analyse the blobs a bit more to find the corners as @kigurai has said. The AForge library allows you to do this, see the section Finding convex hull on this page for more info. The screenshot below (from the page) shows a small sample of what the convex hull is.

You want to take a look at the GetBlobsLeftAndRightEdges function and the GrahamConvexHull class.

Answer 4

If any 1 is interested, is this the way i did it.

Blobsprocessing: using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Drawing; using System.Drawing.Imaging;

using AForge;
using AForge.Imaging;
using AForge.Imaging.Filters;
using AForge.Imaging.Textures;
using AForge.Math.Geometry;

namespace CDIO.Library
{
    public class Blobsprocessing
    {
        Bitmap image;
        BlobCounter BlobCounter;
        Blob[] blobs;
        List<Polygon> hulls;

        public Blobsprocessing(Bitmap image)
        {
            this.image = image; 
        }

        public void Process()
        {
            BlobCounter = new BlobCounter();

            processBlobs();
            extractConvexHull();
        }
        public List<Polygon> getHulls()
        {
            return hulls;
        }

        private void processBlobs()
        {
            BlobCounter.FilterBlobs = true;
            BlobCounter.MinWidth = 5;
            BlobCounter.MinHeight = 5;
            // set ordering options
            BlobCounter.ObjectsOrder = ObjectsOrder.Size;
            // process binary image
            BlobCounter.ProcessImage(image);

            blobs = BlobCounter.GetObjectsInformation();
        }

        private void extractConvexHull()
        {
            GrahamConvexHull hullFinder = new GrahamConvexHull();

            // process each blob
            hulls = new List<Polygon>();
            foreach (Blob blob in blobs)
            {
                List<IntPoint> leftPoints, rightPoints, edgePoints;
                edgePoints = new List<IntPoint>();

                // get blob's edge points
                BlobCounter.GetBlobsLeftAndRightEdges(blob,
                    out leftPoints, out rightPoints);

                edgePoints.AddRange(leftPoints);
                edgePoints.AddRange(rightPoints);

                // blob's convex hull
                List<IntPoint> hull = hullFinder.FindHull(edgePoints);
                hulls.Add(new Polygon(hull));
            }
        }
    }
}

MapFilters:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Drawing;
using System.Drawing.Imaging;

using AForge;
using AForge.Imaging;
using AForge.Imaging.Filters;
using AForge.Imaging.Textures;
using AForge.Math.Geometry;

namespace CDIO.Library
{
    public class MapFilters
    {
        private Bitmap image;
        private Bitmap processedImage;
        private Rectangle[] rectangels;

        public void initialize(Bitmap image)
        {
            this.image = image;
        }

        public void process()
        {
            processedImage = image;
            processedImage = applyFilters(processedImage);
            processedImage = filterWhite(processedImage);
        }

        public Bitmap getProcessedImage
        {
            get
            {
                return processedImage;
            }
        }

        private Bitmap applyFilters(Bitmap image)
        {
            image = new ContrastCorrection(2).Apply(image);
            image = new GaussianBlur(10, 10).Apply(image);
            return image;
        }

        private Bitmap filterWhite(Bitmap image)
        {
            Bitmap test = new Bitmap(image.Width, image.Height);

            for (int width = 0; width < image.Width; width++)
            {
                for (int height = 0; height < image.Height; height++)
                {
                    if (image.GetPixel(width, height).R > 200 &&
                        image.GetPixel(width, height).G > 200 &&
                        image.GetPixel(width, height).B > 200)
                    {
                        test.SetPixel(width, height, Color.White);
                    }
                    else
                        test.SetPixel(width, height, Color.Black);
                }
            }
            return test;
        }
    }
}

Polygon: using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Drawing; using System.Drawing.Imaging; using System.Threading;

using AForge;
using AForge.Imaging;
using AForge.Imaging.Filters;
using AForge.Imaging.Textures;
using AForge.Math.Geometry;

namespace CDIO.Library
{
    public class Polygon
    {
        List<IntPoint> hull;
        public Polygon(List<IntPoint> hull)
        {
            this.hull = hull;
        }

        public bool inPoly(int x, int y)
        {
            int i, j = hull.Count - 1;
            bool oddNodes = false;

            for (i = 0; i < hull.Count; i++)
            {
                if (hull[i].Y < y && hull[j].Y >= y
                || hull[j].Y < y && hull[i].Y >= y)
                {
                    try
                    {
                        if (hull[i].X + (y - hull[i].X) / (hull[j].X - hull[i].X) * (hull[j].X - hull[i].X) < x)
                        {
                            oddNodes = !oddNodes;
                        }
                    }
                    catch (DivideByZeroException e)
                    {
                        if (0 < x)
                        {
                            oddNodes = !oddNodes;
                        }
                    }
                }
                j = i;
            }
            return oddNodes;
        }

        public Rectangle getRectangle()
        {
            int x = -1, y = -1, width = -1, height = -1;
            foreach (IntPoint item in hull)
            {
                if (item.X < x || x == -1)
                    x = item.X;
                if (item.Y < y || y == -1)
                    y = item.Y;


                if (item.X > width || width == -1)
                    width = item.X;
                if (item.Y > height || height == -1)
                    height = item.Y;


            }
            return new Rectangle(x, y, width-x, height-y);
        }
        public Bitmap drawRectangle(Bitmap image)
        {
            Rectangle rect = getRectangle();

            Bitmap clonimage = (Bitmap)image.Clone();
            BitmapData data = clonimage.LockBits(new Rectangle(0, 0, image.Width, image.Height), ImageLockMode.ReadWrite, image.PixelFormat);
            Drawing.FillRectangle (data, rect, getRandomColor());
            clonimage.UnlockBits(data);
            return clonimage;
        }

        public Point[] getMap()
        {
            List<Point> points = new List<Point>();
            Rectangle rect = getRectangle();
            for (int x = rect.X; x <= rect.X + rect.Width; x++)
            {
                for (int y = rect.Y; y <= rect.Y + rect.Height; y++)
                {
                    if (inPoly(x, y))
                        points.Add(new Point(x, y));
                }
            }
            return points.ToArray();
        }

        public float calculateArea()
        {
            List<IntPoint> list = new List<IntPoint>();
            list.AddRange(hull);
            list.Add(hull[0]);

            float area = 0.0f;
            for (int i = 0; i < hull.Count; i++)
            {
                area += list[i].X * list[i + 1].Y - list[i].Y * list[i + 1].X;
            }
            area = area / 2;
            if (area < 0)
                area = area * -1;
            return area;
        }

        public Bitmap draw(Bitmap image)
        {
            Bitmap clonimage = (Bitmap)image.Clone();
            BitmapData data = clonimage.LockBits(new Rectangle(0, 0, image.Width, image.Height), ImageLockMode.ReadWrite, image.PixelFormat);
            Drawing.Polygon(data, hull, Color.Red);
            clonimage.UnlockBits(data);
            return clonimage;
        }

        static Random random = new Random();
        int Color1, Color2, Color3;
        public Color getRandomColor()
        {
            Color1 = random.Next(0, 255);
            Color2 = random.Next(0, 255);
            Color3 = random.Next(0, 255);
            Color color = Color.FromArgb(Color1, Color2, Color3);
            Console.WriteLine("R: " + Color1 + " G: " + Color2 + " B: " + Color3 + " = " + color.Name);
            return color;
        }
    }
}