cvUndistort2 () and cvRemap () crash

Question

Hi!

I was doing example 11-1 of "Learning OpenCV" by Bradski. Unfortunately the given example doesn't work on my computer.

The program is supposed to calibrate camera using chessboard and then output undistorted video output from the camera.

The calibration part works fine, the problem arises when the program tries to undistort the image. I tried both cvUndistort2() and cvRemap(), in both cases the output windows freeze and the the program crashes, so that I have to force close it.

Here is the code:

#include <cv.h>
#include <highgui.h>
#include <stdio.h>
#include <stdlib.h>
#include <iostream>

int n_boards = 0;
int board_dt = 15;
int board_w;
int board_h;
int main(int argc, char* argv[]) {

  CvCapture* capture;
  /*
  if(argc != 5){
    printf("\nERROR: Wrong number of input parameters");
    help();
    return -1;
  }
  */

  board_w  = 7;//atoi(argv[1]);
  board_h  = 7;//atoi(argv[2]);
  n_boards = 8;//atoi(argv[3]);

  CvSize board_sz = cvSize( board_w, board_h );
  capture = cvCreateCameraCapture( 0 );
  if(!capture) { printf("\nCouldn't open the camera\n"); return -1;}

  cvNamedWindow( "Calibration" );
  IplImage *image = cvQueryFrame( capture );

  int board_n  = board_w * board_h;

  //ALLOCATE STORAGE
  CvMat* image_points      = cvCreateMat(n_boards*board_n,2,CV_32FC1);
  CvMat* object_points     = cvCreateMat(n_boards*board_n,3,CV_32FC1);
  CvMat* point_counts      = cvCreateMat(n_boards,1,CV_32SC1);
  CvMat* intrinsic_matrix  = cvCreateMat(3,3,CV_32FC1);
  CvMat* distortion_coeffs = cvCreateMat(5,1,CV_32FC1);

  CvPoint2D32f* corners = new CvPoint2D32f[ board_n ];
  int corner_count;
  int successes = 0;
  int step, frame = 0;

  IplImage *gray_image = cvCreateImage(cvGetSize(image),8,1);//subpixel

  // CAPTURE CORNER VIEWS LOOP UNTIL WE'VE GOT n_boards
  // SUCCESSFUL CAPTURES (ALL CORNERS ON THE BOARD ARE FOUND)
  while (successes < n_boards) {
        //Skip every board_dt frames to allow user to move chessboard
        if((frame++ % board_dt) == 0) {
           //Find chessboard corners:
           int found = cvFindChessboardCorners(
                    image, board_sz, corners, &corner_count,
                    CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FILTER_QUADS
           );

           //Get Subpixel accuracy on those corners
           cvCvtColor(image, gray_image, CV_BGR2GRAY);
           cvFindCornerSubPix(gray_image, corners, corner_count,
                      cvSize(11,11),cvSize(-1,-1), cvTermCriteria(
                      CV_TERMCRIT_EPS+CV_TERMCRIT_ITER, 30, 0.1 ));

           //Draw it
           cvDrawChessboardCorners(image, board_sz, corners,
                      corner_count, found);
           cvShowImage( "Calibration", image );

           // If we got a good board, add it to our data
           if( corner_count == board_n ) {
               step = successes*board_n;
               for( int i=step, j=0; j<board_n; ++i,++j ) {
                   CV_MAT_ELEM(*image_points, float,i,0) = corners[j].x;
                   CV_MAT_ELEM(*image_points, float,i,1) = corners[j].y;
                   CV_MAT_ELEM(*object_points,float,i,0) = j/board_w;
                   CV_MAT_ELEM(*object_points,float,i,1) = j%board_w;
                   CV_MAT_ELEM(*object_points,float,i,2) = 0.0f;
               }
               CV_MAT_ELEM(*point_counts, int,successes,0) = board_n;
               successes++;
           }
        } //end skip board_dt between chessboard capture
       //Handle pause/unpause and ESC
       int c = cvWaitKey(15);
       if (c == 'p'){
           c = 0;
           while(c != 'p' && c != 27){
               c = cvWaitKey(250);
           }
       }
       if(c == 27)
           return 0;
       image = cvQueryFrame( capture ); //Get next image
  } //END COLLECTION WHILE LOOP.

   //ALLOCATE MATRICES ACCORDING TO HOW MANY CHESSBOARDS FOUND
   CvMat* object_points2 = cvCreateMat(successes*board_n,3,CV_32FC1);
   CvMat* image_points2 = cvCreateMat(successes*board_n,2,CV_32FC1);
   CvMat* point_counts2 = cvCreateMat(successes,1,CV_32SC1);
   //TRANSFER THE POINTS INTO THE CORRECT SIZE MATRICES
   for(int i = 0; i<successes*board_n; ++i) {
       CV_MAT_ELEM( *image_points2, float, i, 0) = CV_MAT_ELEM( *image_points, float, i, 0);
       CV_MAT_ELEM( *image_points2, float, i, 1) = CV_MAT_ELEM( *image_points, float, i, 1);
       CV_MAT_ELEM( *object_points2, float, i, 0) = CV_MAT_ELEM( *object_points, float, i, 0) ;
       CV_MAT_ELEM( *object_points2, float, i, 1) = CV_MAT_ELEM( *object_points, float, i, 1) ;
       CV_MAT_ELEM( *object_points2, float, i, 2) = CV_MAT_ELEM( *object_points, float, i, 2) ;
   }
   for(int i=0; i<successes; ++i){ //These are all the same number
       CV_MAT_ELEM( *point_counts2, int, i, 0) = CV_MAT_ELEM( *point_counts, int, i, 0);
   }

   // Initialize the intrinsic matrix such that the two focal
   // lengths have a ratio of 1.0
   CV_MAT_ELEM( *intrinsic_matrix, float, 0, 0 ) = 1.0f;
   CV_MAT_ELEM( *intrinsic_matrix, float, 1, 1 ) = 1.0f;
   //CALIBRATE THE CAMERA!
   cvCalibrateCamera2(
       object_points2, image_points2,
       point_counts2, cvGetSize( image ),
       intrinsic_matrix, distortion_coeffs,
       NULL, NULL,0 //CV_CALIB_FIX_ASPECT_RATIO
   );

   // SAVE THE INTRINSICS AND DISTORTIONS
   cvSave("Intrinsics.xml",intrinsic_matrix);
   cvSave("Distortion.xml",distortion_coeffs);

   cvReleaseMat(&object_points);
   cvReleaseMat(&image_points);
   cvReleaseMat(&point_counts);
   cvReleaseMat(&object_points2);
   cvReleaseMat(&image_points2);
   cvReleaseMat(&point_counts2);
   cvReleaseMat(&intrinsic_matrix);
   cvReleaseMat(&distortion_coeffs);

   // EXAMPLE OF LOADING THESE MATRICES BACK IN:
   CvMat *intrinsic = (CvMat*)cvLoad("Intrinsics.xml");
   CvMat *distortion= (CvMat*)cvLoad("Distortion.xml");

   // Build the undistort map that we will use for all
   // subsequent frames.
   IplImage* mapx = cvCreateImage( cvGetSize(image), IPL_DEPTH_32F, 1 );
   IplImage* mapy = cvCreateImage( cvGetSize(image), IPL_DEPTH_32F, 1 );
   cvInitUndistortMap(
       intrinsic,
       distortion,
       mapx,
       mapy
   );

   // Just run the camera to the screen, now showing the raw and
   // the undistorted image
   cvNamedWindow( "Undistort" );
   while(image) {
       IplImage *t = cvCloneImage(image);

       // PROBLEM HERE!
       //cvRemap( image, t, mapx, mapy ); // Undistort image
       cvUndistort2(image, t, intrinsic, distortion);

       cvShowImage( "Calibration", image ); // Show raw image
       cvShowImage( "Undistort", t); // Show corrected image
       cvReleaseImage(&t);
       //Handle pause/unpause and ESC
       int c = cvWaitKey(30);
       if(c == 'p') {
           c = 0;
           while(c != 'p' && c != 27) {
               c = cvWaitKey(250);
           }
       }
       if(c == 27)
           break;
       image = cvQueryFrame( capture );
   }
   cvReleaseMat(&intrinsic);
   cvReleaseMat(&distortion);

   cvReleaseImage(&image);
   cvReleaseImage(&gray_image);
   cvReleaseImage(&mapx);
   cvReleaseImage(&mapy);
   cvDestroyWindow("Calibration");
   cvDestroyWindow("Undistort");
   return 0;
}

Here are the calibration results saved in Intrinsics.xml and Distortion.xml:

Intrinsics.xml:

<?xml version="1.0"?>
<opencv_storage>
<Intrinsics type_id="opencv-matrix">
  <rows>3</rows>
  <cols>3</cols>
  <dt>f</dt>
  <data>
    649.64843750 0. 288.47882080 0. 647.89129639 271.92953491 0. 0. 1.</data></Intrinsics>
</opencv_storage>

Distortion.xml:

<?xml version="1.0"?>
<opencv_storage>
<Distortion type_id="opencv-matrix">
  <rows>5</rows>
  <cols>1</cols>
  <dt>f</dt>
  <data>
    -0.37764871 22.05950546 0.06449836 -0.03288389 -209.10910034</data></Distortion>
</opencv_storage>

I use OpenCV2.0, Eclipse, Windows Vista. Camera is a webcam of the laptop.

Answer 1

The problem is that cvUndistort2 and cvRemap receive 8bit images only. Hence, in order to process color images, one must use cvSplit and cvMerge:

   IplImage *r = cvCreateImage(cvGetSize(image),8,1);//subpixel
   IplImage *g = cvCreateImage(cvGetSize(image),8,1);//subpixel
   IplImage *b = cvCreateImage(cvGetSize(image),8,1);//subpixel

   while(image) {
       cvShowImage( "Calibration", image ); // Show raw image
       //cvCvtColor(image, gray_image, CV_BGR2GRAY);
       cvSplit(image, r,g,b, NULL);

       cvRemap( r, r, mapx, mapy ); // Undistort image
       cvRemap( g, g, mapx, mapy ); // Undistort image
       cvRemap( b, b, mapx, mapy ); // Undistort image

       cvMerge(r,g,b, NULL, image);
   cvShowImage( "Undistort", image); // Show corrected image