The dialog command prints the result of the user's selections on stderr. This means you will have to capture stderr, not stdout. This is a bit tricky. I'm going to go test this myself, but my guess is that the easiest thing to do is use popen like this:
FILE *dialog = popen("(dialog --menu plus other arguments >/dev/tty) 2>&1");
Then you can read from the dialog file (provided it's not NULL of course).
This works because the argument of popen is actually passed to an invocation of sh. This means that popen really runs sh -c <argument of popen> so all the standard shell redirections work. So you use parenthesis to get exactly what you want, which is for the dialog program itself to send its output to its controlling terminal, and for its stderr to be redirected to the place where you can read it with popen.
There is another method that has the same disadvantages as the popen solution, and has the additional disadvantage of requiring you to open and read another file after dialog finishes. But it has the advantage of simplicity. Unfortunately it also requires you to be able to write to the filesystem, and the most natural place to do that (/tmp) is fraught with security issues relating to making sure someone else doesn't somehow hijack your file. That method is to do system("dialog --menu plus other arguments 2>temp_file");. Then you read from temp_file when it's done.
These are both a bit ugly, especially since dialog takes a lot of arguments that are likely to have shell metacharacters in them. So even if the above work, I would highly recommend using a combination of pipe, fork, execvp, fdopen, and waitpid to get the result you're after.
A skeleton for that would look something like this:
#include <stdio.h>
#include <stddef.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <signal.h>
int main(int argc, const char *argv[])
{
const char *dia_args[] = {
"dialog",
"--output-fd",
NULL,
"--menu",
"Hi there",
"60", "15", "15",
"t1", "i1",
"t2", "i2",
"t3", "i3",
"t4", "i4",
"t5", "i5",
NULL
};
int pipefds[2];
if (pipe(pipefds) < 0) {
perror("pipe failed");
return 1;
} else {
const pid_t child = fork();
if (child < 0) {
perror("fork failed");
return 1;
} else if (child == 0) {
char pipefdstr[60];
close(pipefds[0]);
if (snprintf(pipefdstr, sizeof(pipefdstr) - 1, "%u", pipefds[1]) < 0) {
perror("snprintf failed");
return 1;
} else {
pipefdstr[sizeof(pipefdstr) - 1] = '\0'; /* Protect against bugs in snprintf */
dia_args[2] = pipefdstr;
execvp(dia_args[0], dia_args);
perror("Unable to exec dialog command");
return 1;
}
} else { /* child > 0 */
FILE *dialog = fdopen(pipefds[0], "r");
char inbuf[200];
int waitresult = 0;
if (dialog == NULL) {
perror("Unable to fdopen");
kill(child, SIGKILL);
return 1;
}
close(pipefds[1]);
while (fgets(inbuf, sizeof(inbuf) - 1, dialog)) {
inbuf[sizeof(inbuf) - 1] = '\0';
printf("Got [%s] from dialog.\n", inbuf);
}
fclose(dialog);
if (waitpid(child, &waitresult, 0) < 0) {
perror("waitpid failed");
return 1;
}
if (!WIFEXITED(waitresult) || (WEXITSTATUS(waitresult) != 0)) {
fprintf(stderr, "dialog exited abnormally.");
return 1;
}
}
}
return 0;
}