EDIT: Changed to use WindowFocusListener instead of FocusListener, as well as check for descending components on the focus lost in order to not hide if a child component gains focus.
A simple way would be to add a window focus listener on the dialog that hides it when focus is lost. I don't see the need for modality in this case. For example:
import javax.swing.*;
import java.awt.*;
import java.awt.event.WindowEvent;
import java.awt.event.WindowFocusListener;
import java.awt.event.ActionListener;
import java.awt.event.ActionEvent;
public class ClickAwayDialog extends JDialog {
public ClickAwayDialog(final Frame owner) {
super(owner);
JPanel pnl = new JPanel(new BorderLayout());
pnl.add(new JLabel("Click outside this dialog in the parent frame to close it"), BorderLayout.NORTH);
JButton btn = new JButton("Click Me");
btn.addActionListener(new ActionListener() {
public void actionPerformed(ActionEvent e) {
JOptionPane.showMessageDialog(ClickAwayDialog.this, "New Child Window");
}
});
pnl.add(btn, BorderLayout.CENTER);
this.setContentPane(pnl);
this.pack();
this.setDefaultCloseOperation(WindowConstants.DISPOSE_ON_CLOSE);
this.setLocationRelativeTo(owner);
this.setAlwaysOnTop(true);
this.addWindowFocusListener(new WindowFocusListener() {
public void windowGainedFocus(WindowEvent e) {
//do nothing
}
public void windowLostFocus(WindowEvent e) {
if (SwingUtilities.isDescendingFrom(e.getOppositeWindow(), ClickAwayDialog.this)) {
return;
}
ClickAwayDialog.this.setVisible(false);
}
});
}
public static void main(String[] args) {
SwingUtilities.invokeLater(new Runnable() {
public void run() {
JFrame parent = new JFrame();
parent.setDefaultCloseOperation(WindowConstants.DISPOSE_ON_CLOSE);
parent.setSize(300, 300);
parent.setLocationByPlatform(true);
parent.setVisible(true);
ClickAwayDialog dlg = new ClickAwayDialog(parent);
dlg.setVisible(true);
}
});
}
}