I am using a GridBagLayout to (currently) display two rows. I am aware this layout is overkill for this task, but am trying to learn how to use it. The problem is that I have added the two panels to the two separate rows and there is a huge gap around the content (see image and code below):

Image background;
    public Table(){
        super();
        ImageIcon ii = new ImageIcon(this.getClass().getResource("pokerTableV2.png"));
        background = ii.getImage();
        setSize(Constants.FRAME_WIDTH, Constants.TABLE_HEIGHT);
        setLayout(new GridBagLayout());
        GridBagConstraints constraints = new GridBagConstraints();
        constraints.gridx = 0;
        constraints.gridy = 0;
        constraints.fill = GridBagConstraints.HORIZONTAL;
        JButton button = new JButton("hello world");
        JPanel panel1 = new JPanel();
        panel1.setPreferredSize(new Dimension(800,100));
        panel1.add(button, BorderLayout.CENTER);
        panel1.setBackground(Color.yellow);
        add(panel1, constraints);
        constraints.gridx = 0;
        constraints.gridy = 1;
        JPanel middlePanel = new JPanel();
        middlePanel.setPreferredSize(new Dimension(800,350));
        middlePanel.add(button, BorderLayout.CENTER);
        middlePanel.setBackground(Color.blue);
        add(middlePanel, constraints);
    }