Can someone spot the memory leak?

Question

Hello I been trying to find this memory leak for a while now and no luck.

What I have is a server that serves requests for a flash client security acceptance. It only sends out one packet nothing more but has the power to hold over 10,000 concurrent connections? maybe 65,534 if it has too.

Anyways after serving about 210,000+ users in 24 hours the memory usage went up to 74,000 MB from 12,000 MB. I thought it would stabilize at that and stay thinking maybe its some stuff that got unpacked registered and will be re-used. But after 48 hours it's up to 90 MB and at this rate i'll have to restart the server every 2 days to prevent OutOfMemoryException (Haven't got it yet but I know i'll get it with only 300 MB of ram to spare).

Anyways if anyone has any time and would like to help me out i'd really appreciate it, here is the source.

import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
import java.nio.channels.CancelledKeyException;
import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
import java.nio.channels.ServerSocketChannel;
import java.nio.channels.SocketChannel;
import java.nio.channels.spi.SelectorProvider;
import java.util.*;

public class PolicyServer implements Runnable {
    public static final String POLICY_REQUEST = "<policy-file-request/>";
    public static final String POLICY_XML =
        "<?xml version=\"1.0\"?>"
        + "<cross-domain-policy>"
        + "<allow-access-from domain=\"*\" to-ports=\"*\" />"
        + "</cross-domain-policy>"
        + (char)0;


    // The host:port combination to listen on
    private InetAddress hostAddress;
    private int port;

    // The channel on which we'll accept connections
    private ServerSocketChannel serverChannel;

    // The selector we'll be monitoring
    private Selector selector;

    // The buffer into which we'll read data when it's available
    private ByteBuffer readBuffer = ByteBuffer.allocate(255);

    // This decodes raw bytes into ascii data.
    private CharsetDecoder asciiDecoder;

    // A list of PendingChange instances
    private List<ChangeRequest> pendingChanges = new LinkedList<ChangeRequest>();

    // Maps a SocketChannel to a list of ByteBuffer instances
    private Map<SocketChannel, List<ByteBuffer>> pendingData = new HashMap<SocketChannel, List<ByteBuffer>>();

    public PolicyServer(InetAddress hostAddress, int port) throws IOException {
        this.hostAddress = hostAddress;
        this.port = port;
        this.selector = this.initSelector();
        this.asciiDecoder = Charset.forName("US-ASCII").newDecoder().onMalformedInput(
                                CodingErrorAction.REPLACE).onUnmappableCharacter(
                                CodingErrorAction.REPLACE);
    }

    public void send(SocketChannel socket, byte[] data) {
        synchronized (this.pendingChanges) {
            // Indicate we want the interest ops set changed
            this.pendingChanges.add(new ChangeRequest(socket, ChangeRequest.CHANGEOPS, SelectionKey.OP_WRITE));

            // And queue the data we want written
            synchronized (this.pendingData) {
                List<ByteBuffer> queue = (List<ByteBuffer>) this.pendingData.get(socket);
                if (queue == null) {
                    queue = new ArrayList<ByteBuffer>();
                    this.pendingData.put(socket, queue);
                }
                queue.add(ByteBuffer.wrap(data));
            }
        }

        // Finally, wake up our selecting thread so it can make the required changes
        this.selector.wakeup();
    }

    public void run() {
        while (true) {
            try {
                // Process any pending changes
                synchronized (this.pendingChanges) {
                    Iterator changes = this.pendingChanges.iterator();
                    while (changes.hasNext()) {
                        ChangeRequest change = (ChangeRequest) changes.next();
                        if(change == null) continue;
                        switch (change.type) {
                        case ChangeRequest.CHANGEOPS:
                            SelectionKey key = change.socket.keyFor(this.selector);
                            try {
                                if(key!=null)
                                    key.interestOps(change.ops);
                            } catch(Exception ex) {
                                if (key!=null)
                                    key.cancel();
                            }
                        }
                    }
                    this.pendingChanges.clear();
                }

                // Wait for an event one of the registered channels
                this.selector.select();

                // Iterate over the set of keys for which events are available
                Iterator selectedKeys = this.selector.selectedKeys().iterator();
                while (selectedKeys.hasNext()) {
                    SelectionKey key = (SelectionKey) selectedKeys.next();
                    selectedKeys.remove();

                    if (!key.isValid()) {
                        continue;
                    }

                    // Check what event is available and deal with it
                    if (key.isAcceptable()) {
                        this.accept(key);
                    } else if (key.isReadable()) {
                        this.read(key);
                    } else if (key.isWritable()) {
                        this.write(key);
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    private void accept(SelectionKey key) throws IOException {
        // For an accept to be pending the channel must be a server socket channel.
        ServerSocketChannel serverSocketChannel = (ServerSocketChannel) key.channel();

        // Accept the connection and make it non-blocking
        SocketChannel socketChannel = serverSocketChannel.accept();
        Socket socket = socketChannel.socket();
        socketChannel.configureBlocking(false);

        // Register the new SocketChannel with our Selector, indicating
        // we'd like to be notified when there's data waiting to be read
        // also contains a attachment of a new StringBuffer (for storing imcomplete/multi packets)
        socketChannel.register(this.selector, SelectionKey.OP_READ, new StringBuffer());
    }

    private void read(SelectionKey key) throws IOException {
        SocketChannel socketChannel = (SocketChannel) key.channel();

        // Clear out our read buffer so it's ready for new data
        this.readBuffer.clear();

        // Attempt to read off the channel
        int numRead;
        try {
            numRead = socketChannel.read(this.readBuffer);
        } catch (IOException e) {
            // The remote forcibly closed the connection, cancel
            // the selection key and close the channel.
            key.cancel();
            socketChannel.close();
            return;
        }

        if (numRead == -1) {
            // Remote entity shut the socket down cleanly. Do the
            // same from our end and cancel the channel.
            key.channel().close();
            key.cancel();
            return;
        }

        // Grab the StringBuffer we stored as the attachment
        StringBuffer sb = (StringBuffer)key.attachment();

        // Flips the readBuffer by setting the current position of
        // packet stream to beginning.
        // Append the data to the attachment StringBuffer
        this.readBuffer.flip();
        sb.append(this.asciiDecoder.decode(this.readBuffer).toString());
        this.readBuffer.clear();

        // Get the policy request as complete packet
        if(sb.indexOf("\0") != -1) {
            String packets = sb.substring(0, sb.lastIndexOf("\0")+1);
            sb.delete(0, sb.lastIndexOf("\0")+1);

            if(packets.indexOf(POLICY_REQUEST) != -1)
                send(socketChannel, POLICY_XML.getBytes());
        }
    }

    private void write(SelectionKey key) throws IOException {
        SocketChannel socketChannel = (SocketChannel) key.channel();

        synchronized (this.pendingData) {
            List queue = (List) this.pendingData.get(socketChannel);

            // Write until there's not more data ...
            while (!queue.isEmpty()) {
                ByteBuffer buf = (ByteBuffer) queue.get(0);
                try {
                    socketChannel.write(buf);
                    if (buf.remaining() > 0) {
                        // ... or the socket's buffer fills up
                        break;
                    }
                } catch(IOException e) {
                    // The remote forcibly closed the connection, cancel
                    // the selection key and close the channel.
                    key.cancel();
                    socketChannel.close();
                    return;
                }
                queue.remove(0);
            }

            if (queue.isEmpty()) {
                // We wrote away all data, so we're no longer interested
                // in writing on this socket. Switch back to waiting for
                // data.
                try {
                    if (key!=null)
                        key.interestOps(SelectionKey.OP_READ);
                } catch(Exception ex) {
                    if (key!=null)
                        key.cancel();
                }
            }
        }
    }

    private Selector initSelector() throws IOException {
        // Create a new selector
        Selector socketSelector = SelectorProvider.provider().openSelector();

        // Create a new non-blocking server socket channel
        this.serverChannel = ServerSocketChannel.open();
        serverChannel.configureBlocking(false);

        // Bind the server socket to the specified address and port
        InetSocketAddress isa = new InetSocketAddress(this.hostAddress, this.port);
        serverChannel.socket().bind(isa);

        // Register the server socket channel, indicating an interest in
        // accepting new connections
        serverChannel.register(socketSelector, SelectionKey.OP_ACCEPT);

        return socketSelector;
    }

    public static void main(String[] args) {
        try {
            new Thread(new PolicyServer(null, 5556)).start();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

Here is the ChangeRequest class

import java.nio.channels.SocketChannel;

public class ChangeRequest {
    public static final int REGISTER = 1;
    public static final int CHANGEOPS = 2;

    public SocketChannel socket;
    public int type;
    public int ops;

    public ChangeRequest(SocketChannel socket, int type, int ops) {
        this.socket = socket;
        this.type = type;
        this.ops = ops;
    }
}

Answer 1

might not be the only issue, but I don't think you ever call pendingChanges.remove()

Also, you should try using a profiler - it can be easier than manual review.

Answer 2

You have a couple of collections in your class:

private List<ChangeRequest> pendingChanges = ...;
private Map<SocketChannel, List<ByteBuffer>> pendingData = ...;

These would be my first targets of inspection - is any of these growing over time without limit? Are you removing unused elements properly? A few debug printouts at the right places should be able to give you the answer...

If this doesn't give you an obvious solution, I second the opinion about trying a memory profiler.

Answer 3

Are you leaving any connections implicitly open?

If a connection is open with a remote machine, and the variable falls out of scope, it's still active, and won't be garbage collected until the connection is broken (by the other side, or by a network failure).

Answer 4

Here is what I found so far.

After running a fast hacked packet spammer app

import java.io.*;
import java.util.*;
import java.net.*;
public class Spam {

public static final String POLICY_REQUEST = "<policy-file-request/>";
public static int sbyte;
    public static void main(String args[]) {
        while(true) {
            try {
                Socket s = new Socket("127.0.0.1", 5556);
                PrintWriter out = new PrintWriter(s.getOutputStream(), true);
                        BufferedReader in = new BufferedReader(new InputStreamReader(s.getInputStream()));
                out.println(POLICY_REQUEST+(char)0);
                                while(s.getInputStream().available() > 0)
                  sbyte = in.read();
                                //Simulate never closing connection. or until Timeout
                //out.close();
                //in.close();
                //s.close();
            } catch(UnknownHostException uhe) {
            } catch(Exception e) {
            }
        }

    }
}

Added outputs to read() in PolicyServer right under readerBuffer.clear

    System.out.println("Total pendingChanges = " + this.pendingChanges.size());
    System.out.println("Total pendingData = " + this.pendingData.size());
    System.out.println("Total sb = " + sb.length());

Here is the output

Total pendingChanges = 0
Total pendingData = 0
Total sb = 25
Total pendingChanges = 0
Total pendingData = 1
Total sb = 25
Total pendingChanges = 0
Total pendingData = 2
Total sb = 25
Total pendingChanges = 0
Total pendingData = 3
Total sb = 25
Total pendingChanges = 0
Total pendingData = 4
Total sb = 25
Total pendingChanges = 0
Total pendingData = 5
Total sb = 25
Total pendingChanges = 0
Total pendingData = 6
Total sb = 25
Total pendingChanges = 0
Total pendingData = 7
Total sb = 25
Total pendingChanges = 0
Total pendingData = 8
Total sb = 25
Total pendingChanges = 0
Total pendingData = 9
Total sb = 25
Total pendingChanges = 0
Total pendingData = 10
Total sb = 25
Total pendingChanges = 0
Total pendingData = 11
Total sb = 25
Total pendingChanges = 0
Total pendingData = 12
Total sb = 25
Total pendingChanges = 0
Total pendingData = 13
Total sb = 25
Total pendingChanges = 0
Total pendingData = 14
Total sb = 25
Total pendingChanges = 0
Total pendingData = 15
Total sb = 25
Total pendingChanges = 0
Total pendingData = 16
Total sb = 25
Total pendingChanges = 0
Total pendingData = 17
Total sb = 25
Total pendingChanges = 0
Total pendingData = 18
Total sb = 25
Total pendingChanges = 0
Total pendingData = 19
Total sb = 25
Total pendingChanges = 0
Total pendingData = 20
Total sb = 25
Total pendingChanges = 0
Total pendingData = 21
Total sb = 25
Total pendingChanges = 0
Total pendingData = 22
Total sb = 25
Total pendingChanges = 0
Total pendingData = 23
Total sb = 25
Total pendingChanges = 0
Total pendingData = 24
Total sb = 25
Total pendingChanges = 0
Total pendingData = 25
Total sb = 25
Total pendingChanges = 0
Total pendingData = 26
Total sb = 25
Total pendingChanges = 0
Total pendingData = 27
Total sb = 25
Total pendingChanges = 0
Total pendingData = 28
Total sb = 25
Total pendingChanges = 0
Total pendingData = 29
Total sb = 25
Total pendingChanges = 0
Total pendingData = 30
Total sb = 25
Total pendingChanges = 0
Total pendingData = 31
Total sb = 25
Total pendingChanges = 0
Total pendingData = 32
Total sb = 25
Total pendingChanges = 0
Total pendingData = 33
Total sb = 25
Total pendingChanges = 0
Total pendingData = 34
Total sb = 25
Total pendingChanges = 0
Total pendingData = 35
Total sb = 25
Total pendingChanges = 0
Total pendingData = 36
Total sb = 25
Total pendingChanges = 0
Total pendingData = 37
Total sb = 25
Total pendingChanges = 0
Total pendingData = 38
Total sb = 25
Total pendingChanges = 0
Total pendingData = 39
Total sb = 25
Total pendingChanges = 0
Total pendingData = 40
Total sb = 25
Total pendingChanges = 0
Total pendingData = 41
Total sb = 25
Total pendingChanges = 0
Total pendingData = 42
Total sb = 25
Total pendingChanges = 0
Total pendingData = 43
Total sb = 25
Total pendingChanges = 0
Total pendingData = 44
Total sb = 25
Total pendingChanges = 0
Total pendingData = 45
Total sb = 25
Total pendingChanges = 0
Total pendingData = 46
Total sb = 25
Total pendingChanges = 0
Total pendingData = 47
Total sb = 25
Total pendingChanges = 0
Total pendingData = 48
Total sb = 25
Total pendingChanges = 0
Total pendingData = 49
Total sb = 25
Total pendingChanges = 0
Total pendingData = 50
Total sb = 25
Total pendingChanges = 0
Total pendingData = 51
Total sb = 25
Total pendingChanges = 0
Total pendingData = 52
Total sb = 25
Total pendingChanges = 0
Total pendingData = 53
Total sb = 25
Total pendingChanges = 0
Total pendingData = 54
Total sb = 25
Total pendingChanges = 0
Total pendingData = 55
Total sb = 25
Total pendingChanges = 0
Total pendingData = 56
Total sb = 25
Total pendingChanges = 0
Total pendingData = 57
Total sb = 25
Total pendingChanges = 0
Total pendingData = 58
Total sb = 25
Total pendingChanges = 0
Total pendingData = 59
Total sb = 25
Total pendingChanges = 0
Total pendingData = 60
Total sb = 25
Total pendingChanges = 0
Total pendingData = 61
Total sb = 25
Total pendingChanges = 0
Total pendingData = 62
Total sb = 25
Total pendingChanges = 0
Total pendingData = 63
Total sb = 25
Total pendingChanges = 0
Total pendingData = 64
Total sb = 25
Total pendingChanges = 0
Total pendingData = 65
Total sb = 25
Total pendingChanges = 0
Total pendingData = 66
Total sb = 25
Total pendingChanges = 0
Total pendingData = 67
Total sb = 25
Total pendingChanges = 0
Total pendingData = 68
Total sb = 25
Total pendingChanges = 0
Total pendingData = 69
Total sb = 25
Total pendingChanges = 0
Total pendingData = 70
Total sb = 25
Total pendingChanges = 0
Total pendingData = 71
Total sb = 25
Total pendingChanges = 0
Total pendingData = 72
Total sb = 25
Total pendingChanges = 0
Total pendingData = 73
Total sb = 25
Total pendingChanges = 0
Total pendingData = 74
Total sb = 25
Total pendingChanges = 0
Total pendingData = 75
Total sb = 25
Total pendingChanges = 0
Total pendingData = 76
Total sb = 25
Total pendingChanges = 0
Total pendingData = 77
Total sb = 25
Total pendingChanges = 0
Total pendingData = 78
Total sb = 25
Total pendingChanges = 0
Total pendingData = 79
Total sb = 25
Total pendingChanges = 0
Total pendingData = 80
Total sb = 25
Total pendingChanges = 0
Total pendingData = 81
Total sb = 25
Total pendingChanges = 0
Total pendingData = 82
Total sb = 25
Total pendingChanges = 0
Total pendingData = 83
Total sb = 25
Total pendingChanges = 0
Total pendingData = 84
Total sb = 25
Total pendingChanges = 0
Total pendingData = 85
Total sb = 25
Total pendingChanges = 0
Total pendingData = 86
Total sb = 25
Total pendingChanges = 0
Total pendingData = 87
Total sb = 25
Total pendingChanges = 0
Total pendingData = 88
Total sb = 25
Total pendingChanges = 0
Total pendingData = 89
Total sb = 25
Total pendingChanges = 0
Total pendingData = 90
Total sb = 25
Total pendingChanges = 0
Total pendingData = 91
Total sb = 25
Total pendingChanges = 0
Total pendingData = 92
Total sb = 25
Total pendingChanges = 0
Total pendingData = 93
Total sb = 25
Total pendingChanges = 0
Total pendingData = 94
Total sb = 25
Total pendingChanges = 0
Total pendingData = 95
Total sb = 25
Total pendingChanges = 0
Total pendingData = 96
Total sb = 25
Total pendingChanges = 0
Total pendingData = 97
Total sb = 25
Total pendingChanges = 0
Total pendingData = 98
Total sb = 25
Total pendingChanges = 0
Total pendingData = 99
Total sb = 25
Total pendingChanges = 0
Total pendingData = 100
Total sb = 25
Total pendingChanges = 0
Total pendingData = 101
Total sb = 25
Total pendingChanges = 0
Total pendingData = 102
Total sb = 25
Total pendingChanges = 0
Total pendingData = 103
Total sb = 25
Total pendingChanges = 0
Total pendingData = 104
Total sb = 25
Total pendingChanges = 0
Total pendingData = 105
Total sb = 25
Total pendingChanges = 0
Total pendingData = 106
Total sb = 25
Total pendingChanges = 0
Total pendingData = 107
Total sb = 25
Total pendingChanges = 0
Total pendingData = 108
Total sb = 25
Total pendingChanges = 0
Total pendingData = 109
Total sb = 25
Total pendingChanges = 0
Total pendingData = 110
Total sb = 25
Total pendingChanges = 0
Total pendingData = 111
Total sb = 25
Total pendingChanges = 0
Total pendingData = 112
Total sb = 25
Total pendingChanges = 0
Total pendingData = 113
Total sb = 25
Total pendingChanges = 0
Total pendingData = 114
Total sb = 25
Total pendingChanges = 0
Total pendingData = 115
Total sb = 25
Total pendingChanges = 0
Total pendingData = 116
Total sb = 25
Total pendingChanges = 0
Total pendingData = 117
Total sb = 25
Total pendingChanges = 0
Total pendingData = 118
Total sb = 25
Total pendingChanges = 0
Total pendingData = 119
Total sb = 25
Total pendingChanges = 0
Total pendingData = 120
Total sb = 25
Total pendingChanges = 0
Total pendingData = 121
Total sb = 25
Total pendingChanges = 0
Total pendingData = 122
Total sb = 25
Total pendingChanges = 0
Total pendingData = 123
Total sb = 25
Total pendingChanges = 0
Total pendingData = 124
Total sb = 25
Total pendingChanges = 0
Total pendingData = 125
Total sb = 25
Total pendingChanges = 0
Total pendingData = 126
Total sb = 25
Total pendingChanges = 0
Total pendingData = 127
Total sb = 25
Total pendingChanges = 0
Total pendingData = 128
Total sb = 25
Total pendingChanges = 0
Total pendingData = 129
Total sb = 25
Total pendingChanges = 0
Total pendingData = 130
Total sb = 25
Total pendingChanges = 0
Total pendingData = 131
Total sb = 25
Total pendingChanges = 0
Total pendingData = 132
Total sb = 25
Total pendingChanges = 0
Total pendingData = 133
Total sb = 25
Total pendingChanges = 0
Total pendingData = 134
Total sb = 25
Total pendingChanges = 0
Total pendingData = 135
Total sb = 25
Total pendingChanges = 0
Total pendingData = 136
Total sb = 25
Total pendingChanges = 0
Total pendingData = 137
Total sb = 25
Total pendingChanges = 0
Total pendingData = 138
Total sb = 25
Total pendingChanges = 0
Total pendingData = 139
Total sb = 25
Total pendingChanges = 0
Total pendingData = 140
Total sb = 25
Total pendingChanges = 0
Total pendingData = 141
Total sb = 25
Total pendingChanges = 0
Total pendingData = 142
Total sb = 25
Total pendingChanges = 0
Total pendingData = 143
Total sb = 25
Total pendingChanges = 0
Total pendingData = 144
Total sb = 25
Total pendingChanges = 0
Total pendingData = 145
Total sb = 25
Total pendingChanges = 0
Total pendingData = 146
Total sb = 25
Total pendingChanges = 0
Total pendingData = 147
Total sb = 25
Total pendingChanges = 0
Total pendingData = 148
Total sb = 25
Total pendingChanges = 0
Total pendingData = 149
Total sb = 25
Total pendingChanges = 0
Total pendingData = 150
Total sb = 25
Total pendingChanges = 0
Total pendingData = 151
Total sb = 25
Total pendingChanges = 0
Total pendingData = 152
Total sb = 25
Total pendingChanges = 0
Total pendingData = 153
Total sb = 25
Total pendingChanges = 0
Total pendingData = 154
Total sb = 25
Total pendingChanges = 0
Total pendingData = 155
Total sb = 25
Total pendingChanges = 0
Total pendingData = 156
Total sb = 25
Total pendingChanges = 0
Total pendingData = 157
Total sb = 25
Total pendingChanges = 0
Total pendingData = 158
Total sb = 25
Total pendingChanges = 0
Total pendingData = 159
Total sb = 25
Total pendingChanges = 0
Total pendingData = 160
Total sb = 25

Answer 5

If you have Java5 or greater, load up JConsole (in the %JAVA_HOME%/bin/ dir) and it will list the running java processes it can find. connect to your application. and there is a button there to perform Garbage collection.

I have a suspicion that this isn't a memory leak, but memory being allocated to old gen. By performing a manual garbage collection you will perform a full GC, which will clear this out. If it is a true memory leak, then most if not all of this memory will hang around.

If this is a memory leak, be careful of your use of lists, maps etc. Ensure that as a user comes, the information is added, and when they leave it is destroyed.