I've just written some threaded code which handles huge objects. Even when I clean up everything from an object (without deleting it), it keep eating GBs of RAM. FYI, I reproduced the issue in a smaller environment.
This code creates a structure and an object containing a vector of pointers to the structure. Then I fill the object with some pointers to the structure (created with new). I suppose that the object should own only the size of the pointers, not the size of all pointed structures, but when I run the code, the size of the object uses 300mb.
When I delete all members of the vector and then clean the vector, the memory occupied (but unused now ) remains high. The only way to free this memory seems to be deleting the whole object which contained the vector. Why does the vector keep so much occupied RAM if it was only a vector of pointer? How can I free it without having to delete and re-create the object?
The function -> void f_create_heapCleaner_string creates a string then deletes it. Sometimes the heap is cleaned by this trick.
#include <string>
#include <malloc.h>
#include <vector>
#include <iostream>
using namespace std;
struct struct_test_struct {
string s_internal_data;
};
struct struct_test_struct* BASEDATA_struct_test_struct;
class objhandle_TestStruct__class {
public:
vector<struct struct_test_struct *> v_pointer_TestStruct;
unsigned int ui_v_size;
objhandle_TestStruct__class() {
ui_v_size = 3000;
}
void f_create_heapCleaner_string() {
string * s_tmp = new string();
(*s_tmp).assign(1000000, '*');
(*s_tmp) = "";
delete s_tmp;
}
void f_create_vector(unsigned int ui_size_str) {
cout << " f_create_vector() start " << endl;
malloc_stats();
for (unsigned int ui = 0; ui < ui_v_size; ui++) {
struct struct_test_struct * tmp_newstruct = new struct_test_struct();
(*tmp_newstruct).s_internal_data.assign((ui_size_str + ui), '*');
v_pointer_TestStruct.push_back(tmp_newstruct);
}
cout << " f_create_vector() end " << endl;
malloc_stats();
}
void f_delete_vector_content() {
cout << " f_delete_vector_content() start " << endl;
malloc_stats();
for (unsigned int ui = 0; ui < ui_v_size; ui++) {
delete v_pointer_TestStruct[ui];
}
f_create_heapCleaner_string();
cout << " f_delete_vector_content() end " << endl;
malloc_stats();
}
void f_clear_vector() {
cout << " f_clear_vector() start " << endl;
malloc_stats();
v_pointer_TestStruct.clear();
f_create_heapCleaner_string();
cout << " f_clear_vector() end " << endl;
malloc_stats();
}
void f_RUN_FULL_TEST(unsigned int ui_size_str) {
cout << " .... start test with string size of = " << ui_size_str << endl;
f_create_vector(ui_size_str);
f_delete_vector_content();
f_clear_vector();
}
};
int main(int argc, char**argv) {
objhandle_TestStruct__class * ptr_objhandle_TestStruct__class = new objhandle_TestStruct__class();
(*ptr_objhandle_TestStruct__class).f_RUN_FULL_TEST(100000);
(*ptr_objhandle_TestStruct__class).f_RUN_FULL_TEST(10000);
cout << " DELETE OBJECT start " << endl;
malloc_stats();
delete ptr_objhandle_TestStruct__class;
cout << " DELETE OBJECT finished " << endl;
malloc_stats();
return 0;
}
--- compile with: g++ -o a test.cc
then ./a
Output:
.... start test with string size of = 100000
f_create_vector() start
Arena 0:
system bytes = 135168
in use bytes = 48
Total (incl. mmap):
system bytes = 135168
in use bytes = 48
max mmap regions = 0
max mmap bytes = 0
f_create_vector() end
Arena 0:
system bytes = 309997568
in use bytes = 309972064
Total (incl. mmap):
system bytes = 309997568
in use bytes = 309972064
max mmap regions = 0
max mmap bytes = 0
f_delete_vector_content() start
Arena 0:
system bytes = 309997568
in use bytes = 309972064
Total (incl. mmap):
system bytes = 309997568
in use bytes = 309972064
max mmap regions = 0
max mmap bytes = 0
f_delete_vector_content() end
Arena 0:
system bytes = 309997568
in use bytes = 32832
Total (incl. mmap):
system bytes = 309997568
in use bytes = 32832
max mmap regions = 0
max mmap bytes = 0
f_clear_vector() start
Arena 0:
system bytes = 309997568
in use bytes = 32832
Total (incl. mmap):
system bytes = 309997568
in use bytes = 32832
max mmap regions = 0
max mmap bytes = 0
f_clear_vector() end
Arena 0:
system bytes = 309997568
in use bytes = 32832
Total (incl. mmap):
system bytes = 309997568
in use bytes = 32832
max mmap regions = 0
max mmap bytes = 0
.... start test with string size of = 10000
f_create_vector() start
Arena 0:
system bytes = 309997568
in use bytes = 32832
Total (incl. mmap):
system bytes = 309997568
in use bytes = 32832
max mmap regions = 0
max mmap bytes = 0
f_create_vector() end
Arena 0:
system bytes = 309997568
in use bytes = 40094656
Total (incl. mmap):
system bytes = 309997568
in use bytes = 40094656
max mmap regions = 0
max mmap bytes = 0
f_delete_vector_content() start
Arena 0:
system bytes = 309997568
in use bytes = 40094656
Total (incl. mmap):
system bytes = 309997568
in use bytes = 40094656
max mmap regions = 0
max mmap bytes = 0
f_delete_vector_content() end
Arena 0:
system bytes = 250077184
in use bytes = 32832
Total (incl. mmap):
system bytes = 250077184
in use bytes = 32832
max mmap regions = 0
max mmap bytes = 0
f_clear_vector() start
Arena 0:
system bytes = 250077184
in use bytes = 32832
Total (incl. mmap):
system bytes = 250077184
in use bytes = 32832
max mmap regions = 0
max mmap bytes = 0
f_clear_vector() end
Arena 0:
system bytes = 250077184
in use bytes = 32832
Total (incl. mmap):
system bytes = 250077184
in use bytes = 32832
max mmap regions = 0
max mmap bytes = 0
DELETE OBJECT start
Arena 0:
system bytes = 250077184
in use bytes = 32832
Total (incl. mmap):
system bytes = 250077184
in use bytes = 32832
max mmap regions = 0
max mmap bytes = 0
DELETE OBJECT finished
Arena 0:
system bytes = 135168
in use bytes = 0
Total (incl. mmap):
system bytes = 135168
in use bytes = 0
max mmap regions = 0
max mmap bytes = 0
Thank you, Francesco
-------------- edit using and object container between the object and the structure, this release the memory when deleted.. struct struct_test_struct { string s_internal_data; };
class objstruct_test_struct_OWNER {
public:
vector<struct struct_test_struct *> v_pointer_TestStruct;
};
class objhandle_TestStruct__class {
public:
class objstruct_test_struct_OWNER * ptr_OBJ;
unsigned int ui_v_size;
objhandle_TestStruct__class() {
ui_v_size = 3000;
}
.........
void f_create_vector(unsigned int ui_size_str) {
.....
ptr_OBJ = new objstruct_test_struct_OWNER();
cout << " f_create_vector() start " << endl;
malloc_stats();
for (unsigned int ui = 0; ui < ui_v_size; ui++) {
struct struct_test_struct * tmp_newstruct = new struct_test_struct();
(*tmp_newstruct).s_internal_data.assign((ui_size_str + ui), '*');
(*ptr_OBJ).v_pointer_TestStruct.push_back(tmp_newstruct);
}
.........
void f_clear_vector() {
.........
delete ptr_OBJ;
.........
by this way the program works, this is the output
.... start test with string size of = 100000
f_create_vector() start
Arena 0:
system bytes = 135168
in use bytes = 64
Total (incl. mmap):
system bytes = 135168
in use bytes = 64
max mmap regions = 0
max mmap bytes = 0
f_create_vector() end
Arena 0:
system bytes = 309997568
in use bytes = 309972080
Total (incl. mmap):
system bytes = 309997568
in use bytes = 309972080
max mmap regions = 0
max mmap bytes = 0
f_delete_vector_content() start
Arena 0:
system bytes = 309997568
in use bytes = 309972080
Total (incl. mmap):
system bytes = 309997568
in use bytes = 309972080
max mmap regions = 0
max mmap bytes = 0
f_delete_vector_content() end
Arena 0:
system bytes = 309997568
in use bytes = 32848
Total (incl. mmap):
system bytes = 309997568
in use bytes = 32848
max mmap regions = 0
max mmap bytes = 0
f_clear_vector() start
Arena 0:
system bytes = 309997568
in use bytes = 32848
Total (incl. mmap):
system bytes = 309997568
in use bytes = 32848
max mmap regions = 0
max mmap bytes = 0
f_clear_vector() end
Arena 0:
system bytes = 135168
in use bytes = 32
Total (incl. mmap):
system bytes = 135168
in use bytes = 32
max mmap regions = 1
max mmap bytes = 1007616
.... start test with string size of = 10000
f_create_vector() start
Arena 0:
system bytes = 135168
in use bytes = 64
Total (incl. mmap):
system bytes = 135168
in use bytes = 64
max mmap regions = 1
max mmap bytes = 1007616
f_create_vector() end
Arena 0:
system bytes = 40161280
in use bytes = 40094816
Total (incl. mmap):
system bytes = 40161280
in use bytes = 40094816
max mmap regions = 1
max mmap bytes = 1007616
f_delete_vector_content() start
Arena 0:
system bytes = 40161280
in use bytes = 40094816
Total (incl. mmap):
system bytes = 40161280
in use bytes = 40094816
max mmap regions = 1
max mmap bytes = 1007616
f_delete_vector_content() end
Arena 0:
system bytes = 40161280
in use bytes = 32848
Total (incl. mmap):
system bytes = 40161280
in use bytes = 32848
max mmap regions = 1
max mmap bytes = 1007616
f_clear_vector() start
Arena 0:
system bytes = 40161280
in use bytes = 32848
Total (incl. mmap):
system bytes = 40161280
in use bytes = 32848
max mmap regions = 1
max mmap bytes = 1007616
f_clear_vector() end
Arena 0:
system bytes = 1138688
in use bytes = 32
Total (incl. mmap):
system bytes = 1138688
in use bytes = 32
max mmap regions = 1
max mmap bytes = 1007616
DELETE OBJECT start
Arena 0:
system bytes = 1138688
in use bytes = 32
Total (incl. mmap):
system bytes = 1138688
in use bytes = 32
max mmap regions = 1
max mmap bytes = 1007616
DELETE OBJECT finished
Arena 0:
system bytes = 1138688
in use bytes = 0
Total (incl. mmap):
system bytes = 1138688
in use bytes = 0
max mmap regions = 1
max mmap bytes = 1007616
my issue that i'm only showing here is that objects seems to keep allocated memory for themselves till are not deleted, so seems that only way to free memory for my appliance is to place data into another sub-object and deleting it..
------------------- finally.. i've found that when the program keep a portion of memory mapped for future utilization, this is re-used as we all know and that's ok.. the problem on my multithreaded program was that malloc create these "Arenas" and when 2 malloc are called in the same moment inside a big object malloc create another "Arena" reserving a new map of memory for it.. in my program i finally end having no free ram, with 4 "Arenas" having mapped more than 3gb of ram each, but really using less than 100mb each! so the problem was both memory mapping (~impossible to free manually) and threaded access to memory wich multiply this unused ram into these "Arenas" , so i create a mutex_lock to all thread when accessing to these objects, so these are kept into the same Arena without "wasting" memory (mapped but not used) into multiple arenas..
i hope i have explained a bit my issue and resolution.. hope this could help someone else ;) thank you again, Francesco
----- i still testing.. i've seen also this http://google-perftools.googlecode.com/svn/trunk/doc/tcmalloc.html they say exactly what my issue was.. "In ptmalloc2 memory can never move from one arena to another. This can lead to huge amounts of wasted space. " and create a different memory allocator wich should help..