I'm assuming you want to be able to call dynamically added functions. This code should explain it relatively simply. Note I don't do much error checking and make a few assumptions, don't copy paste this as a solution.
typedef struct
{
    int number;
    int reference;
    lua_State *L;
} TestUserdata;
static int m_newindex( lua_State *L )
{
    /* This is passed three values, first ( at -3 ) is the object, bring this to the front */
    lua_getfenv( L, -3 );
    /* Now bring the second arg forward, the key */
    lua_pushvalue( L, -3 );
    /* And the third arg, the value */
    lua_pushvalue( L, -3 );
    /* And we're done */
    lua_rawset( L, -3 );
    return 0;
}
static int m_tostring( lua_State *L )
{
    lua_pushstring( L, "TestUserdata" );
    return 1;
}
static int callobject( lua_State *L )
{
    /* Grab the object passed, check it's the right type */
    TestUserdata *data = luaL_checkudata( L, 1, "TestUserdata" );
    /* Grab the function environment we gave it in createobject, and look in there for newmethod */
    lua_getfenv( L, -1 );
    lua_pushstring( L, "newmethod" );
    lua_rawget( L, -2 );
    /* Call the function */
    lua_pushinteger( L, data->number );
    lua_call( L, 1, 0 );
    return 0;
}
static const struct luaL_reg userdata_m[] = {
    { "__newindex", m_newindex },
    { "__tostring", m_tostring },
    { NULL, NULL }
};
int main (int argc, char *argv[])
{
    lua_State *L = luaL_newstate();
    luaL_openlibs( L );
    /* Let's create us a userdatum metatable, and fill it up with goodies */
    luaL_newmetatable( L, "TestUserdata" );
    /* Use luaL_register to fill up the metatable for us */
    luaL_register( L, NULL, userdata_m );
    lua_pop( L, 1 ); /* Clean up the stack, we won't need the metatable left here */
    TestUserdata *data = lua_newuserdata( L, sizeof( TestUserdata ) );
    lua_pushvalue( L, -1 ); /* Copy for luaL_ref */
    int ref = luaL_ref( L, LUA_REGISTRYINDEX );
    data->reference = ref;
    data->number = 42;
    data->L = L;
    /* Load the metatable from before and 'give' it to this userdatum */
    luaL_getmetatable( L, "TestUserdata" );
    lua_setmetatable( L, -2 );
    /* Give this object an empty function environment */
    lua_newtable( L );
    lua_setfenv( L, -2 );
    lua_setglobal( L, "test" );
    luaL_dostring( L, "function test.newmethod( num ) print( num ) end" );
    /* Now provided we have the object, we can call any method defined anywhere */
    lua_rawgeti( data->L, LUA_REGISTRYINDEX, data->reference );
    lua_getfenv( data->L, -1 );
    lua_pushstring( data->L, "newmethod" );
    lua_rawget( data->L, -2 );
    lua_remove( data->L, -2 );
    if( lua_isfunction( data->L, -1 ) == 1 )
    {
     lua_pushinteger( data->L, data->number );
     lua_pcall( data->L, 1, 0, 0 );
    }
    lua_close( L );
    return 0;
}
Check that, I think that's what you're after.